loading
Generated 2025-05-21T18:04:21+00:00

All Files ( 96.12% covered at 95751.32 hits/line )

105 files in total.
7697 relevant lines, 7398 lines covered and 299 lines missed. ( 96.12% )
File % covered Lines Relevant Lines Lines covered Lines missed Avg. Hits / Line
lib/httpx.rb 100.00 % 66 39 39 0 807.64
lib/httpx/adapters/datadog.rb 87.04 % 345 162 141 21 40.91
lib/httpx/adapters/faraday.rb 98.13 % 298 160 157 3 133.83
lib/httpx/adapters/sentry.rb 100.00 % 121 62 62 0 111.61
lib/httpx/adapters/webmock.rb 100.00 % 175 93 93 0 96.02
lib/httpx/altsvc.rb 96.39 % 163 83 80 3 243.23
lib/httpx/buffer.rb 100.00 % 61 27 27 0 403525.00
lib/httpx/callbacks.rb 100.00 % 35 19 19 0 167424.63
lib/httpx/chainable.rb 95.65 % 106 46 44 2 1083.83
lib/httpx/connection.rb 92.16 % 953 485 447 38 183702.35
lib/httpx/connection/http1.rb 89.14 % 400 221 197 24 3870.14
lib/httpx/connection/http2.rb 95.09 % 452 265 252 13 286134.19
lib/httpx/domain_name.rb 95.56 % 145 45 43 2 247.22
lib/httpx/errors.rb 97.62 % 109 42 41 1 87.02
lib/httpx/extensions.rb 67.86 % 59 28 19 9 420.07
lib/httpx/headers.rb 100.00 % 176 71 71 0 18740.15
lib/httpx/io.rb 100.00 % 11 5 5 0 27.00
lib/httpx/io/ssl.rb 96.55 % 166 87 84 3 2376.17
lib/httpx/io/tcp.rb 91.15 % 208 113 103 10 8121.59
lib/httpx/io/udp.rb 100.00 % 62 35 35 0 376.51
lib/httpx/io/unix.rb 94.29 % 70 35 33 2 20.49
lib/httpx/loggable.rb 100.00 % 53 20 20 0 583788.80
lib/httpx/options.rb 98.79 % 375 165 163 2 18243.16
lib/httpx/parser/http1.rb 100.00 % 186 109 109 0 7848.85
lib/httpx/plugins/auth.rb 100.00 % 25 9 9 0 24.00
lib/httpx/plugins/auth/basic.rb 100.00 % 20 10 10 0 86.60
lib/httpx/plugins/auth/digest.rb 100.00 % 102 61 61 0 127.85
lib/httpx/plugins/auth/ntlm.rb 100.00 % 35 19 19 0 2.95
lib/httpx/plugins/auth/socks5.rb 100.00 % 22 11 11 0 23.82
lib/httpx/plugins/aws_sdk_authentication.rb 100.00 % 109 43 43 0 13.21
lib/httpx/plugins/aws_sigv4.rb 100.00 % 237 121 121 0 101.93
lib/httpx/plugins/basic_auth.rb 100.00 % 29 12 12 0 35.33
lib/httpx/plugins/brotli.rb 100.00 % 50 25 25 0 10.80
lib/httpx/plugins/callbacks.rb 100.00 % 115 55 55 0 132.93
lib/httpx/plugins/circuit_breaker.rb 100.00 % 145 66 66 0 74.82
lib/httpx/plugins/circuit_breaker/circuit.rb 100.00 % 100 47 47 0 58.21
lib/httpx/plugins/circuit_breaker/circuit_store.rb 100.00 % 53 23 23 0 98.48
lib/httpx/plugins/content_digest.rb 100.00 % 202 102 102 0 79.51
lib/httpx/plugins/cookies.rb 100.00 % 107 52 52 0 122.63
lib/httpx/plugins/cookies/cookie.rb 100.00 % 174 77 77 0 318.21
lib/httpx/plugins/cookies/jar.rb 100.00 % 95 46 46 0 260.85
lib/httpx/plugins/cookies/set_cookie_parser.rb 100.00 % 143 72 72 0 156.88
lib/httpx/plugins/digest_auth.rb 100.00 % 65 29 29 0 99.24
lib/httpx/plugins/expect.rb 100.00 % 118 56 56 0 87.61
lib/httpx/plugins/follow_redirects.rb 100.00 % 231 108 108 0 188895.89
lib/httpx/plugins/grpc.rb 100.00 % 280 133 133 0 117.79
lib/httpx/plugins/grpc/call.rb 90.91 % 63 33 30 3 40.18
lib/httpx/plugins/grpc/grpc_encoding.rb 97.87 % 90 47 46 1 76.85
lib/httpx/plugins/grpc/message.rb 95.83 % 55 24 23 1 39.50
lib/httpx/plugins/h2c.rb 94.92 % 117 59 56 3 14.34
lib/httpx/plugins/ntlm_auth.rb 100.00 % 60 30 30 0 3.97
lib/httpx/plugins/oauth.rb 100.00 % 175 87 87 0 70.23
lib/httpx/plugins/persistent.rb 100.00 % 73 28 28 0 212.89
lib/httpx/plugins/proxy.rb 98.01 % 308 151 148 3 323.60
lib/httpx/plugins/proxy/http.rb 100.00 % 184 104 104 0 185.74
lib/httpx/plugins/proxy/socks4.rb 97.47 % 135 79 77 2 177.23
lib/httpx/plugins/proxy/socks5.rb 99.12 % 194 113 112 1 277.71
lib/httpx/plugins/proxy/ssh.rb 92.31 % 92 52 48 4 8.19
lib/httpx/plugins/push_promise.rb 100.00 % 81 41 41 0 10.49
lib/httpx/plugins/query.rb 100.00 % 35 15 15 0 10.07
lib/httpx/plugins/rate_limiter.rb 100.00 % 55 18 18 0 41.72
lib/httpx/plugins/response_cache.rb 100.00 % 331 143 143 0 187.59
lib/httpx/plugins/response_cache/file_store.rb 100.00 % 140 72 72 0 166.58
lib/httpx/plugins/response_cache/store.rb 100.00 % 33 16 16 0 153.94
lib/httpx/plugins/retries.rb 96.91 % 228 97 94 3 205010.22
lib/httpx/plugins/ssrf_filter.rb 100.00 % 145 61 61 0 144.25
lib/httpx/plugins/stream.rb 97.78 % 183 90 88 2 116.06
lib/httpx/plugins/stream_bidi.rb 99.30 % 315 142 141 1 88.27
lib/httpx/plugins/upgrade.rb 100.00 % 78 34 34 0 49.32
lib/httpx/plugins/upgrade/h2.rb 91.67 % 54 24 22 2 8.29
lib/httpx/plugins/webdav.rb 100.00 % 86 39 39 0 23.10
lib/httpx/plugins/xml.rb 100.00 % 76 34 34 0 84.35
lib/httpx/pmatch_extensions.rb 100.00 % 33 17 17 0 28.71
lib/httpx/pool.rb 100.00 % 185 89 89 0 4871.98
lib/httpx/punycode.rb 100.00 % 22 9 9 0 18.78
lib/httpx/request.rb 100.00 % 317 132 132 0 6722.19
lib/httpx/request/body.rb 100.00 % 153 66 66 0 3222.94
lib/httpx/resolver.rb 100.00 % 161 82 82 0 1827.05
lib/httpx/resolver/https.rb 86.11 % 254 144 124 20 34.19
lib/httpx/resolver/multi.rb 88.24 % 93 51 45 6 3638.71
lib/httpx/resolver/native.rb 89.00 % 538 309 275 34 1150.64
lib/httpx/resolver/resolver.rb 84.52 % 169 84 71 13 1795.38
lib/httpx/resolver/system.rb 78.99 % 253 138 109 29 17.30
lib/httpx/response.rb 100.00 % 304 115 115 0 1938.55
lib/httpx/response/body.rb 100.00 % 242 109 109 0 2979.97
lib/httpx/response/buffer.rb 92.19 % 120 64 59 5 1578.28
lib/httpx/selector.rb 93.52 % 221 108 101 7 1990874.16
lib/httpx/session.rb 95.60 % 553 273 261 12 122083.50
lib/httpx/session_extensions.rb 100.00 % 29 14 14 0 6.64
lib/httpx/timers.rb 93.94 % 133 66 62 4 3412979.29
lib/httpx/transcoder.rb 100.00 % 91 52 52 0 281.75
lib/httpx/transcoder/body.rb 100.00 % 43 26 26 0 965.31
lib/httpx/transcoder/chunker.rb 100.00 % 115 66 66 0 222.02
lib/httpx/transcoder/deflate.rb 100.00 % 37 20 20 0 30.70
lib/httpx/transcoder/form.rb 100.00 % 80 42 42 0 439.62
lib/httpx/transcoder/gzip.rb 100.00 % 71 40 40 0 95.60
lib/httpx/transcoder/json.rb 100.00 % 71 33 33 0 42.61
lib/httpx/transcoder/multipart.rb 100.00 % 17 10 10 0 921.70
lib/httpx/transcoder/multipart/decoder.rb 94.05 % 141 84 79 5 29.20
lib/httpx/transcoder/multipart/encoder.rb 100.00 % 120 70 70 0 1945.89
lib/httpx/transcoder/multipart/mime_type_detector.rb 92.11 % 78 38 35 3 177.47
lib/httpx/transcoder/multipart/part.rb 100.00 % 35 18 18 0 505.17
lib/httpx/transcoder/utils/body_reader.rb 92.00 % 46 25 23 2 99.88
lib/httpx/transcoder/utils/deflater.rb 100.00 % 75 37 37 0 102.92
lib/httpx/utils.rb 100.00 % 75 39 39 0 550164.59

lib/httpx.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "httpx/version"
  3. # Top-Level Namespace
  4. #
  5. 27 module HTTPX
  6. 27 EMPTY = [].freeze
  7. 27 EMPTY_HASH = {}.freeze
  8. # All plugins should be stored under this module/namespace. Can register and load
  9. # plugins.
  10. #
  11. 27 module Plugins
  12. 27 @plugins = {}
  13. 27 @plugins_mutex = Thread::Mutex.new
  14. # Loads a plugin based on a name. If the plugin hasn't been loaded, tries to load
  15. # it from the load path under "httpx/plugins/" directory.
  16. #
  17. 27 def self.load_plugin(name)
  18. 5712 h = @plugins
  19. 5712 m = @plugins_mutex
  20. 11424 unless (plugin = m.synchronize { h[name] })
  21. 272 require "httpx/plugins/#{name}"
  22. 544 raise "Plugin #{name} hasn't been registered" unless (plugin = m.synchronize { h[name] })
  23. end
  24. 5712 plugin
  25. end
  26. # Registers a plugin (+mod+) in the central store indexed by +name+.
  27. #
  28. 27 def self.register_plugin(name, mod)
  29. 336 h = @plugins
  30. 336 m = @plugins_mutex
  31. 640 m.synchronize { h[name] = mod }
  32. end
  33. end
  34. end
  35. 27 require "httpx/extensions"
  36. 27 require "httpx/errors"
  37. 27 require "httpx/utils"
  38. 27 require "httpx/punycode"
  39. 27 require "httpx/domain_name"
  40. 27 require "httpx/altsvc"
  41. 27 require "httpx/callbacks"
  42. 27 require "httpx/loggable"
  43. 27 require "httpx/transcoder"
  44. 27 require "httpx/timers"
  45. 27 require "httpx/pool"
  46. 27 require "httpx/headers"
  47. 27 require "httpx/request"
  48. 27 require "httpx/response"
  49. 27 require "httpx/options"
  50. 27 require "httpx/chainable"
  51. 27 require "httpx/session"
  52. 27 require "httpx/session_extensions"
  53. # load integrations when possible
  54. 27 require "httpx/adapters/datadog" if defined?(DDTrace) || defined?(Datadog::Tracing)
  55. 27 require "httpx/adapters/sentry" if defined?(Sentry)
  56. 27 require "httpx/adapters/webmock" if defined?(WebMock)

lib/httpx/adapters/datadog.rb

87.04% lines covered

162 relevant lines. 141 lines covered and 21 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "datadog/tracing/contrib/integration"
  3. 6 require "datadog/tracing/contrib/configuration/settings"
  4. 6 require "datadog/tracing/contrib/patcher"
  5. 6 module Datadog::Tracing
  6. 6 module Contrib
  7. 6 module HTTPX
  8. 6 DATADOG_VERSION = defined?(::DDTrace) ? ::DDTrace::VERSION : ::Datadog::VERSION
  9. 6 METADATA_MODULE = Datadog::Tracing::Metadata
  10. 6 TYPE_OUTBOUND = Datadog::Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
  11. 6 TAG_BASE_SERVICE = Datadog::Tracing::Contrib::Ext::Metadata::TAG_BASE_SERVICE
  12. 6 TAG_PEER_HOSTNAME = Datadog::Tracing::Metadata::Ext::TAG_PEER_HOSTNAME
  13. 6 TAG_KIND = Datadog::Tracing::Metadata::Ext::TAG_KIND
  14. 6 TAG_CLIENT = Datadog::Tracing::Metadata::Ext::SpanKind::TAG_CLIENT
  15. 6 TAG_COMPONENT = Datadog::Tracing::Metadata::Ext::TAG_COMPONENT
  16. 6 TAG_OPERATION = Datadog::Tracing::Metadata::Ext::TAG_OPERATION
  17. 6 TAG_URL = Datadog::Tracing::Metadata::Ext::HTTP::TAG_URL
  18. 6 TAG_METHOD = Datadog::Tracing::Metadata::Ext::HTTP::TAG_METHOD
  19. 6 TAG_TARGET_HOST = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_HOST
  20. 6 TAG_TARGET_PORT = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_PORT
  21. 6 TAG_STATUS_CODE = Datadog::Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE
  22. # HTTPX Datadog Plugin
  23. #
  24. # Enables tracing for httpx requests.
  25. #
  26. # A span will be created for each request transaction; the span is created lazily only when
  27. # buffering a request, and it is fed the start time stored inside the tracer object.
  28. #
  29. 6 module Plugin
  30. 6 module RequestTracer
  31. 6 extend Contrib::HttpAnnotationHelper
  32. 6 module_function
  33. 6 SPAN_REQUEST = "httpx.request"
  34. # initializes tracing on the +request+.
  35. 6 def call(request)
  36. 258 return unless configuration(request).enabled
  37. 84 span = nil
  38. # request objects are reused, when already buffered requests get rerouted to a different
  39. # connection due to connection issues, or when they already got a response, but need to
  40. # be retried. In such situations, the original span needs to be extended for the former,
  41. # while a new is required for the latter.
  42. 84 request.on(:idle) do
  43. 18 span = nil
  44. end
  45. # the span is initialized when the request is buffered in the parser, which is the closest
  46. # one gets to actually sending the request.
  47. 84 request.on(:headers) do
  48. 96 next if span
  49. 96 span = initialize_span(request, now)
  50. end
  51. 84 request.on(:response) do |response|
  52. 96 unless span
  53. 6 next unless response.is_a?(::HTTPX::ErrorResponse) && response.error.respond_to?(:connection)
  54. # handles the case when the +error+ happened during name resolution, which means
  55. # that the tracing start point hasn't been triggered yet; in such cases, the approximate
  56. # initial resolving time is collected from the connection, and used as span start time,
  57. # and the tracing object in inserted before the on response callback is called.
  58. 6 span = initialize_span(request, response.error.connection.init_time)
  59. end
  60. 96 finish(response, span)
  61. end
  62. end
  63. 6 def finish(response, span)
  64. 96 if response.is_a?(::HTTPX::ErrorResponse)
  65. 6 span.set_error(response.error)
  66. else
  67. 90 span.set_tag(TAG_STATUS_CODE, response.status.to_s)
  68. 90 span.set_error(::HTTPX::HTTPError.new(response)) if response.status >= 400 && response.status <= 599
  69. 90 span.set_tags(
  70. Datadog.configuration.tracing.header_tags.response_tags(response.headers.to_h)
  71. )
  72. end
  73. 96 span.finish
  74. end
  75. # return a span initialized with the +@request+ state.
  76. 6 def initialize_span(request, start_time)
  77. 102 verb = request.verb
  78. 102 uri = request.uri
  79. 102 config = configuration(request)
  80. 102 span = create_span(request, config, start_time)
  81. 102 span.resource = verb
  82. # Tag original global service name if not used
  83. 102 span.set_tag(TAG_BASE_SERVICE, Datadog.configuration.service) if span.service != Datadog.configuration.service
  84. 102 span.set_tag(TAG_KIND, TAG_CLIENT)
  85. 102 span.set_tag(TAG_COMPONENT, "httpx")
  86. 102 span.set_tag(TAG_OPERATION, "request")
  87. 102 span.set_tag(TAG_URL, request.path)
  88. 102 span.set_tag(TAG_METHOD, verb)
  89. 102 span.set_tag(TAG_TARGET_HOST, uri.host)
  90. 102 span.set_tag(TAG_TARGET_PORT, uri.port)
  91. 102 span.set_tag(TAG_PEER_HOSTNAME, uri.host)
  92. # Tag as an external peer service
  93. # span.set_tag(TAG_PEER_SERVICE, span.service)
  94. 102 if config[:distributed_tracing]
  95. 96 propagate_trace_http(
  96. Datadog::Tracing.active_trace,
  97. request.headers
  98. )
  99. end
  100. # Set analytics sample rate
  101. 102 if Contrib::Analytics.enabled?(config[:analytics_enabled])
  102. 12 Contrib::Analytics.set_sample_rate(span, config[:analytics_sample_rate])
  103. end
  104. 102 span.set_tags(
  105. Datadog.configuration.tracing.header_tags.request_tags(request.headers.to_h)
  106. )
  107. 102 span
  108. rescue StandardError => e
  109. Datadog.logger.error("error preparing span for http request: #{e}")
  110. Datadog.logger.error(e.backtrace)
  111. end
  112. 6 def now
  113. 96 ::Datadog::Core::Utils::Time.now.utc
  114. end
  115. 6 def configuration(request)
  116. 360 Datadog.configuration.tracing[:httpx, request.uri.host]
  117. end
  118. 6 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("2.0.0")
  119. 3 def propagate_trace_http(trace, headers)
  120. 48 Datadog::Tracing::Contrib::HTTP.inject(trace, headers)
  121. end
  122. 3 def create_span(request, configuration, start_time)
  123. 51 Datadog::Tracing.trace(
  124. SPAN_REQUEST,
  125. service: service_name(request.uri.host, configuration),
  126. type: TYPE_OUTBOUND,
  127. start_time: start_time
  128. )
  129. end
  130. else
  131. 3 def propagate_trace_http(trace, headers)
  132. 48 Datadog::Tracing::Propagation::HTTP.inject!(trace.to_digest, headers)
  133. end
  134. 3 def create_span(request, configuration, start_time)
  135. 51 Datadog::Tracing.trace(
  136. SPAN_REQUEST,
  137. service: service_name(request.uri.host, configuration),
  138. span_type: TYPE_OUTBOUND,
  139. start_time: start_time
  140. )
  141. end
  142. end
  143. end
  144. 6 module RequestMethods
  145. # intercepts request initialization to inject the tracing logic.
  146. 6 def initialize(*)
  147. 258 super
  148. 258 return unless Datadog::Tracing.enabled?
  149. 258 RequestTracer.call(self)
  150. end
  151. end
  152. 6 module ConnectionMethods
  153. 6 attr_reader :init_time
  154. 6 def initialize(*)
  155. 245 super
  156. 245 @init_time = ::Datadog::Core::Utils::Time.now.utc
  157. end
  158. end
  159. end
  160. 6 module Configuration
  161. # Default settings for httpx
  162. #
  163. 6 class Settings < Datadog::Tracing::Contrib::Configuration::Settings
  164. 6 DEFAULT_ERROR_HANDLER = lambda do |response|
  165. Datadog::Ext::HTTP::ERROR_RANGE.cover?(response.status)
  166. end
  167. 6 option :service_name, default: "httpx"
  168. 6 option :distributed_tracing, default: true
  169. 6 option :split_by_domain, default: false
  170. 6 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  171. 6 option :enabled do |o|
  172. 6 o.type :bool
  173. 6 o.env "DD_TRACE_HTTPX_ENABLED"
  174. 6 o.default true
  175. end
  176. 6 option :analytics_enabled do |o|
  177. 6 o.type :bool
  178. 6 o.env "DD_TRACE_HTTPX_ANALYTICS_ENABLED"
  179. 6 o.default false
  180. end
  181. 6 option :analytics_sample_rate do |o|
  182. 6 o.type :float
  183. 6 o.env "DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE"
  184. 6 o.default 1.0
  185. end
  186. else
  187. option :enabled do |o|
  188. o.default { env_to_bool("DD_TRACE_HTTPX_ENABLED", true) }
  189. o.lazy
  190. end
  191. option :analytics_enabled do |o|
  192. o.default { env_to_bool(%w[DD_TRACE_HTTPX_ANALYTICS_ENABLED DD_HTTPX_ANALYTICS_ENABLED], false) }
  193. o.lazy
  194. end
  195. option :analytics_sample_rate do |o|
  196. o.default { env_to_float(%w[DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE DD_HTTPX_ANALYTICS_SAMPLE_RATE], 1.0) }
  197. o.lazy
  198. end
  199. end
  200. 6 if defined?(Datadog::Tracing::Contrib::SpanAttributeSchema)
  201. 6 option :service_name do |o|
  202. 6 o.default do
  203. 66 Datadog::Tracing::Contrib::SpanAttributeSchema.fetch_service_name(
  204. "DD_TRACE_HTTPX_SERVICE_NAME",
  205. "httpx"
  206. )
  207. end
  208. 6 o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  209. end
  210. else
  211. option :service_name do |o|
  212. o.default do
  213. ENV.fetch("DD_TRACE_HTTPX_SERVICE_NAME", "httpx")
  214. end
  215. o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  216. end
  217. end
  218. 6 option :distributed_tracing, default: true
  219. 6 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.15.0")
  220. 6 option :error_handler do |o|
  221. 6 o.type :proc
  222. 6 o.default_proc(&DEFAULT_ERROR_HANDLER)
  223. end
  224. elsif Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  225. option :error_handler do |o|
  226. o.type :proc
  227. o.experimental_default_proc(&DEFAULT_ERROR_HANDLER)
  228. end
  229. else
  230. option :error_handler, default: DEFAULT_ERROR_HANDLER
  231. end
  232. end
  233. end
  234. # Patcher enables patching of 'httpx' with datadog components.
  235. #
  236. 6 module Patcher
  237. 6 include Datadog::Tracing::Contrib::Patcher
  238. 6 module_function
  239. 6 def target_version
  240. 12 Integration.version
  241. end
  242. # loads a session instannce with the datadog plugin, and replaces the
  243. # base HTTPX::Session with the patched session class.
  244. 6 def patch
  245. 6 datadog_session = ::HTTPX.plugin(Plugin)
  246. 6 ::HTTPX.send(:remove_const, :Session)
  247. 6 ::HTTPX.send(:const_set, :Session, datadog_session.class)
  248. end
  249. end
  250. # Datadog Integration for HTTPX.
  251. #
  252. 6 class Integration
  253. 6 include Contrib::Integration
  254. 6 MINIMUM_VERSION = Gem::Version.new("0.10.2")
  255. 6 register_as :httpx
  256. 6 def self.version
  257. 246 Gem.loaded_specs["httpx"] && Gem.loaded_specs["httpx"].version
  258. end
  259. 6 def self.loaded?
  260. 78 defined?(::HTTPX::Request)
  261. end
  262. 6 def self.compatible?
  263. 78 super && version >= MINIMUM_VERSION
  264. end
  265. 6 def new_configuration
  266. 156 Configuration::Settings.new
  267. end
  268. 6 def patcher
  269. 156 Patcher
  270. end
  271. end
  272. end
  273. end
  274. end

lib/httpx/adapters/faraday.rb

98.13% lines covered

160 relevant lines. 157 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 15 require "delegate"
  3. 15 require "httpx"
  4. 15 require "faraday"
  5. 15 module Faraday
  6. 15 class Adapter
  7. 15 class HTTPX < Faraday::Adapter
  8. 15 module RequestMixin
  9. 15 def build_connection(env)
  10. 287 return @connection if defined?(@connection)
  11. 287 @connection = ::HTTPX.plugin(:persistent).plugin(ReasonPlugin)
  12. 287 @connection = @connection.with(@connection_options) unless @connection_options.empty?
  13. 287 connection_opts = options_from_env(env)
  14. 287 if (bind = env.request.bind)
  15. 8 @bind = TCPSocket.new(bind[:host], bind[:port])
  16. 7 connection_opts[:io] = @bind
  17. end
  18. 287 @connection = @connection.with(connection_opts)
  19. 287 if (proxy = env.request.proxy)
  20. 8 proxy_options = { uri: proxy.uri }
  21. 8 proxy_options[:username] = proxy.user if proxy.user
  22. 8 proxy_options[:password] = proxy.password if proxy.password
  23. 8 @connection = @connection.plugin(:proxy).with(proxy: proxy_options)
  24. end
  25. 287 @connection = @connection.plugin(OnDataPlugin) if env.request.stream_response?
  26. 287 @connection = @config_block.call(@connection) || @connection if @config_block
  27. 287 @connection
  28. end
  29. 15 def close
  30. 294 @connection.close if @connection
  31. 294 @bind.close if @bind
  32. end
  33. 15 private
  34. 15 def connect(env, &blk)
  35. 287 connection(env, &blk)
  36. rescue ::HTTPX::TLSError => e
  37. 8 raise Faraday::SSLError, e
  38. rescue Errno::ECONNABORTED,
  39. Errno::ECONNREFUSED,
  40. Errno::ECONNRESET,
  41. Errno::EHOSTUNREACH,
  42. Errno::EINVAL,
  43. Errno::ENETUNREACH,
  44. Errno::EPIPE,
  45. ::HTTPX::ConnectionError => e
  46. 8 raise Faraday::ConnectionFailed, e
  47. end
  48. 15 def build_request(env)
  49. 295 meth = env[:method]
  50. 30 request_options = {
  51. 264 headers: env.request_headers,
  52. body: env.body,
  53. **options_from_env(env),
  54. }
  55. 295 [meth.to_s.upcase, env.url, request_options]
  56. end
  57. 15 def options_from_env(env)
  58. 582 timeout_options = {}
  59. 582 req_opts = env.request
  60. 582 if (sec = request_timeout(:read, req_opts))
  61. 28 timeout_options[:read_timeout] = sec
  62. end
  63. 582 if (sec = request_timeout(:write, req_opts))
  64. 14 timeout_options[:write_timeout] = sec
  65. end
  66. 582 if (sec = request_timeout(:open, req_opts))
  67. 14 timeout_options[:connect_timeout] = sec
  68. end
  69. 59 {
  70. 522 ssl: ssl_options_from_env(env),
  71. timeout: timeout_options,
  72. }
  73. end
  74. 15 if defined?(::OpenSSL)
  75. 15 def ssl_options_from_env(env)
  76. 582 ssl_options = {}
  77. 582 unless env.ssl.verify.nil?
  78. 28 ssl_options[:verify_mode] = env.ssl.verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
  79. end
  80. 582 ssl_options[:ca_file] = env.ssl.ca_file if env.ssl.ca_file
  81. 582 ssl_options[:ca_path] = env.ssl.ca_path if env.ssl.ca_path
  82. 582 ssl_options[:cert_store] = env.ssl.cert_store if env.ssl.cert_store
  83. 582 ssl_options[:cert] = env.ssl.client_cert if env.ssl.client_cert
  84. 582 ssl_options[:key] = env.ssl.client_key if env.ssl.client_key
  85. 582 ssl_options[:ssl_version] = env.ssl.version if env.ssl.version
  86. 582 ssl_options[:verify_depth] = env.ssl.verify_depth if env.ssl.verify_depth
  87. 582 ssl_options[:min_version] = env.ssl.min_version if env.ssl.min_version
  88. 582 ssl_options[:max_version] = env.ssl.max_version if env.ssl.max_version
  89. 582 ssl_options
  90. end
  91. else
  92. skipped # :nocov:
  93. skipped def ssl_options_from_env(*)
  94. skipped {}
  95. skipped end
  96. skipped # :nocov:
  97. end
  98. end
  99. 15 include RequestMixin
  100. 15 module OnDataPlugin
  101. 15 module RequestMethods
  102. 15 attr_writer :response_on_data
  103. 15 def response=(response)
  104. 16 super
  105. 16 return if response.is_a?(::HTTPX::ErrorResponse)
  106. 16 response.body.on_data = @response_on_data
  107. end
  108. end
  109. 15 module ResponseBodyMethods
  110. 15 attr_writer :on_data
  111. 15 def write(chunk)
  112. 42 return super unless @on_data
  113. 42 @on_data.call(chunk, chunk.bytesize)
  114. end
  115. end
  116. end
  117. 15 module ReasonPlugin
  118. 15 def self.load_dependencies(*)
  119. 287 require "net/http/status"
  120. end
  121. 15 module ResponseMethods
  122. 15 def reason
  123. 241 Net::HTTP::STATUS_CODES.fetch(@status, "Non-Standard status code")
  124. end
  125. end
  126. end
  127. 15 class ParallelManager
  128. 15 class ResponseHandler < SimpleDelegator
  129. 15 attr_reader :env
  130. 15 def initialize(env)
  131. 32 @env = env
  132. 32 super
  133. end
  134. 15 def on_response(&blk)
  135. 64 if blk
  136. 32 @on_response = ->(response) do
  137. 32 blk.call(response)
  138. end
  139. 32 self
  140. else
  141. 32 @on_response
  142. end
  143. end
  144. 15 def on_complete(&blk)
  145. 32 if blk
  146. @on_complete = blk
  147. self
  148. else
  149. 32 @on_complete
  150. end
  151. end
  152. end
  153. 15 include RequestMixin
  154. 15 def initialize(options)
  155. 32 @handlers = []
  156. 32 @connection_options = options
  157. end
  158. 15 def enqueue(request)
  159. 32 handler = ResponseHandler.new(request)
  160. 32 @handlers << handler
  161. 32 handler
  162. end
  163. 15 def run
  164. 32 return unless @handlers.last
  165. 24 env = @handlers.last.env
  166. 24 connect(env) do |session|
  167. 56 requests = @handlers.map { |handler| session.build_request(*build_request(handler.env)) }
  168. 24 if env.request.stream_response?
  169. 8 requests.each do |request|
  170. 8 request.response_on_data = env.request.on_data
  171. end
  172. end
  173. 24 responses = session.request(*requests)
  174. 24 Array(responses).each_with_index do |response, index|
  175. 32 handler = @handlers[index]
  176. 32 handler.on_response.call(response)
  177. 32 handler.on_complete.call(handler.env) if handler.on_complete
  178. end
  179. end
  180. rescue ::HTTPX::TimeoutError => e
  181. raise Faraday::TimeoutError, e
  182. end
  183. # from Faraday::Adapter#connection
  184. 15 def connection(env)
  185. 24 conn = build_connection(env)
  186. 24 return conn unless block_given?
  187. 24 yield conn
  188. end
  189. 15 private
  190. # from Faraday::Adapter#request_timeout
  191. 15 def request_timeout(type, options)
  192. 168 key = Faraday::Adapter::TIMEOUT_KEYS[type]
  193. 168 options[key] || options[:timeout]
  194. end
  195. end
  196. 15 self.supports_parallel = true
  197. 15 class << self
  198. 15 def setup_parallel_manager(options = {})
  199. 32 ParallelManager.new(options)
  200. end
  201. end
  202. 15 def call(env)
  203. 295 super
  204. 295 if parallel?(env)
  205. 32 handler = env[:parallel_manager].enqueue(env)
  206. 32 handler.on_response do |response|
  207. 32 if response.is_a?(::HTTPX::Response)
  208. 24 save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
  209. 24 response_headers.merge!(response.headers)
  210. end
  211. else
  212. 7 env[:error] = response.error
  213. 8 save_response(env, 0, "", {}, nil)
  214. end
  215. end
  216. 28 return handler
  217. end
  218. 263 response = connect_and_request(env)
  219. 217 save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
  220. 217 response_headers.merge!(response.headers)
  221. end
  222. 217 @app.call(env)
  223. end
  224. 15 private
  225. 15 def connect_and_request(env)
  226. 263 connect(env) do |session|
  227. 263 request = session.build_request(*build_request(env))
  228. 263 request.response_on_data = env.request.on_data if env.request.stream_response?
  229. 263 response = session.request(request)
  230. # do not call #raise_for_status for HTTP 4xx or 5xx, as faraday has a middleware for that.
  231. 263 response.raise_for_status unless response.is_a?(::HTTPX::Response)
  232. 217 response
  233. end
  234. rescue ::HTTPX::TimeoutError => e
  235. 24 raise Faraday::TimeoutError, e
  236. end
  237. 15 def parallel?(env)
  238. 295 env[:parallel_manager]
  239. end
  240. end
  241. 15 register_middleware httpx: HTTPX
  242. end
  243. end

lib/httpx/adapters/sentry.rb

100.0% lines covered

62 relevant lines. 62 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "sentry-ruby"
  3. 6 module HTTPX::Plugins
  4. 6 module Sentry
  5. 6 module Tracer
  6. 6 module_function
  7. 6 def call(request)
  8. 148 sentry_span = start_sentry_span
  9. 148 return unless sentry_span
  10. 148 set_sentry_trace_header(request, sentry_span)
  11. 148 request.on(:response, &method(:finish_sentry_span).curry(3)[sentry_span, request])
  12. end
  13. 6 def start_sentry_span
  14. 148 return unless ::Sentry.initialized? && (span = ::Sentry.get_current_scope.get_span)
  15. 148 return if span.sampled == false
  16. 148 span.start_child(op: "httpx.client", start_timestamp: ::Sentry.utc_now.to_f)
  17. end
  18. 6 def set_sentry_trace_header(request, sentry_span)
  19. 148 return unless sentry_span
  20. 148 config = ::Sentry.configuration
  21. 148 url = request.uri.to_s
  22. 296 return unless config.propagate_traces && config.trace_propagation_targets.any? { |target| url.match?(target) }
  23. 148 trace = ::Sentry.get_current_client.generate_sentry_trace(sentry_span)
  24. 148 request.headers[::Sentry::SENTRY_TRACE_HEADER_NAME] = trace if trace
  25. end
  26. 6 def finish_sentry_span(span, request, response)
  27. 154 return unless ::Sentry.initialized?
  28. 154 record_sentry_breadcrumb(request, response)
  29. 154 record_sentry_span(request, response, span)
  30. end
  31. 6 def record_sentry_breadcrumb(req, res)
  32. 154 return unless ::Sentry.configuration.breadcrumbs_logger.include?(:http_logger)
  33. 154 request_info = extract_request_info(req)
  34. 154 data = if res.is_a?(HTTPX::ErrorResponse)
  35. 14 { error: res.error.message, **request_info }
  36. else
  37. 140 { status: res.status, **request_info }
  38. end
  39. 154 crumb = ::Sentry::Breadcrumb.new(
  40. level: :info,
  41. category: "httpx",
  42. type: :info,
  43. data: data
  44. )
  45. 154 ::Sentry.add_breadcrumb(crumb)
  46. end
  47. 6 def record_sentry_span(req, res, sentry_span)
  48. 154 return unless sentry_span
  49. 154 request_info = extract_request_info(req)
  50. 154 sentry_span.set_description("#{request_info[:method]} #{request_info[:url]}")
  51. 154 if res.is_a?(HTTPX::ErrorResponse)
  52. 14 sentry_span.set_data(:error, res.error.message)
  53. else
  54. 140 sentry_span.set_data(:status, res.status)
  55. end
  56. 154 sentry_span.set_timestamp(::Sentry.utc_now.to_f)
  57. end
  58. 6 def extract_request_info(req)
  59. 308 uri = req.uri
  60. result = {
  61. 308 method: req.verb,
  62. }
  63. 308 if ::Sentry.configuration.send_default_pii
  64. 24 uri += "?#{req.query}" unless req.query.empty?
  65. 24 result[:body] = req.body.to_s unless req.body.empty? || req.body.unbounded_body?
  66. end
  67. 308 result[:url] = uri.to_s
  68. 308 result
  69. end
  70. end
  71. 6 module RequestMethods
  72. 6 def __sentry_enable_trace!
  73. 154 return if @__sentry_enable_trace
  74. 148 Tracer.call(self)
  75. 148 @__sentry_enable_trace = true
  76. end
  77. end
  78. 6 module ConnectionMethods
  79. 6 def send(request)
  80. 154 request.__sentry_enable_trace!
  81. 154 super
  82. end
  83. end
  84. end
  85. end
  86. 6 Sentry.register_patch(:httpx) do
  87. 30 sentry_session = HTTPX.plugin(HTTPX::Plugins::Sentry)
  88. 30 HTTPX.send(:remove_const, :Session)
  89. 30 HTTPX.send(:const_set, :Session, sentry_session.class)
  90. end

lib/httpx/adapters/webmock.rb

100.0% lines covered

93 relevant lines. 93 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module WebMock
  3. 8 module HttpLibAdapters
  4. 8 require "net/http/status"
  5. 8 HTTP_REASONS = Net::HTTP::STATUS_CODES
  6. #
  7. # HTTPX plugin for webmock.
  8. #
  9. # Requests are "hijacked" at the session, before they're distributed to a connection.
  10. #
  11. 8 module Plugin
  12. 8 class << self
  13. 8 def build_webmock_request_signature(request)
  14. 188 uri = WebMock::Util::URI.heuristic_parse(request.uri)
  15. 188 uri.query = request.query
  16. 188 uri.path = uri.normalized_path.gsub("[^:]//", "/")
  17. 188 WebMock::RequestSignature.new(
  18. request.verb.downcase.to_sym,
  19. uri.to_s,
  20. body: request.body.to_s,
  21. headers: request.headers.to_h
  22. )
  23. end
  24. 8 def build_webmock_response(_request, response)
  25. 6 webmock_response = WebMock::Response.new
  26. 6 webmock_response.status = [response.status, HTTP_REASONS[response.status]]
  27. 6 webmock_response.body = response.body.to_s
  28. 6 webmock_response.headers = response.headers.to_h
  29. 6 webmock_response
  30. end
  31. 8 def build_from_webmock_response(request, webmock_response)
  32. 158 return build_error_response(request, HTTPX::TimeoutError.new(1, "Timed out")) if webmock_response.should_timeout
  33. 140 return build_error_response(request, webmock_response.exception) if webmock_response.exception
  34. 133 request.options.response_class.new(request,
  35. webmock_response.status[0],
  36. "2.0",
  37. webmock_response.headers).tap do |res|
  38. 133 res.mocked = true
  39. end
  40. end
  41. 8 def build_error_response(request, exception)
  42. 25 HTTPX::ErrorResponse.new(request, exception)
  43. end
  44. end
  45. 8 module InstanceMethods
  46. 8 private
  47. 8 def do_init_connection(connection, selector)
  48. 170 super
  49. 170 connection.once(:unmock_connection) do
  50. 24 next unless connection.current_session == self
  51. 24 unless connection.addresses
  52. # reset Happy Eyeballs, fail early
  53. 24 connection.sibling = nil
  54. 24 deselect_connection(connection, selector)
  55. end
  56. 24 resolve_connection(connection, selector)
  57. end
  58. end
  59. end
  60. 8 module ResponseMethods
  61. 8 attr_accessor :mocked
  62. 8 def initialize(*)
  63. 157 super
  64. 157 @mocked = false
  65. end
  66. end
  67. 8 module ResponseBodyMethods
  68. 8 def decode_chunk(chunk)
  69. 96 return chunk if @response.mocked
  70. 42 super
  71. end
  72. end
  73. 8 module ConnectionMethods
  74. 8 def initialize(*)
  75. 170 super
  76. 170 @mocked = true
  77. end
  78. 8 def open?
  79. 194 return true if @mocked
  80. 24 super
  81. end
  82. 8 def interests
  83. 276 return if @mocked
  84. 246 super
  85. end
  86. 8 def terminate
  87. 145 force_reset
  88. end
  89. 8 def send(request)
  90. 188 request_signature = Plugin.build_webmock_request_signature(request)
  91. 188 WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)
  92. 188 if (mock_response = WebMock::StubRegistry.instance.response_for_request(request_signature))
  93. 158 response = Plugin.build_from_webmock_response(request, mock_response)
  94. 158 WebMock::CallbackRegistry.invoke_callbacks({ lib: :httpx }, request_signature, mock_response)
  95. 158 log { "mocking #{request.uri} with #{mock_response.inspect}" }
  96. 158 request.transition(:headers)
  97. 158 request.transition(:body)
  98. 158 request.transition(:trailers)
  99. 158 request.transition(:done)
  100. 158 response.finish!
  101. 158 request.response = response
  102. 158 request.emit(:response, response)
  103. 158 request_signature.headers = request.headers.to_h
  104. 158 response << mock_response.body.dup unless response.is_a?(HTTPX::ErrorResponse)
  105. 30 elsif WebMock.net_connect_allowed?(request_signature.uri)
  106. 24 if WebMock::CallbackRegistry.any_callbacks?
  107. 6 request.on(:response) do |resp|
  108. 6 unless resp.is_a?(HTTPX::ErrorResponse)
  109. 6 webmock_response = Plugin.build_webmock_response(request, resp)
  110. 6 WebMock::CallbackRegistry.invoke_callbacks(
  111. { lib: :httpx, real_request: true }, request_signature,
  112. webmock_response
  113. )
  114. end
  115. end
  116. end
  117. 24 @mocked = false
  118. 24 emit(:unmock_connection, self)
  119. 24 super
  120. else
  121. 6 raise WebMock::NetConnectNotAllowedError, request_signature
  122. end
  123. end
  124. end
  125. end
  126. 8 class HttpxAdapter < HttpLibAdapter
  127. 8 adapter_for :httpx
  128. 8 class << self
  129. 8 def enable!
  130. 370 @original_session ||= HTTPX::Session
  131. 370 webmock_session = HTTPX.plugin(Plugin)
  132. 370 HTTPX.send(:remove_const, :Session)
  133. 370 HTTPX.send(:const_set, :Session, webmock_session.class)
  134. end
  135. 8 def disable!
  136. 370 return unless @original_session
  137. 362 HTTPX.send(:remove_const, :Session)
  138. 362 HTTPX.send(:const_set, :Session, @original_session)
  139. end
  140. end
  141. end
  142. end
  143. end

lib/httpx/altsvc.rb

96.39% lines covered

83 relevant lines. 80 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "strscan"
  3. 27 module HTTPX
  4. 27 module AltSvc
  5. # makes connections able to accept requests destined to primary service.
  6. 27 module ConnectionMixin
  7. 27 using URIExtensions
  8. 27 def send(request)
  9. 8 request.headers["alt-used"] = @origin.authority if @parser && !@write_buffer.full? && match_altsvcs?(request.uri)
  10. 8 super
  11. end
  12. 27 def match?(uri, options)
  13. 8 return false if !used? && (@state == :closing || @state == :closed)
  14. 8 match_altsvcs?(uri) && match_altsvc_options?(uri, options)
  15. end
  16. 27 private
  17. # checks if this is connection is an alternative service of
  18. # +uri+
  19. 27 def match_altsvcs?(uri)
  20. 24 @origins.any? { |origin| altsvc_match?(uri, origin) } ||
  21. AltSvc.cached_altsvc(@origin).any? do |altsvc|
  22. origin = altsvc["origin"]
  23. altsvc_match?(origin, uri.origin)
  24. end
  25. end
  26. 27 def match_altsvc_options?(uri, options)
  27. 8 return @options == options unless @options.ssl.all? do |k, v|
  28. 8 v == (k == :hostname ? uri.host : options.ssl[k])
  29. end
  30. 8 @options.options_equals?(options, Options::REQUEST_BODY_IVARS + %i[@ssl])
  31. end
  32. 27 def altsvc_match?(uri, other_uri)
  33. 16 other_uri = URI(other_uri)
  34. 16 uri.origin == other_uri.origin || begin
  35. 7 case uri.scheme
  36. when "h2"
  37. (other_uri.scheme == "https" || other_uri.scheme == "h2") &&
  38. uri.host == other_uri.host &&
  39. uri.port == other_uri.port
  40. else
  41. 8 false
  42. end
  43. end
  44. end
  45. end
  46. 27 @altsvc_mutex = Thread::Mutex.new
  47. 48 @altsvcs = Hash.new { |h, k| h[k] = [] }
  48. 27 module_function
  49. 27 def cached_altsvc(origin)
  50. 40 now = Utils.now
  51. 40 @altsvc_mutex.synchronize do
  52. 40 lookup(origin, now)
  53. end
  54. end
  55. 27 def cached_altsvc_set(origin, entry)
  56. 24 now = Utils.now
  57. 24 @altsvc_mutex.synchronize do
  58. 24 return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
  59. 24 entry["TTL"] = Integer(entry["ma"]) + now if entry.key?("ma")
  60. 24 @altsvcs[origin] << entry
  61. 24 entry
  62. end
  63. end
  64. 27 def lookup(origin, ttl)
  65. 40 return [] unless @altsvcs.key?(origin)
  66. 28 @altsvcs[origin] = @altsvcs[origin].select do |entry|
  67. 24 !entry.key?("TTL") || entry["TTL"] > ttl
  68. end
  69. 48 @altsvcs[origin].reject { |entry| entry["noop"] }
  70. end
  71. 27 def emit(request, response)
  72. 8011 return unless response.respond_to?(:headers)
  73. # Alt-Svc
  74. 7973 return unless response.headers.key?("alt-svc")
  75. 79 origin = request.origin
  76. 79 host = request.uri.host
  77. 79 altsvc = response.headers["alt-svc"]
  78. # https://datatracker.ietf.org/doc/html/rfc7838#section-3
  79. # A field value containing the special value "clear" indicates that the
  80. # origin requests all alternatives for that origin to be invalidated
  81. # (including those specified in the same response, in case of an
  82. # invalid reply containing both "clear" and alternative services).
  83. 79 if altsvc == "clear"
  84. 8 @altsvc_mutex.synchronize do
  85. 8 @altsvcs[origin].clear
  86. end
  87. 7 return
  88. end
  89. 71 parse(altsvc) do |alt_origin, alt_params|
  90. 8 alt_origin.host ||= host
  91. 8 yield(alt_origin, origin, alt_params)
  92. end
  93. end
  94. 27 def parse(altsvc)
  95. 183 return enum_for(__method__, altsvc) unless block_given?
  96. 127 scanner = StringScanner.new(altsvc)
  97. 133 until scanner.eos?
  98. 127 alt_service = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
  99. 127 alt_params = []
  100. 127 loop do
  101. 151 alt_param = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
  102. 151 alt_params << alt_param.strip if alt_param
  103. 151 scanner.skip(/;/)
  104. 151 break if scanner.eos? || scanner.scan(/ *, */)
  105. end
  106. 254 alt_params = Hash[alt_params.map { |field| field.split("=", 2) }]
  107. 127 alt_proto, alt_authority = alt_service.split("=", 2)
  108. 127 alt_origin = parse_altsvc_origin(alt_proto, alt_authority)
  109. 127 return unless alt_origin
  110. 48 yield(alt_origin, alt_params.merge("proto" => alt_proto))
  111. end
  112. end
  113. 27 def parse_altsvc_scheme(alt_proto)
  114. 134 case alt_proto
  115. when "h2c"
  116. 8 "http"
  117. when "h2"
  118. 56 "https"
  119. end
  120. end
  121. 27 def parse_altsvc_origin(alt_proto, alt_origin)
  122. 127 alt_scheme = parse_altsvc_scheme(alt_proto)
  123. 127 return unless alt_scheme
  124. 48 alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
  125. 48 URI.parse("#{alt_scheme}://#{alt_origin}")
  126. end
  127. end
  128. end

lib/httpx/buffer.rb

100.0% lines covered

27 relevant lines. 27 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 module HTTPX
  4. # Internal class to abstract a string buffer, by wrapping a string and providing the
  5. # minimum possible API and functionality required.
  6. #
  7. # buffer = Buffer.new(640)
  8. # buffer.full? #=> false
  9. # buffer << "aa"
  10. # buffer.capacity #=> 638
  11. #
  12. 27 class Buffer
  13. 27 extend Forwardable
  14. 27 def_delegator :@buffer, :to_s
  15. 27 def_delegator :@buffer, :to_str
  16. 27 def_delegator :@buffer, :empty?
  17. 27 def_delegator :@buffer, :bytesize
  18. 27 def_delegator :@buffer, :clear
  19. 27 def_delegator :@buffer, :replace
  20. 27 attr_reader :limit
  21. 27 if RUBY_VERSION >= "3.4.0"
  22. 15 def initialize(limit)
  23. 2914 @buffer = String.new("", encoding: Encoding::BINARY, capacity: limit)
  24. 2914 @limit = limit
  25. end
  26. 15 def <<(chunk)
  27. 12218 @buffer.append_as_bytes(chunk)
  28. end
  29. else
  30. 12 def initialize(limit)
  31. 18993 @buffer = "".b
  32. 18993 @limit = limit
  33. end
  34. 12 def_delegator :@buffer, :<<
  35. end
  36. 27 def full?
  37. 10816567 @buffer.bytesize >= @limit
  38. end
  39. 27 def capacity
  40. 12 @limit - @buffer.bytesize
  41. end
  42. 27 def shift!(fin)
  43. 22105 @buffer = @buffer.byteslice(fin..-1) || "".b
  44. end
  45. end
  46. end

lib/httpx/callbacks.rb

100.0% lines covered

19 relevant lines. 19 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Callbacks
  4. 27 def on(type, &action)
  5. 354408 callbacks(type) << action
  6. 354408 action
  7. end
  8. 27 def once(type, &block)
  9. 192506 on(type) do |*args, &callback|
  10. 99119 block.call(*args, &callback)
  11. 99055 :delete
  12. end
  13. end
  14. 27 def emit(type, *args)
  15. 129548 log { "emit #{type.inspect} callbacks" } if respond_to?(:log)
  16. 271388 callbacks(type).delete_if { |pr| :delete == pr.call(*args) } # rubocop:disable Style/YodaCondition
  17. end
  18. 27 def callbacks_for?(type)
  19. 3234 @callbacks.key?(type) && @callbacks[type].any?
  20. end
  21. 27 protected
  22. 27 def callbacks(type = nil)
  23. 487301 return @callbacks unless type
  24. 702671 @callbacks ||= Hash.new { |h, k| h[k] = [] }
  25. 487214 @callbacks[type]
  26. end
  27. end
  28. end

lib/httpx/chainable.rb

95.65% lines covered

46 relevant lines. 44 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # Session mixin, implements most of the APIs that the users call.
  4. # delegates to a default session when extended.
  5. 27 module Chainable
  6. 27 %w[head get post put delete trace options connect patch].each do |meth|
  7. 234 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  8. 9 def #{meth}(*uri, **options) # def get(*uri, **options)
  9. 18 request("#{meth.upcase}", uri, **options) # request("GET", uri, **options)
  10. end # end
  11. MOD
  12. end
  13. # delegates to the default session (see HTTPX::Session#request).
  14. 27 def request(*args, **options)
  15. 2670 branch(default_options).request(*args, **options)
  16. end
  17. 27 def accept(type)
  18. 16 with(headers: { "accept" => String(type) })
  19. end
  20. # delegates to the default session (see HTTPX::Session#wrap).
  21. 27 def wrap(&blk)
  22. 89 branch(default_options).wrap(&blk)
  23. end
  24. # returns a new instance loaded with the +pl+ plugin and +options+.
  25. 27 def plugin(pl, options = nil, &blk)
  26. 5271 klass = is_a?(S) ? self.class : Session
  27. 5271 klass = Class.new(klass)
  28. 5271 klass.instance_variable_set(:@default_options, klass.default_options.merge(default_options))
  29. 5271 klass.plugin(pl, options, &blk).new
  30. end
  31. # returns a new instance loaded with +options+.
  32. 27 def with(options, &blk)
  33. 2761 branch(default_options.merge(options), &blk)
  34. end
  35. 27 private
  36. # returns default instance of HTTPX::Options.
  37. 27 def default_options
  38. 10847 @options || Session.default_options
  39. end
  40. # returns a default instance of HTTPX::Session.
  41. 27 def branch(options, &blk)
  42. 5504 return self.class.new(options, &blk) if is_a?(S)
  43. 3211 Session.new(options, &blk)
  44. end
  45. 27 def method_missing(meth, *args, **options, &blk)
  46. 652 case meth
  47. when /\Awith_(.+)/
  48. 729 option = Regexp.last_match(1)
  49. 729 return super unless option
  50. 729 with(option.to_sym => args.first || options)
  51. when /\Aon_(.+)/
  52. 9 callback = Regexp.last_match(1)
  53. 6 return super unless %w[
  54. connection_opened connection_closed
  55. request_error
  56. request_started request_body_chunk request_completed
  57. response_started response_body_chunk response_completed
  58. 2 ].include?(callback)
  59. 9 warn "DEPRECATION WARNING: calling `.#{meth}` on plain HTTPX sessions is deprecated. " \
  60. 1 "Use `HTTPX.plugin(:callbacks).#{meth}` instead."
  61. 9 plugin(:callbacks).__send__(meth, *args, **options, &blk)
  62. else
  63. super
  64. end
  65. end
  66. 27 def respond_to_missing?(meth, *)
  67. 49 case meth
  68. when /\Awith_(.+)/
  69. 40 option = Regexp.last_match(1)
  70. 40 default_options.respond_to?(option) || super
  71. when /\Aon_(.+)/
  72. 16 callback = Regexp.last_match(1)
  73. 12 %w[
  74. connection_opened connection_closed
  75. request_error
  76. request_started request_body_chunk request_completed
  77. response_started response_body_chunk response_completed
  78. 3 ].include?(callback) || super
  79. else
  80. super
  81. end
  82. end
  83. end
  84. 27 extend Chainable
  85. end

lib/httpx/connection.rb

92.16% lines covered

485 relevant lines. 447 lines covered and 38 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "forwardable"
  4. 27 require "httpx/io"
  5. 27 require "httpx/buffer"
  6. 27 module HTTPX
  7. # The Connection can be watched for IO events.
  8. #
  9. # It contains the +io+ object to read/write from, and knows what to do when it can.
  10. #
  11. # It defers connecting until absolutely necessary. Connection should be triggered from
  12. # the IO selector (until then, any request will be queued).
  13. #
  14. # A connection boots up its parser after connection is established. All pending requests
  15. # will be redirected there after connection.
  16. #
  17. # A connection can be prevented from closing by the parser, that is, if there are pending
  18. # requests. This will signal that the connection was prematurely closed, due to a possible
  19. # number of conditions:
  20. #
  21. # * Remote peer closed the connection ("Connection: close");
  22. # * Remote peer doesn't support pipelining;
  23. #
  24. # A connection may also route requests for a different host for which the +io+ was connected
  25. # to, provided that the IP is the same and the port and scheme as well. This will allow to
  26. # share the same socket to send HTTP/2 requests to different hosts.
  27. #
  28. 27 class Connection
  29. 27 extend Forwardable
  30. 27 include Loggable
  31. 27 include Callbacks
  32. 27 using URIExtensions
  33. 27 require "httpx/connection/http2"
  34. 27 require "httpx/connection/http1"
  35. 27 def_delegator :@io, :closed?
  36. 27 def_delegator :@write_buffer, :empty?
  37. 27 attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling
  38. 27 attr_writer :current_selector
  39. 27 attr_accessor :current_session, :family
  40. 27 protected :sibling
  41. 27 def initialize(uri, options)
  42. 7424 @current_session = @current_selector = @sibling = @coalesced_connection = nil
  43. 7424 @exhausted = @cloned = @main_sibling = false
  44. 7424 @options = Options.new(options)
  45. 7424 @type = initialize_type(uri, @options)
  46. 7424 @origins = [uri.origin]
  47. 7424 @origin = Utils.to_uri(uri.origin)
  48. 7424 @window_size = @options.window_size
  49. 7424 @read_buffer = Buffer.new(@options.buffer_size)
  50. 7424 @write_buffer = Buffer.new(@options.buffer_size)
  51. 7424 @pending = []
  52. 7424 on(:error, &method(:on_error))
  53. 7424 if @options.io
  54. # if there's an already open IO, get its
  55. # peer address, and force-initiate the parser
  56. 66 transition(:already_open)
  57. 66 @io = build_socket
  58. 66 parser
  59. else
  60. 7358 transition(:idle)
  61. end
  62. 7424 on(:close) do
  63. 7581 next if @exhausted # it'll reset
  64. # may be called after ":close" above, so after the connection has been checked back in.
  65. # next unless @current_session
  66. 7573 next unless @current_session
  67. 7573 @current_session.deselect_connection(self, @current_selector, @cloned)
  68. end
  69. 7424 on(:terminate) do
  70. 2816 next if @exhausted # it'll reset
  71. 2808 current_session = @current_session
  72. 2808 current_selector = @current_selector
  73. # may be called after ":close" above, so after the connection has been checked back in.
  74. 2808 next unless current_session && current_selector
  75. 16 current_session.deselect_connection(self, current_selector)
  76. end
  77. 7424 on(:altsvc) do |alt_origin, origin, alt_params|
  78. 8 build_altsvc_connection(alt_origin, origin, alt_params)
  79. end
  80. 7424 @inflight = 0
  81. 7424 @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
  82. 7424 self.addresses = @options.addresses if @options.addresses
  83. end
  84. 27 def peer
  85. 15086 @origin
  86. end
  87. # this is a semi-private method, to be used by the resolver
  88. # to initiate the io object.
  89. 27 def addresses=(addrs)
  90. 7202 if @io
  91. 249 @io.add_addresses(addrs)
  92. else
  93. 6953 @io = build_socket(addrs)
  94. end
  95. end
  96. 27 def addresses
  97. 14829 @io && @io.addresses
  98. end
  99. 27 def match?(uri, options)
  100. 2096 return false if !used? && (@state == :closing || @state == :closed)
  101. 312 (
  102. 1822 @origins.include?(uri.origin) &&
  103. # if there is more than one origin to match, it means that this connection
  104. # was the result of coalescing. To prevent blind trust in the case where the
  105. # origin came from an ORIGIN frame, we're going to verify the hostname with the
  106. # SSL certificate
  107. 1889 (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
  108. ) && @options == options
  109. end
  110. 27 def expired?
  111. return false unless @io
  112. @io.expired?
  113. end
  114. 27 def mergeable?(connection)
  115. 318 return false if @state == :closing || @state == :closed || !@io
  116. 64 return false unless connection.addresses
  117. 2 (
  118. 64 (open? && @origin == connection.origin) ||
  119. 64 !(@io.addresses & (connection.addresses || [])).empty?
  120. ) && @options == connection.options
  121. end
  122. # coalesces +self+ into +connection+.
  123. 27 def coalesce!(connection)
  124. 13 @coalesced_connection = connection
  125. 13 close_sibling
  126. 13 connection.merge(self)
  127. end
  128. # coalescable connections need to be mergeable!
  129. # but internally, #mergeable? is called before #coalescable?
  130. 27 def coalescable?(connection)
  131. 27 if @io.protocol == "h2" &&
  132. @origin.scheme == "https" &&
  133. connection.origin.scheme == "https" &&
  134. @io.can_verify_peer?
  135. 13 @io.verify_hostname(connection.origin.host)
  136. else
  137. 14 @origin == connection.origin
  138. end
  139. end
  140. 27 def create_idle(options = {})
  141. self.class.new(@origin, @options.merge(options))
  142. end
  143. 27 def merge(connection)
  144. 34 @origins |= connection.instance_variable_get(:@origins)
  145. 37 if connection.ssl_session
  146. 14 @ssl_session = connection.ssl_session
  147. 2 @io.session_new_cb do |sess|
  148. 21 @ssl_session = sess
  149. 14 end if @io
  150. end
  151. 37 connection.purge_pending do |req|
  152. 7 send(req)
  153. end
  154. end
  155. 27 def purge_pending(&block)
  156. 37 pendings = []
  157. 37 if @parser
  158. 21 @inflight -= @parser.pending.size
  159. 24 pendings << @parser.pending
  160. end
  161. 37 pendings << @pending
  162. 37 pendings.each do |pending|
  163. 61 pending.reject!(&block)
  164. end
  165. end
  166. 27 def io_connected?
  167. return @coalesced_connection.io_connected? if @coalesced_connection
  168. @io && @io.state == :connected
  169. end
  170. 27 def connecting?
  171. 10851930 @state == :idle
  172. end
  173. 27 def inflight?
  174. 2931 @parser && (
  175. # parser may be dealing with other requests (possibly started from a different fiber)
  176. 2460 !@parser.empty? ||
  177. # connection may be doing connection termination handshake
  178. !@write_buffer.empty?
  179. )
  180. end
  181. 27 def interests
  182. # connecting
  183. 10840225 if connecting?
  184. 11390 connect
  185. 11389 return @io.interests if connecting?
  186. end
  187. # if the write buffer is full, we drain it
  188. 10829400 return :w unless @write_buffer.empty?
  189. 10787567 return @parser.interests if @parser
  190. 21 nil
  191. rescue StandardError => e
  192. emit(:error, e)
  193. nil
  194. end
  195. 27 def to_io
  196. 23100 @io.to_io
  197. end
  198. 27 def call
  199. 20272 case @state
  200. when :idle
  201. 10678 connect
  202. 10663 consume
  203. when :closed
  204. return
  205. when :closing
  206. consume
  207. transition(:closed)
  208. when :open
  209. 11245 consume
  210. end
  211. 8543 nil
  212. rescue StandardError => e
  213. 23 @write_buffer.clear
  214. 23 emit(:error, e)
  215. 23 raise e
  216. end
  217. 27 def close
  218. 2768 transition(:active) if @state == :inactive
  219. 2768 @parser.close if @parser
  220. end
  221. 27 def terminate
  222. 2444 case @state
  223. when :idle
  224. purge_after_closed
  225. emit(:terminate)
  226. when :closed
  227. 8 @connected_at = nil
  228. end
  229. 2768 close
  230. end
  231. # bypasses the state machine to force closing of connections still connecting.
  232. # **only** used for Happy Eyeballs v2.
  233. 27 def force_reset(cloned = false)
  234. 292 @state = :closing
  235. 292 @cloned = cloned
  236. 292 transition(:closed)
  237. end
  238. 27 def reset
  239. 7499 return if @state == :closing || @state == :closed
  240. 7438 transition(:closing)
  241. 7438 transition(:closed)
  242. end
  243. 27 def send(request)
  244. 9014 return @coalesced_connection.send(request) if @coalesced_connection
  245. 9008 if @parser && !@write_buffer.full?
  246. 441 if @response_received_at && @keep_alive_timeout &&
  247. Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
  248. # when pushing a request into an existing connection, we have to check whether there
  249. # is the possibility that the connection might have extended the keep alive timeout.
  250. # for such cases, we want to ping for availability before deciding to shovel requests.
  251. 20 log(level: 3) { "keep alive timeout expired, pinging connection..." }
  252. 20 @pending << request
  253. 20 transition(:active) if @state == :inactive
  254. 20 parser.ping
  255. 20 request.ping!
  256. 18 return
  257. end
  258. 421 send_request_to_parser(request)
  259. else
  260. 8567 @pending << request
  261. end
  262. end
  263. 27 def timeout
  264. 10714324 return if @state == :closed || @state == :inactive
  265. 10714324 return @timeout if @timeout
  266. 10701457 return @options.timeout[:connect_timeout] if @state == :idle
  267. 10701457 @options.timeout[:operation_timeout]
  268. end
  269. 27 def idling
  270. 809 purge_after_closed
  271. 809 @write_buffer.clear
  272. 809 transition(:idle)
  273. 809 @parser = nil if @parser
  274. end
  275. 27 def used?
  276. 2315 @connected_at
  277. end
  278. 27 def deactivate
  279. 366 transition(:inactive)
  280. end
  281. 27 def open?
  282. 7240 @state == :open || @state == :inactive
  283. end
  284. 27 def handle_socket_timeout(interval)
  285. 32 error = OperationTimeoutError.new(interval, "timed out while waiting on select")
  286. 32 error.set_backtrace(caller)
  287. 32 on_error(error)
  288. end
  289. 27 def sibling=(connection)
  290. 24 @sibling = connection
  291. 24 return unless connection
  292. @main_sibling = connection.sibling.nil?
  293. return unless @main_sibling
  294. connection.sibling = self
  295. end
  296. 27 def handle_connect_error(error)
  297. 284 @connect_error = error
  298. 284 return handle_error(error) unless @sibling && @sibling.connecting?
  299. @sibling.merge(self)
  300. force_reset(true)
  301. end
  302. 27 def disconnect
  303. 7715 return unless @current_session && @current_selector
  304. 7581 emit(:close)
  305. 7565 @current_session = nil
  306. 7565 @current_selector = nil
  307. end
  308. skipped # :nocov:
  309. skipped def inspect
  310. skipped "#<#{self.class}:#{object_id} " \
  311. skipped "@origin=#{@origin} " \
  312. skipped "@state=#{@state} " \
  313. skipped "@pending=#{@pending.size} " \
  314. skipped "@io=#{@io}>"
  315. skipped end
  316. skipped # :nocov:
  317. 27 private
  318. 27 def connect
  319. 21060 transition(:open)
  320. end
  321. 27 def consume
  322. 25135 return unless @io
  323. 25135 catch(:called) do
  324. 25135 epiped = false
  325. 25135 loop do
  326. # connection may have
  327. 45587 return if @state == :idle
  328. 42285 parser.consume
  329. # we exit if there's no more requests to process
  330. #
  331. # this condition takes into account:
  332. #
  333. # * the number of inflight requests
  334. # * the number of pending requests
  335. # * whether the write buffer has bytes (i.e. for close handshake)
  336. 42269 if @pending.empty? && @inflight.zero? && @write_buffer.empty?
  337. 2892 log(level: 3) { "NO MORE REQUESTS..." }
  338. 2876 return
  339. end
  340. 39393 @timeout = @current_timeout
  341. 39393 read_drained = false
  342. 39393 write_drained = nil
  343. #
  344. # tight read loop.
  345. #
  346. # read as much of the socket as possible.
  347. #
  348. # this tight loop reads all the data it can from the socket and pipes it to
  349. # its parser.
  350. #
  351. 7534 loop do
  352. 51089 siz = @io.read(@window_size, @read_buffer)
  353. 51216 log(level: 3, color: :cyan) { "IO READ: #{siz} bytes... (wsize: #{@window_size}, rbuffer: #{@read_buffer.bytesize})" }
  354. 51089 unless siz
  355. 24 @write_buffer.clear
  356. 24 ex = EOFError.new("descriptor closed")
  357. 24 ex.set_backtrace(caller)
  358. 24 on_error(ex)
  359. 24 return
  360. end
  361. # socket has been drained. mark and exit the read loop.
  362. 51065 if siz.zero?
  363. 10699 read_drained = @read_buffer.empty?
  364. 10699 epiped = false
  365. 10699 break
  366. end
  367. 40366 parser << @read_buffer.to_s
  368. # continue reading if possible.
  369. 35939 break if interests == :w && !epiped
  370. # exit the read loop if connection is preparing to be closed
  371. 28444 break if @state == :closing || @state == :closed
  372. # exit #consume altogether if all outstanding requests have been dealt with
  373. 28429 return if @pending.empty? && @inflight.zero?
  374. 39393 end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
  375. #
  376. # tight write loop.
  377. #
  378. # flush as many bytes as the sockets allow.
  379. #
  380. 6160 loop do
  381. # buffer has been drainned, mark and exit the write loop.
  382. 24185 if @write_buffer.empty?
  383. # we only mark as drained on the first loop
  384. 2834 write_drained = write_drained.nil? && @inflight.positive?
  385. 2834 break
  386. end
  387. 2105 begin
  388. 21351 siz = @io.write(@write_buffer)
  389. rescue Errno::EPIPE
  390. # this can happen if we still have bytes in the buffer to send to the server, but
  391. # the server wants to respond immediately with some message, or an error. An example is
  392. # when one's uploading a big file to an unintended endpoint, and the server stops the
  393. # consumption, and responds immediately with an authorization of even method not allowed error.
  394. # at this point, we have to let the connection switch to read-mode.
  395. 21 log(level: 2) { "pipe broken, could not flush buffer..." }
  396. 21 epiped = true
  397. 21 read_drained = false
  398. 21 break
  399. end
  400. 21416 log(level: 3, color: :cyan) { "IO WRITE: #{siz} bytes..." }
  401. 21321 unless siz
  402. @write_buffer.clear
  403. ex = EOFError.new("descriptor closed")
  404. ex.set_backtrace(caller)
  405. on_error(ex)
  406. return
  407. end
  408. # socket closed for writing. mark and exit the write loop.
  409. 21321 if siz.zero?
  410. 16 write_drained = !@write_buffer.empty?
  411. 16 break
  412. end
  413. # exit write loop if marked to consume from peer, or is closing.
  414. 21305 break if interests == :r || @state == :closing || @state == :closed
  415. 3069 write_drained = false
  416. 31814 end unless (ints = interests) == :r
  417. 31805 send_pending if @state == :open
  418. # return if socket is drained
  419. 31805 next unless (ints != :r || read_drained) && (ints != :w || write_drained)
  420. # gotta go back to the event loop. It happens when:
  421. #
  422. # * the socket is drained of bytes or it's not the interest of the conn to read;
  423. # * theres nothing more to write, or it's not in the interest of the conn to write;
  424. 11392 log(level: 3) { "(#{ints}): WAITING FOR EVENTS..." }
  425. 11353 return
  426. end
  427. end
  428. end
  429. 27 def send_pending
  430. 82175 while !@write_buffer.full? && (request = @pending.shift)
  431. 18721 send_request_to_parser(request)
  432. end
  433. end
  434. 27 def parser
  435. 110457 @parser ||= build_parser
  436. end
  437. 27 def send_request_to_parser(request)
  438. 18214 @inflight += 1
  439. 19142 request.peer_address = @io.ip
  440. 19142 set_request_timeouts(request)
  441. 19142 parser.send(request)
  442. 19142 return unless @state == :inactive
  443. 7 transition(:active)
  444. end
  445. 27 def build_parser(protocol = @io.protocol)
  446. 7487 parser = parser_type(protocol).new(@write_buffer, @options)
  447. 7487 set_parser_callbacks(parser)
  448. 7487 parser
  449. end
  450. 27 def set_parser_callbacks(parser)
  451. 7598 parser.on(:response) do |request, response|
  452. 8003 AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
  453. 8 emit(:altsvc, alt_origin, origin, alt_params)
  454. end
  455. 8003 @response_received_at = Utils.now
  456. 7154 @inflight -= 1
  457. 8003 response.finish!
  458. 8003 request.emit(:response, response)
  459. end
  460. 7598 parser.on(:altsvc) do |alt_origin, origin, alt_params|
  461. emit(:altsvc, alt_origin, origin, alt_params)
  462. end
  463. 7598 parser.on(:pong, &method(:send_pending))
  464. 7598 parser.on(:promise) do |request, stream|
  465. 24 request.emit(:promise, parser, stream)
  466. end
  467. 7598 parser.on(:exhausted) do
  468. 8 @exhausted = true
  469. 8 current_session = @current_session
  470. 8 current_selector = @current_selector
  471. begin
  472. 8 parser.close
  473. 8 @pending.concat(parser.pending)
  474. ensure
  475. 8 @current_session = current_session
  476. 8 @current_selector = current_selector
  477. end
  478. 7 case @state
  479. when :closed
  480. 8 idling
  481. 8 @exhausted = false
  482. when :closing
  483. once(:closed) do
  484. idling
  485. @exhausted = false
  486. end
  487. end
  488. end
  489. 7598 parser.on(:origin) do |origin|
  490. @origins |= [origin]
  491. end
  492. 7598 parser.on(:close) do |force|
  493. 2816 if force
  494. 2816 reset
  495. 2808 emit(:terminate)
  496. end
  497. end
  498. 7598 parser.on(:close_handshake) do
  499. 8 consume
  500. end
  501. 7598 parser.on(:reset) do
  502. 4044 @pending.concat(parser.pending) unless parser.empty?
  503. 4044 current_session = @current_session
  504. 4044 current_selector = @current_selector
  505. 4044 reset
  506. 4036 unless @pending.empty?
  507. 179 idling
  508. 179 @current_session = current_session
  509. 179 @current_selector = current_selector
  510. end
  511. end
  512. 7598 parser.on(:current_timeout) do
  513. 3208 @current_timeout = @timeout = parser.timeout
  514. end
  515. 7598 parser.on(:timeout) do |tout|
  516. 2751 @timeout = tout
  517. end
  518. 7598 parser.on(:error) do |request, error|
  519. 58 case error
  520. when :http_1_1_required
  521. 16 current_session = @current_session
  522. 16 current_selector = @current_selector
  523. 16 parser.close
  524. 16 other_connection = current_session.find_connection(@origin, current_selector,
  525. @options.merge(ssl: { alpn_protocols: %w[http/1.1] }))
  526. 16 other_connection.merge(self)
  527. 16 request.transition(:idle)
  528. 16 other_connection.send(request)
  529. 16 next
  530. when OperationTimeoutError
  531. # request level timeouts should take precedence
  532. next unless request.active_timeouts.empty?
  533. end
  534. 44 @inflight -= 1
  535. 49 response = ErrorResponse.new(request, error)
  536. 49 request.response = response
  537. 49 request.emit(:response, response)
  538. end
  539. end
  540. 27 def transition(nextstate)
  541. 46441 handle_transition(nextstate)
  542. rescue Errno::ECONNABORTED,
  543. Errno::ECONNREFUSED,
  544. Errno::ECONNRESET,
  545. Errno::EADDRNOTAVAIL,
  546. Errno::EHOSTUNREACH,
  547. Errno::EINVAL,
  548. Errno::ENETUNREACH,
  549. Errno::EPIPE,
  550. Errno::ENOENT,
  551. SocketError,
  552. IOError => e
  553. # connect errors, exit gracefully
  554. 81 error = ConnectionError.new(e.message)
  555. 81 error.set_backtrace(e.backtrace)
  556. 81 handle_connect_error(error) if connecting?
  557. 81 @state = :closed
  558. 81 purge_after_closed
  559. 81 disconnect
  560. rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
  561. # connect errors, exit gracefully
  562. 24 handle_error(e)
  563. 24 handle_connect_error(e) if connecting?
  564. 24 @state = :closed
  565. 24 purge_after_closed
  566. 24 disconnect
  567. end
  568. 27 def handle_transition(nextstate)
  569. 41061 case nextstate
  570. when :idle
  571. 8183 @timeout = @current_timeout = @options.timeout[:connect_timeout]
  572. 8183 @connected_at = @response_received_at = nil
  573. when :open
  574. 21356 return if @state == :closed
  575. 21356 @io.connect
  576. 21251 close_sibling if @io.state == :connected
  577. 21251 return unless @io.connected?
  578. 7492 @connected_at = Utils.now
  579. 7492 send_pending
  580. 7492 @timeout = @current_timeout = parser.timeout
  581. 7492 emit(:open)
  582. when :inactive
  583. 366 return unless @state == :open
  584. # do not deactivate connection in use
  585. 365 return if @inflight.positive?
  586. when :closing
  587. 7446 return unless @state == :idle || @state == :open
  588. 7446 unless @write_buffer.empty?
  589. # preset state before handshake, as error callbacks
  590. # may take it back here.
  591. 2762 @state = nextstate
  592. # handshakes, try sending
  593. 2762 consume
  594. 2762 @write_buffer.clear
  595. 2762 return
  596. end
  597. when :closed
  598. 7738 return unless @state == :closing
  599. 7738 return unless @write_buffer.empty?
  600. 7712 purge_after_closed
  601. 7712 disconnect if @pending.empty?
  602. when :already_open
  603. 66 nextstate = :open
  604. # the first check for given io readiness must still use a timeout.
  605. # connect is the reasonable choice in such a case.
  606. 66 @timeout = @options.timeout[:connect_timeout]
  607. 66 send_pending
  608. when :active
  609. 197 return unless @state == :inactive
  610. 197 nextstate = :open
  611. # activate
  612. 197 @current_session.select_connection(self, @current_selector)
  613. end
  614. 29252 @state = nextstate
  615. end
  616. 27 def close_sibling
  617. 9888 return unless @sibling
  618. if @sibling.io_connected?
  619. reset
  620. # TODO: transition connection to closed
  621. end
  622. unless @sibling.state == :closed
  623. merge(@sibling) unless @main_sibling
  624. @sibling.force_reset(true)
  625. end
  626. @sibling = nil
  627. end
  628. 27 def purge_after_closed
  629. 8634 @io.close if @io
  630. 8634 @read_buffer.clear
  631. 8634 @timeout = nil
  632. end
  633. 27 def initialize_type(uri, options)
  634. 7058 options.transport || begin
  635. 6254 case uri.scheme
  636. when "http"
  637. 4021 "tcp"
  638. when "https"
  639. 3009 "ssl"
  640. else
  641. raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
  642. end
  643. end
  644. end
  645. # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
  646. 27 def build_altsvc_connection(alt_origin, origin, alt_params)
  647. # do not allow security downgrades on altsvc negotiation
  648. 8 return if @origin.scheme == "https" && alt_origin.scheme != "https"
  649. 8 altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
  650. # altsvc already exists, somehow it wasn't advertised, probably noop
  651. 8 return unless altsvc
  652. 8 alt_options = @options.merge(ssl: @options.ssl.merge(hostname: URI(origin).host))
  653. 8 connection = @current_session.find_connection(alt_origin, @current_selector, alt_options)
  654. # advertised altsvc is the same origin being used, ignore
  655. 8 return if connection == self
  656. 8 connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
  657. 8 log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
  658. 8 connection.merge(self)
  659. 8 terminate
  660. rescue UnsupportedSchemeError
  661. altsvc["noop"] = true
  662. nil
  663. end
  664. 27 def build_socket(addrs = nil)
  665. 6215 case @type
  666. when "tcp"
  667. 4077 TCP.new(peer, addrs, @options)
  668. when "ssl"
  669. 2914 SSL.new(peer, addrs, @options) do |sock|
  670. 2892 sock.ssl_session = @ssl_session
  671. 2892 sock.session_new_cb do |sess|
  672. 5025 @ssl_session = sess
  673. 5025 sock.ssl_session = sess
  674. end
  675. end
  676. when "unix"
  677. 28 path = Array(addrs).first
  678. 28 path = String(path) if path
  679. 28 UNIX.new(peer, path, @options)
  680. else
  681. raise Error, "unsupported transport (#{@type})"
  682. end
  683. end
  684. 27 def on_error(error, request = nil)
  685. 615 if error.is_a?(OperationTimeoutError)
  686. # inactive connections do not contribute to the select loop, therefore
  687. # they should not fail due to such errors.
  688. 32 return if @state == :inactive
  689. 32 if @timeout
  690. 28 @timeout -= error.timeout
  691. 32 return unless @timeout <= 0
  692. end
  693. 32 error = error.to_connection_error if connecting?
  694. end
  695. 615 handle_error(error, request)
  696. 615 reset
  697. end
  698. 27 def handle_error(error, request = nil)
  699. 923 parser.handle_error(error, request) if @parser && parser.respond_to?(:handle_error)
  700. 2033 while (req = @pending.shift)
  701. 436 next if request && req == request
  702. 436 response = ErrorResponse.new(req, error)
  703. 436 req.response = response
  704. 420 req.emit(:response, response)
  705. end
  706. 907 return unless request
  707. 357 @inflight -= 1
  708. 408 response = ErrorResponse.new(request, error)
  709. 408 request.response = response
  710. 408 request.emit(:response, response)
  711. end
  712. 27 def set_request_timeouts(request)
  713. 19142 set_request_write_timeout(request)
  714. 19142 set_request_read_timeout(request)
  715. 19142 set_request_request_timeout(request)
  716. end
  717. 27 def set_request_read_timeout(request)
  718. 19142 read_timeout = request.read_timeout
  719. 19142 return if read_timeout.nil? || read_timeout.infinite?
  720. 18828 set_request_timeout(:read_timeout, request, read_timeout, :done, :response) do
  721. 16 read_timeout_callback(request, read_timeout)
  722. end
  723. end
  724. 27 def set_request_write_timeout(request)
  725. 19142 write_timeout = request.write_timeout
  726. 19142 return if write_timeout.nil? || write_timeout.infinite?
  727. 19142 set_request_timeout(:write_timeout, request, write_timeout, :headers, %i[done response]) do
  728. 16 write_timeout_callback(request, write_timeout)
  729. end
  730. end
  731. 27 def set_request_request_timeout(request)
  732. 18858 request_timeout = request.request_timeout
  733. 18858 return if request_timeout.nil? || request_timeout.infinite?
  734. 549 set_request_timeout(:request_timeout, request, request_timeout, :headers, :complete) do
  735. 412 read_timeout_callback(request, request_timeout, RequestTimeoutError)
  736. end
  737. end
  738. 27 def write_timeout_callback(request, write_timeout)
  739. 16 return if request.state == :done
  740. 16 @write_buffer.clear
  741. 16 error = WriteTimeoutError.new(request, nil, write_timeout)
  742. 16 on_error(error, request)
  743. end
  744. 27 def read_timeout_callback(request, read_timeout, error_type = ReadTimeoutError)
  745. 428 response = request.response
  746. 428 return if response && response.finished?
  747. 392 @write_buffer.clear
  748. 392 error = error_type.new(request, request.response, read_timeout)
  749. 392 on_error(error, request)
  750. end
  751. 27 def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
  752. 38599 request.set_timeout_callback(start_event) do
  753. 38399 timer = @current_selector.after(timeout, callback)
  754. 38399 request.active_timeouts << label
  755. 38399 Array(finish_events).each do |event|
  756. # clean up request timeouts if the connection errors out
  757. 57533 request.set_timeout_callback(event) do
  758. 57074 timer.cancel
  759. 57074 request.active_timeouts.delete(label)
  760. end
  761. end
  762. end
  763. end
  764. 27 def parser_type(protocol)
  765. 6746 case protocol
  766. 3208 when "h2" then HTTP2
  767. 4407 when "http/1.1" then HTTP1
  768. else
  769. raise Error, "unsupported protocol (##{protocol})"
  770. end
  771. end
  772. end
  773. end

lib/httpx/connection/http1.rb

89.14% lines covered

221 relevant lines. 197 lines covered and 24 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "httpx/parser/http1"
  3. 27 module HTTPX
  4. 27 class Connection::HTTP1
  5. 27 include Callbacks
  6. 27 include Loggable
  7. 27 MAX_REQUESTS = 200
  8. 27 CRLF = "\r\n"
  9. 27 attr_reader :pending, :requests
  10. 27 attr_accessor :max_concurrent_requests
  11. 27 def initialize(buffer, options)
  12. 4407 @options = options
  13. 4407 @max_concurrent_requests = @options.max_concurrent_requests || MAX_REQUESTS
  14. 4407 @max_requests = @options.max_requests
  15. 4407 @parser = Parser::HTTP1.new(self)
  16. 4407 @buffer = buffer
  17. 4407 @version = [1, 1]
  18. 4407 @pending = []
  19. 4407 @requests = []
  20. 4407 @handshake_completed = false
  21. end
  22. 27 def timeout
  23. 4290 @options.timeout[:operation_timeout]
  24. end
  25. 27 def interests
  26. # this means we're processing incoming response already
  27. 30344 return :r if @request
  28. 25212 return if @requests.empty?
  29. 25190 request = @requests.first
  30. 25190 return unless request
  31. 25190 return :w if request.interests == :w || !@buffer.empty?
  32. 22265 :r
  33. end
  34. 27 def reset
  35. 4174 @max_requests = @options.max_requests || MAX_REQUESTS
  36. 4174 @parser.reset!
  37. 4174 @handshake_completed = false
  38. 4174 @pending.concat(@requests) unless @requests.empty?
  39. end
  40. 27 def close
  41. 82 reset
  42. 82 emit(:close, true)
  43. end
  44. 27 def exhausted?
  45. 529 !@max_requests.positive?
  46. end
  47. 27 def empty?
  48. # this means that for every request there's an available
  49. # partial response, so there are no in-flight requests waiting.
  50. 4093 @requests.empty? || (
  51. # checking all responses can be time-consuming. Alas, as in HTTP/1, responses
  52. # do not come out of order, we can get away with checking first and last.
  53. 185 !@requests.first.response.nil? &&
  54. (@requests.size == 1 || !@requests.last.response.nil?)
  55. )
  56. end
  57. 27 def <<(data)
  58. 7166 @parser << data
  59. end
  60. 27 def send(request)
  61. 15504 unless @max_requests.positive?
  62. @pending << request
  63. return
  64. end
  65. 15504 return if @requests.include?(request)
  66. 15504 @requests << request
  67. 15504 @pipelining = true if @requests.size > 1
  68. end
  69. 27 def consume
  70. 16079 requests_limit = [@max_requests, @requests.size].min
  71. 16079 concurrent_requests_limit = [@max_concurrent_requests, requests_limit].min
  72. 16079 @requests.each_with_index do |request, idx|
  73. 18653 break if idx >= concurrent_requests_limit
  74. 16049 next unless request.can_buffer?
  75. 6146 handle(request)
  76. end
  77. end
  78. # HTTP Parser callbacks
  79. #
  80. # must be public methods, or else they won't be reachable
  81. 27 def on_start
  82. 4806 log(level: 2) { "parsing begins" }
  83. end
  84. 27 def on_headers(h)
  85. 4774 @request = @requests.first
  86. 4774 return if @request.response
  87. 4806 log(level: 2) { "headers received" }
  88. 4774 headers = @request.options.headers_class.new(h)
  89. 4774 response = @request.options.response_class.new(@request,
  90. @parser.status_code,
  91. @parser.http_version.join("."),
  92. headers)
  93. 4806 log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
  94. 5062 log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{log_redact(v)}" }.join("\n") }
  95. 4774 @request.response = response
  96. 4766 on_complete if response.finished?
  97. end
  98. 27 def on_trailers(h)
  99. 8 return unless @request
  100. 8 response = @request.response
  101. 8 log(level: 2) { "trailer headers received" }
  102. 8 log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{log_redact(v.join(", "))}" }.join("\n") }
  103. 8 response.merge_headers(h)
  104. end
  105. 27 def on_data(chunk)
  106. 5576 request = @request
  107. 5576 return unless request
  108. 5611 log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
  109. 5611 log(level: 2, color: :green) { "-> #{log_redact(chunk.inspect)}" }
  110. 5576 response = request.response
  111. 5576 response << chunk
  112. rescue StandardError => e
  113. 14 error_response = ErrorResponse.new(request, e)
  114. 14 request.response = error_response
  115. 14 dispatch
  116. end
  117. 27 def on_complete
  118. 4736 request = @request
  119. 4736 return unless request
  120. 4768 log(level: 2) { "parsing complete" }
  121. 4736 dispatch
  122. end
  123. 27 def dispatch
  124. 4750 request = @request
  125. 4750 if request.expects?
  126. 72 @parser.reset!
  127. 63 return handle(request)
  128. end
  129. 4678 @request = nil
  130. 4678 @requests.shift
  131. 4678 response = request.response
  132. 4678 emit(:response, request, response)
  133. 4621 if @parser.upgrade?
  134. 32 response << @parser.upgrade_data
  135. 32 throw(:called)
  136. end
  137. 4589 @parser.reset!
  138. 4121 @max_requests -= 1
  139. 4589 if response.is_a?(ErrorResponse)
  140. 14 disable
  141. else
  142. 4575 manage_connection(request, response)
  143. end
  144. 529 if exhausted?
  145. @pending.concat(@requests)
  146. @requests.clear
  147. emit(:exhausted)
  148. else
  149. 529 send(@pending.shift) unless @pending.empty?
  150. end
  151. end
  152. 27 def handle_error(ex, request = nil)
  153. 214 if (ex.is_a?(EOFError) || ex.is_a?(TimeoutError)) && @request && @request.response &&
  154. !@request.response.headers.key?("content-length") &&
  155. !@request.response.headers.key?("transfer-encoding")
  156. # if the response does not contain a content-length header, the server closing the
  157. # connnection is the indicator of response consumed.
  158. # https://greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4
  159. 32 catch(:called) { on_complete }
  160. 14 return
  161. end
  162. 198 if @pipelining
  163. catch(:called) { disable }
  164. else
  165. 198 @requests.each do |req|
  166. 185 next if request && request == req
  167. emit(:error, req, ex)
  168. end
  169. 198 @pending.each do |req|
  170. next if request && request == req
  171. emit(:error, req, ex)
  172. end
  173. end
  174. end
  175. 27 def ping
  176. reset
  177. emit(:reset)
  178. emit(:exhausted)
  179. end
  180. 27 private
  181. 27 def manage_connection(request, response)
  182. 4575 connection = response.headers["connection"]
  183. 4108 case connection
  184. when /keep-alive/i
  185. 529 if @handshake_completed
  186. if @max_requests.zero?
  187. @pending.concat(@requests)
  188. @requests.clear
  189. emit(:exhausted)
  190. end
  191. return
  192. end
  193. 529 keep_alive = response.headers["keep-alive"]
  194. 529 return unless keep_alive
  195. 108 parameters = Hash[keep_alive.split(/ *, */).map do |pair|
  196. 108 pair.split(/ *= */, 2)
  197. end]
  198. 108 @max_requests = parameters["max"].to_i - 1 if parameters.key?("max")
  199. 108 if parameters.key?("timeout")
  200. keep_alive_timeout = parameters["timeout"].to_i
  201. emit(:timeout, keep_alive_timeout)
  202. end
  203. 108 @handshake_completed = true
  204. when /close/i
  205. 4046 disable
  206. when nil
  207. # In HTTP/1.1, it's keep alive by default
  208. return if response.version == "1.1" && request.headers["connection"] != "close"
  209. disable
  210. end
  211. end
  212. 27 def disable
  213. 4060 disable_pipelining
  214. 4060 reset
  215. 4060 emit(:reset)
  216. 4052 throw(:called)
  217. end
  218. 27 def disable_pipelining
  219. 4060 return if @requests.empty?
  220. # do not disable pipelining if already set to 1 request at a time
  221. 187 return if @max_concurrent_requests == 1
  222. 29 @requests.each do |r|
  223. 29 r.transition(:idle)
  224. # when we disable pipelining, we still want to try keep-alive.
  225. # only when keep-alive with one request fails, do we fallback to
  226. # connection: close.
  227. 29 r.headers["connection"] = "close" if @max_concurrent_requests == 1
  228. end
  229. # server doesn't handle pipelining, and probably
  230. # doesn't support keep-alive. Fallback to send only
  231. # 1 keep alive request.
  232. 29 @max_concurrent_requests = 1
  233. 29 @pipelining = false
  234. end
  235. 27 def set_protocol_headers(request)
  236. 4916 if !request.headers.key?("content-length") &&
  237. request.body.bytesize == Float::INFINITY
  238. 32 request.body.chunk!
  239. end
  240. 4916 extra_headers = {}
  241. 4916 unless request.headers.key?("connection")
  242. 4892 connection_value = if request.persistent?
  243. # when in a persistent connection, the request can't be at
  244. # the edge of a renegotiation
  245. 97 if @requests.index(request) + 1 < @max_requests
  246. 97 "keep-alive"
  247. else
  248. "close"
  249. end
  250. else
  251. # when it's not a persistent connection, it sets "Connection: close" always
  252. # on the last request of the possible batch (either allowed max requests,
  253. # or if smaller, the size of the batch itself)
  254. 4795 requests_limit = [@max_requests, @requests.size].min
  255. 4795 if request == @requests[requests_limit - 1]
  256. 4230 "close"
  257. else
  258. 565 "keep-alive"
  259. end
  260. end
  261. 4387 extra_headers["connection"] = connection_value
  262. end
  263. 4916 extra_headers["host"] = request.authority unless request.headers.key?("host")
  264. 4916 extra_headers
  265. end
  266. 27 def handle(request)
  267. 6218 catch(:buffer_full) do
  268. 6218 request.transition(:headers)
  269. 6210 join_headers(request) if request.state == :headers
  270. 6210 request.transition(:body)
  271. 6210 join_body(request) if request.state == :body
  272. 5118 request.transition(:trailers)
  273. # HTTP/1.1 trailers should only work for chunked encoding
  274. 5118 join_trailers(request) if request.body.chunked? && request.state == :trailers
  275. 5118 request.transition(:done)
  276. end
  277. end
  278. 27 def join_headline(request)
  279. 4338 "#{request.verb} #{request.path} HTTP/#{@version.join(".")}"
  280. end
  281. 27 def join_headers(request)
  282. 4916 headline = join_headline(request)
  283. 4916 @buffer << headline << CRLF
  284. 4948 log(color: :yellow) { "<- HEADLINE: #{headline.chomp.inspect}" }
  285. 4916 extra_headers = set_protocol_headers(request)
  286. 4916 join_headers2(request.headers.each(extra_headers))
  287. 4948 log { "<- " }
  288. 4916 @buffer << CRLF
  289. end
  290. 27 def join_body(request)
  291. 5992 return if request.body.empty?
  292. 6565 while (chunk = request.drain_body)
  293. 3280 log(color: :green) { "<- DATA: #{chunk.bytesize} bytes..." }
  294. 3280 log(level: 2, color: :green) { "<- #{log_redact(chunk.inspect)}" }
  295. 3280 @buffer << chunk
  296. 3280 throw(:buffer_full, request) if @buffer.full?
  297. end
  298. 1556 return unless (error = request.drain_error)
  299. raise error
  300. end
  301. 27 def join_trailers(request)
  302. 96 return unless request.trailers? && request.callbacks_for?(:trailers)
  303. 32 join_headers2(request.trailers)
  304. 32 log { "<- " }
  305. 32 @buffer << CRLF
  306. end
  307. 27 def join_headers2(headers)
  308. 4948 headers.each do |field, value|
  309. 30377 field = capitalized(field)
  310. 30537 log(color: :yellow) { "<- HEADER: #{[field, log_redact(value)].join(": ")}" }
  311. 30377 @buffer << "#{field}: #{value}#{CRLF}"
  312. end
  313. end
  314. 27 UPCASED = {
  315. "www-authenticate" => "WWW-Authenticate",
  316. "http2-settings" => "HTTP2-Settings",
  317. "content-md5" => "Content-MD5",
  318. }.freeze
  319. 27 def capitalized(field)
  320. 30377 UPCASED[field] || field.split("-").map(&:capitalize).join("-")
  321. end
  322. end
  323. end

lib/httpx/connection/http2.rb

95.09% lines covered

265 relevant lines. 252 lines covered and 13 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "securerandom"
  3. 27 require "http/2"
  4. 27 module HTTPX
  5. 27 class Connection::HTTP2
  6. 27 include Callbacks
  7. 27 include Loggable
  8. 27 MAX_CONCURRENT_REQUESTS = ::HTTP2::DEFAULT_MAX_CONCURRENT_STREAMS
  9. 27 class Error < Error
  10. 27 def initialize(id, error)
  11. 48 super("stream #{id} closed with error: #{error}")
  12. end
  13. end
  14. 27 class PingError < Error
  15. 27 def initialize
  16. super(0, :ping_error)
  17. end
  18. end
  19. 27 class GoawayError < Error
  20. 27 def initialize
  21. 17 super(0, :no_error)
  22. end
  23. end
  24. 27 attr_reader :streams, :pending
  25. 27 def initialize(buffer, options)
  26. 3248 @options = options
  27. 3248 @settings = @options.http2_settings
  28. 3248 @pending = []
  29. 3248 @streams = {}
  30. 3248 @drains = {}
  31. 3248 @pings = []
  32. 3248 @buffer = buffer
  33. 3248 @handshake_completed = false
  34. 3248 @wait_for_handshake = @settings.key?(:wait_for_handshake) ? @settings.delete(:wait_for_handshake) : true
  35. 3248 @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
  36. 3248 @max_requests = @options.max_requests
  37. 3248 init_connection
  38. end
  39. 27 def timeout
  40. 6410 return @options.timeout[:operation_timeout] if @handshake_completed
  41. 3202 @options.timeout[:settings_timeout]
  42. end
  43. 27 def interests
  44. # waiting for WINDOW_UPDATE frames
  45. 10757142 return :r if @buffer.full?
  46. 10757142 if @connection.state == :closed
  47. 2903 return unless @handshake_completed
  48. 2515 return :w
  49. end
  50. 10754239 unless @connection.state == :connected && @handshake_completed
  51. 12356 return @buffer.empty? ? :r : :rw
  52. end
  53. 10740398 return :w if !@pending.empty? && can_buffer_more_requests?
  54. 10740398 return :w unless @drains.empty?
  55. 10739420 if @buffer.empty?
  56. 10739420 return if @streams.empty? && @pings.empty?
  57. 39646 return :r
  58. end
  59. :rw
  60. end
  61. 27 def close
  62. 2759 unless @connection.state == :closed
  63. 2751 @connection.goaway
  64. 2751 emit(:timeout, @options.timeout[:close_handshake_timeout])
  65. end
  66. 2759 emit(:close, true)
  67. end
  68. 27 def empty?
  69. 2753 @connection.state == :closed || @streams.empty?
  70. end
  71. 27 def exhausted?
  72. 3252 !@max_requests.positive?
  73. end
  74. 27 def <<(data)
  75. 32896 @connection << data
  76. end
  77. 27 def send(request, head = false)
  78. 7093 unless can_buffer_more_requests?
  79. 3438 head ? @pending.unshift(request) : @pending << request
  80. 3438 return false
  81. end
  82. 3655 unless (stream = @streams[request])
  83. 3655 stream = @connection.new_stream
  84. 3655 handle_stream(stream, request)
  85. 3245 @streams[request] = stream
  86. 3245 @max_requests -= 1
  87. end
  88. 3655 handle(request, stream)
  89. 3639 true
  90. rescue ::HTTP2::Error::StreamLimitExceeded
  91. @pending.unshift(request)
  92. false
  93. end
  94. 27 def consume
  95. 25350 @streams.each do |request, stream|
  96. 10022 next unless request.can_buffer?
  97. 1115 handle(request, stream)
  98. end
  99. end
  100. 27 def handle_error(ex, request = nil)
  101. 273 if ex.is_a?(OperationTimeoutError) && !@handshake_completed && @connection.state != :closed
  102. 8 @connection.goaway(:settings_timeout, "closing due to settings timeout")
  103. 8 emit(:close_handshake)
  104. 8 settings_ex = SettingsTimeoutError.new(ex.timeout, ex.message)
  105. 8 settings_ex.set_backtrace(ex.backtrace)
  106. 8 ex = settings_ex
  107. end
  108. 273 @streams.each_key do |req|
  109. 224 next if request && request == req
  110. 16 emit(:error, req, ex)
  111. end
  112. 513 while (req = @pending.shift)
  113. 33 next if request && request == req
  114. 33 emit(:error, req, ex)
  115. end
  116. end
  117. 27 def ping
  118. 20 ping = SecureRandom.gen_random(8)
  119. 20 @connection.ping(ping.dup)
  120. ensure
  121. 20 @pings << ping
  122. end
  123. 27 private
  124. 27 def can_buffer_more_requests?
  125. 7592 (@handshake_completed || !@wait_for_handshake) &&
  126. @streams.size < @max_concurrent_requests &&
  127. @streams.size < @max_requests
  128. end
  129. 27 def send_pending
  130. 8556 while (request = @pending.shift)
  131. 3312 break unless send(request, true)
  132. end
  133. end
  134. 27 def handle(request, stream)
  135. 4914 catch(:buffer_full) do
  136. 4914 request.transition(:headers)
  137. 4906 join_headers(stream, request) if request.state == :headers
  138. 4906 request.transition(:body)
  139. 4906 join_body(stream, request) if request.state == :body
  140. 3816 request.transition(:trailers)
  141. 3816 join_trailers(stream, request) if request.state == :trailers && !request.body.empty?
  142. 3816 request.transition(:done)
  143. end
  144. end
  145. 27 def init_connection
  146. 3248 @connection = ::HTTP2::Client.new(@settings)
  147. 3248 @connection.on(:frame, &method(:on_frame))
  148. 3248 @connection.on(:frame_sent, &method(:on_frame_sent))
  149. 3248 @connection.on(:frame_received, &method(:on_frame_received))
  150. 3248 @connection.on(:origin, &method(:on_origin))
  151. 3248 @connection.on(:promise, &method(:on_promise))
  152. 3248 @connection.on(:altsvc) { |frame| on_altsvc(frame[:origin], frame) }
  153. 3248 @connection.on(:settings_ack, &method(:on_settings))
  154. 3248 @connection.on(:ack, &method(:on_pong))
  155. 3248 @connection.on(:goaway, &method(:on_close))
  156. #
  157. # Some servers initiate HTTP/2 negotiation right away, some don't.
  158. # As such, we have to check the socket buffer. If there is something
  159. # to read, the server initiated the negotiation. If not, we have to
  160. # initiate it.
  161. #
  162. 3248 @connection.send_connection_preface
  163. end
  164. 27 alias_method :reset, :init_connection
  165. 27 public :reset
  166. 27 def handle_stream(stream, request)
  167. 3671 request.on(:refuse, &method(:on_stream_refuse).curry(3)[stream, request])
  168. 3671 stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
  169. 3671 stream.on(:half_close) do
  170. 3637 log(level: 2) { "#{stream.id}: waiting for response..." }
  171. end
  172. 3671 stream.on(:altsvc, &method(:on_altsvc).curry(2)[request.origin])
  173. 3671 stream.on(:headers, &method(:on_stream_headers).curry(3)[stream, request])
  174. 3671 stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
  175. end
  176. 27 def set_protocol_headers(request)
  177. 409 {
  178. 3237 ":scheme" => request.scheme,
  179. ":method" => request.verb,
  180. ":path" => request.path,
  181. ":authority" => request.authority,
  182. }
  183. end
  184. 27 def join_headers(stream, request)
  185. 3647 extra_headers = set_protocol_headers(request)
  186. 3647 if request.headers.key?("host")
  187. 8 log { "forbidden \"host\" header found (#{log_redact(request.headers["host"])}), will use it as authority..." }
  188. 7 extra_headers[":authority"] = request.headers["host"]
  189. end
  190. 3647 log(level: 1, color: :yellow) do
  191. 128 request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{log_redact(v)}" }.join("\n")
  192. end
  193. 3647 stream.headers(request.headers.each(extra_headers), end_stream: request.body.empty?)
  194. end
  195. 27 def join_trailers(stream, request)
  196. 1529 unless request.trailers?
  197. 1521 stream.data("", end_stream: true) if request.callbacks_for?(:trailers)
  198. 1358 return
  199. end
  200. 8 log(level: 1, color: :yellow) do
  201. 15 request.trailers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{log_redact(v)}" }.join("\n")
  202. end
  203. 8 stream.headers(request.trailers.each, end_stream: true)
  204. end
  205. 27 def join_body(stream, request)
  206. 4722 return if request.body.empty?
  207. 2620 chunk = @drains.delete(request) || request.drain_body
  208. 2814 while chunk
  209. 2835 next_chunk = request.drain_body
  210. 2835 send_chunk(request, stream, chunk, next_chunk)
  211. 2739 if next_chunk && (@buffer.full? || request.body.unbounded_body?)
  212. 874 @drains[request] = next_chunk
  213. 994 throw(:buffer_full)
  214. end
  215. 1745 chunk = next_chunk
  216. end
  217. 1530 return unless (error = request.drain_error)
  218. 24 on_stream_refuse(stream, request, error)
  219. end
  220. 27 def send_chunk(request, stream, chunk, next_chunk)
  221. 2856 log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
  222. 2856 log(level: 2, color: :green) { "#{stream.id}: -> #{log_redact(chunk.inspect)}" }
  223. 2835 stream.data(chunk, end_stream: end_stream?(request, next_chunk))
  224. end
  225. 27 def end_stream?(request, next_chunk)
  226. 2739 !(next_chunk || request.trailers? || request.callbacks_for?(:trailers))
  227. end
  228. ######
  229. # HTTP/2 Callbacks
  230. ######
  231. 27 def on_stream_headers(stream, request, h)
  232. 3604 response = request.response
  233. 3604 if response.is_a?(Response) && response.version == "2.0"
  234. 114 on_stream_trailers(stream, response, h)
  235. 114 return
  236. end
  237. 3490 log(color: :yellow) do
  238. 128 h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{log_redact(v)}" }.join("\n")
  239. end
  240. 3490 _, status = h.shift
  241. 3490 headers = request.options.headers_class.new(h)
  242. 3490 response = request.options.response_class.new(request, status, "2.0", headers)
  243. 3490 request.response = response
  244. 3090 @streams[request] = stream
  245. 3482 handle(request, stream) if request.expects?
  246. end
  247. 27 def on_stream_trailers(stream, response, h)
  248. 114 log(color: :yellow) do
  249. h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{log_redact(v)}" }.join("\n")
  250. end
  251. 114 response.merge_headers(h)
  252. end
  253. 27 def on_stream_data(stream, request, data)
  254. 6448 log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
  255. 6448 log(level: 2, color: :green) { "#{stream.id}: <- #{log_redact(data.inspect)}" }
  256. 6429 request.response << data
  257. end
  258. 27 def on_stream_refuse(stream, request, error)
  259. 24 on_stream_close(stream, request, error)
  260. 24 stream.close
  261. end
  262. 27 def on_stream_close(stream, request, error)
  263. 3422 return if error == :stream_closed && !@streams.key?(request)
  264. 3412 log(level: 2) { "#{stream.id}: closing stream" }
  265. 3398 @drains.delete(request)
  266. 3398 @streams.delete(request)
  267. 3398 if error
  268. 24 case error
  269. when :http_1_1_required
  270. emit(:error, request, error)
  271. else
  272. 24 ex = Error.new(stream.id, error)
  273. 24 ex.set_backtrace(caller)
  274. 24 response = ErrorResponse.new(request, ex)
  275. 24 request.response = response
  276. 24 emit(:response, request, response)
  277. end
  278. else
  279. 3374 response = request.response
  280. 3374 if response && response.is_a?(Response) && response.status == 421
  281. 8 emit(:error, request, :http_1_1_required)
  282. else
  283. 3366 emit(:response, request, response)
  284. end
  285. end
  286. 3390 send(@pending.shift) unless @pending.empty?
  287. 3390 return unless @streams.empty? && exhausted?
  288. 8 if @pending.empty?
  289. close
  290. else
  291. 8 emit(:exhausted)
  292. end
  293. end
  294. 27 def on_frame(bytes)
  295. 20144 @buffer << bytes
  296. end
  297. 27 def on_settings(*)
  298. 3208 @handshake_completed = true
  299. 3208 emit(:current_timeout)
  300. 3208 @max_concurrent_requests = [@max_concurrent_requests, @connection.remote_settings[:settings_max_concurrent_streams]].min
  301. 3208 send_pending
  302. end
  303. 27 def on_close(_last_frame, error, _payload)
  304. 32 is_connection_closed = @connection.state == :closed
  305. 32 if error
  306. 32 @buffer.clear if is_connection_closed
  307. 28 case error
  308. when :http_1_1_required
  309. 21 while (request = @pending.shift)
  310. 8 emit(:error, request, error)
  311. end
  312. when :no_error
  313. 17 ex = GoawayError.new
  314. 17 @pending.unshift(*@streams.keys)
  315. 17 @drains.clear
  316. 17 @streams.clear
  317. else
  318. 7 ex = Error.new(0, error)
  319. end
  320. 32 if ex
  321. 24 ex.set_backtrace(caller)
  322. 24 handle_error(ex)
  323. end
  324. end
  325. 32 return unless is_connection_closed && @streams.empty?
  326. 32 emit(:close, is_connection_closed)
  327. end
  328. 27 def on_frame_sent(frame)
  329. 16971 log(level: 2) { "#{frame[:stream]}: frame was sent!" }
  330. 16887 log(level: 2, color: :blue) do
  331. 12 payload =
  332. 83 case frame[:type]
  333. when :data
  334. 24 frame.merge(payload: frame[:payload].bytesize)
  335. when :headers, :ping
  336. 24 frame.merge(payload: log_redact(frame[:payload]))
  337. else
  338. 48 frame
  339. end
  340. 84 "#{frame[:stream]}: #{payload}"
  341. end
  342. end
  343. 27 def on_frame_received(frame)
  344. 17846 log(level: 2) { "#{frame[:stream]}: frame was received!" }
  345. 17785 log(level: 2, color: :magenta) do
  346. 8 payload =
  347. 60 case frame[:type]
  348. when :data
  349. 21 frame.merge(payload: frame[:payload].bytesize)
  350. when :headers, :ping
  351. 16 frame.merge(payload: log_redact(frame[:payload]))
  352. else
  353. 32 frame
  354. end
  355. 61 "#{frame[:stream]}: #{payload}"
  356. end
  357. end
  358. 27 def on_altsvc(origin, frame)
  359. log(level: 2) { "#{frame[:stream]}: altsvc frame was received" }
  360. log(level: 2) { "#{frame[:stream]}: #{log_redact(frame.inspect)}" }
  361. alt_origin = URI.parse("#{frame[:proto]}://#{frame[:host]}:#{frame[:port]}")
  362. params = { "ma" => frame[:max_age] }
  363. emit(:altsvc, origin, alt_origin, origin, params)
  364. end
  365. 27 def on_promise(stream)
  366. 24 emit(:promise, @streams.key(stream.parent), stream)
  367. end
  368. 27 def on_origin(origin)
  369. emit(:origin, origin)
  370. end
  371. 27 def on_pong(ping)
  372. 8 raise PingError unless @pings.delete(ping.to_s)
  373. 8 emit(:pong)
  374. end
  375. end
  376. end

lib/httpx/domain_name.rb

95.56% lines covered

45 relevant lines. 43 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. #
  3. # domain_name.rb - Domain Name manipulation library for Ruby
  4. #
  5. # Copyright (C) 2011-2017 Akinori MUSHA, All rights reserved.
  6. #
  7. # Redistribution and use in source and binary forms, with or without
  8. # modification, are permitted provided that the following conditions
  9. # are met:
  10. # 1. Redistributions of source code must retain the above copyright
  11. # notice, this list of conditions and the following disclaimer.
  12. # 2. Redistributions in binary form must reproduce the above copyright
  13. # notice, this list of conditions and the following disclaimer in the
  14. # documentation and/or other materials provided with the distribution.
  15. #
  16. # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17. # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19. # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20. # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24. # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25. # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26. # SUCH DAMAGE.
  27. 27 require "ipaddr"
  28. 27 module HTTPX
  29. # Represents a domain name ready for extracting its registered domain
  30. # and TLD.
  31. 27 class DomainName
  32. 27 include Comparable
  33. # The full host name normalized, ASCII-ized and downcased using the
  34. # Unicode NFC rules and the Punycode algorithm. If initialized with
  35. # an IP address, the string representation of the IP address
  36. # suitable for opening a connection to.
  37. 27 attr_reader :hostname
  38. # The Unicode representation of the #hostname property.
  39. #
  40. # :attr_reader: hostname_idn
  41. # The least "universally original" domain part of this domain name.
  42. # For example, "example.co.uk" for "www.sub.example.co.uk". This
  43. # may be nil if the hostname does not have one, like when it is an
  44. # IP address, an effective TLD or higher itself, or of a
  45. # non-canonical domain.
  46. 27 attr_reader :domain
  47. 27 class << self
  48. 27 def new(domain)
  49. 856 return domain if domain.is_a?(self)
  50. 792 super(domain)
  51. end
  52. # Normalizes a _domain_ using the Punycode algorithm as necessary.
  53. # The result will be a downcased, ASCII-only string.
  54. 27 def normalize(domain)
  55. 760 unless domain.ascii_only?
  56. domain = domain.chomp(".").unicode_normalize(:nfc)
  57. domain = Punycode.encode_hostname(domain)
  58. end
  59. 760 domain.downcase
  60. end
  61. end
  62. # Parses _hostname_ into a DomainName object. An IP address is also
  63. # accepted. An IPv6 address may be enclosed in square brackets.
  64. 27 def initialize(hostname)
  65. 792 hostname = String(hostname)
  66. 792 raise ArgumentError, "domain name must not start with a dot: #{hostname}" if hostname.start_with?(".")
  67. 98 begin
  68. 792 @ipaddr = IPAddr.new(hostname)
  69. 32 @hostname = @ipaddr.to_s
  70. 32 return
  71. rescue IPAddr::Error
  72. 760 nil
  73. end
  74. 760 @hostname = DomainName.normalize(hostname)
  75. 760 tld = if (last_dot = @hostname.rindex("."))
  76. 184 @hostname[(last_dot + 1)..-1]
  77. else
  78. 576 @hostname
  79. end
  80. # unknown/local TLD
  81. 760 @domain = if last_dot
  82. # fallback - accept cookies down to second level
  83. # cf. http://www.dkim-reputation.org/regdom-libs/
  84. 184 if (penultimate_dot = @hostname.rindex(".", last_dot - 1))
  85. 48 @hostname[(penultimate_dot + 1)..-1]
  86. else
  87. 136 @hostname
  88. end
  89. else
  90. # no domain part - must be a local hostname
  91. 576 tld
  92. end
  93. end
  94. # Checks if the server represented by this domain is qualified to
  95. # send and receive cookies with a domain attribute value of
  96. # _domain_. A true value given as the second argument represents
  97. # cookies without a domain attribute value, in which case only
  98. # hostname equality is checked.
  99. 27 def cookie_domain?(domain, host_only = false)
  100. # RFC 6265 #5.3
  101. # When the user agent "receives a cookie":
  102. 32 return self == @domain if host_only
  103. 32 domain = DomainName.new(domain)
  104. # RFC 6265 #5.1.3
  105. # Do not perform subdomain matching against IP addresses.
  106. 32 @hostname == domain.hostname if @ipaddr
  107. # RFC 6265 #4.1.1
  108. # Domain-value must be a subdomain.
  109. 32 @domain && self <= domain && domain <= @domain
  110. end
  111. 27 def <=>(other)
  112. 48 other = DomainName.new(other)
  113. 48 othername = other.hostname
  114. 48 if othername == @hostname
  115. 16 0
  116. 31 elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == "."
  117. # The other is higher
  118. 16 -1
  119. else
  120. # The other is lower
  121. 16 1
  122. end
  123. end
  124. end
  125. end

lib/httpx/errors.rb

97.62% lines covered

42 relevant lines. 41 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # the default exception class for exceptions raised by HTTPX.
  4. 27 class Error < StandardError; end
  5. 27 class UnsupportedSchemeError < Error; end
  6. 27 class ConnectionError < Error; end
  7. # Error raised when there was a timeout. Its subclasses allow for finer-grained
  8. # control of which timeout happened.
  9. 27 class TimeoutError < Error
  10. # The timeout value which caused this error to be raised.
  11. 27 attr_reader :timeout
  12. # initializes the timeout exception with the +timeout+ causing the error, and the
  13. # error +message+ for it.
  14. 27 def initialize(timeout, message)
  15. 519 @timeout = timeout
  16. 519 super(message)
  17. end
  18. # clones this error into a HTTPX::ConnectionTimeoutError.
  19. 27 def to_connection_error
  20. 24 ex = ConnectTimeoutError.new(@timeout, message)
  21. 24 ex.set_backtrace(backtrace)
  22. 24 ex
  23. end
  24. end
  25. # Raise when it can't acquire a connection from the pool.
  26. 27 class PoolTimeoutError < TimeoutError; end
  27. # Error raised when there was a timeout establishing the connection to a server.
  28. # This may be raised due to timeouts during TCP and TLS (when applicable) connection
  29. # establishment.
  30. 27 class ConnectTimeoutError < TimeoutError; end
  31. # Error raised when there was a timeout while sending a request, or receiving a response
  32. # from the server.
  33. 27 class RequestTimeoutError < TimeoutError
  34. # The HTTPX::Request request object this exception refers to.
  35. 27 attr_reader :request
  36. # initializes the exception with the +request+ and +response+ it refers to, and the
  37. # +timeout+ causing the error, and the
  38. 27 def initialize(request, response, timeout)
  39. 408 @request = request
  40. 408 @response = response
  41. 408 super(timeout, "Timed out after #{timeout} seconds")
  42. end
  43. 27 def marshal_dump
  44. [message]
  45. end
  46. end
  47. # Error raised when there was a timeout while receiving a response from the server.
  48. 27 class ReadTimeoutError < RequestTimeoutError; end
  49. # Error raised when there was a timeout while sending a request from the server.
  50. 27 class WriteTimeoutError < RequestTimeoutError; end
  51. # Error raised when there was a timeout while waiting for the HTTP/2 settings frame from the server.
  52. 27 class SettingsTimeoutError < TimeoutError; end
  53. # Error raised when there was a timeout while resolving a domain to an IP.
  54. 27 class ResolveTimeoutError < TimeoutError; end
  55. # Error raise when there was a timeout waiting for readiness of the socket the request is related to.
  56. 27 class OperationTimeoutError < TimeoutError; end
  57. # Error raised when there was an error while resolving a domain to an IP.
  58. 27 class ResolveError < Error; end
  59. # Error raised when there was an error while resolving a domain to an IP
  60. # using a HTTPX::Resolver::Native resolver.
  61. 27 class NativeResolveError < ResolveError
  62. 27 attr_reader :connection, :host
  63. # initializes the exception with the +connection+ it refers to, the +host+ domain
  64. # which failed to resolve, and the error +message+.
  65. 27 def initialize(connection, host, message = "Can't resolve #{host}")
  66. 136 @connection = connection
  67. 136 @host = host
  68. 136 super(message)
  69. end
  70. end
  71. # The exception class for HTTP responses with 4xx or 5xx status.
  72. 27 class HTTPError < Error
  73. # The HTTPX::Response response object this exception refers to.
  74. 27 attr_reader :response
  75. # Creates the instance and assigns the HTTPX::Response +response+.
  76. 27 def initialize(response)
  77. 84 @response = response
  78. 84 super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
  79. end
  80. # The HTTP response status.
  81. #
  82. # error.status #=> 404
  83. 27 def status
  84. 16 @response.status
  85. end
  86. end
  87. end

lib/httpx/extensions.rb

67.86% lines covered

28 relevant lines. 19 lines covered and 9 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "uri"
  3. 27 module HTTPX
  4. 27 module ArrayExtensions
  5. 27 module FilterMap
  6. refine Array do
  7. # Ruby 2.7 backport
  8. def filter_map
  9. return to_enum(:filter_map) unless block_given?
  10. each_with_object([]) do |item, res|
  11. processed = yield(item)
  12. res << processed if processed
  13. end
  14. end
  15. 26 end unless Array.method_defined?(:filter_map)
  16. end
  17. 27 module Intersect
  18. refine Array do
  19. # Ruby 3.1 backport
  20. 4 def intersect?(arr)
  21. if size < arr.size
  22. smaller = self
  23. else
  24. smaller, arr = arr, self
  25. end
  26. (arr & smaller).size > 0
  27. end
  28. 26 end unless Array.method_defined?(:intersect?)
  29. end
  30. end
  31. 27 module URIExtensions
  32. # uri 0.11 backport, ships with ruby 3.1
  33. 27 refine URI::Generic do
  34. 27 def non_ascii_hostname
  35. 453 @non_ascii_hostname
  36. end
  37. 27 def non_ascii_hostname=(hostname)
  38. 32 @non_ascii_hostname = hostname
  39. end
  40. def authority
  41. 5694 return host if port == default_port
  42. 598 "#{host}:#{port}"
  43. 26 end unless URI::HTTP.method_defined?(:authority)
  44. def origin
  45. 4634 "#{scheme}://#{authority}"
  46. 26 end unless URI::HTTP.method_defined?(:origin)
  47. end
  48. end
  49. end

lib/httpx/headers.rb

100.0% lines covered

71 relevant lines. 71 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 class Headers
  4. 27 class << self
  5. 27 def new(headers = nil)
  6. 25883 return headers if headers.is_a?(self)
  7. 11846 super
  8. end
  9. end
  10. 27 def initialize(headers = nil)
  11. 11846 if headers.nil? || headers.empty?
  12. 1896 @headers = headers.to_h
  13. 1677 return
  14. end
  15. 9950 @headers = {}
  16. 9950 headers.each do |field, value|
  17. 61527 field = downcased(field)
  18. 61527 value = array_value(value)
  19. 61527 current = @headers[field]
  20. 61527 if current.nil?
  21. 54703 @headers[field] = value
  22. else
  23. 53 current.concat(value)
  24. end
  25. end
  26. end
  27. # cloned initialization
  28. 27 def initialize_clone(orig, **kwargs)
  29. 8 super
  30. 8 @headers = orig.instance_variable_get(:@headers).clone(**kwargs)
  31. end
  32. # dupped initialization
  33. 27 def initialize_dup(orig)
  34. 15451 super
  35. 15451 @headers = orig.instance_variable_get(:@headers).dup
  36. end
  37. # freezes the headers hash
  38. 27 def freeze
  39. 20157 @headers.freeze
  40. 20157 super
  41. end
  42. # merges headers with another header-quack.
  43. # the merge rule is, if the header already exists,
  44. # ignore what the +other+ headers has. Otherwise, set
  45. #
  46. 27 def merge(other)
  47. 4867 headers = dup
  48. 4867 other.each do |field, value|
  49. 4395 headers[downcased(field)] = value
  50. end
  51. 4867 headers
  52. end
  53. # returns the comma-separated values of the header field
  54. # identified by +field+, or nil otherwise.
  55. #
  56. 27 def [](field)
  57. 95042 a = @headers[downcased(field)] || return
  58. 28357 a.join(", ")
  59. end
  60. # sets +value+ (if not nil) as single value for the +field+ header.
  61. #
  62. 27 def []=(field, value)
  63. 41839 return unless value
  64. 37486 @headers[downcased(field)] = array_value(value)
  65. end
  66. # deletes all values associated with +field+ header.
  67. #
  68. 27 def delete(field)
  69. 273 canonical = downcased(field)
  70. 273 @headers.delete(canonical) if @headers.key?(canonical)
  71. end
  72. # adds additional +value+ to the existing, for header +field+.
  73. #
  74. 27 def add(field, value)
  75. 592 (@headers[downcased(field)] ||= []) << String(value)
  76. end
  77. # helper to be used when adding an header field as a value to another field
  78. #
  79. # h2_headers.add_header("vary", "accept-encoding")
  80. # h2_headers["vary"] #=> "accept-encoding"
  81. # h1_headers.add_header("vary", "accept-encoding")
  82. # h1_headers["vary"] #=> "Accept-Encoding"
  83. #
  84. 27 alias_method :add_header, :add
  85. # returns the enumerable headers store in pairs of header field + the values in
  86. # the comma-separated string format
  87. #
  88. 27 def each(extra_headers = nil)
  89. 67815 return enum_for(__method__, extra_headers) { @headers.size } unless block_given?
  90. 36104 @headers.each do |field, value|
  91. 49048 yield(field, value.join(", ")) unless value.empty?
  92. end
  93. 5997 extra_headers.each do |field, value|
  94. 24483 yield(field, value) unless value.empty?
  95. 36103 end if extra_headers
  96. end
  97. 27 def ==(other)
  98. 21084 other == to_hash
  99. end
  100. 27 def empty?
  101. 312 @headers.empty?
  102. end
  103. # the headers store in Hash format
  104. 27 def to_hash
  105. 22909 Hash[to_a]
  106. end
  107. 27 alias_method :to_h, :to_hash
  108. # the headers store in array of pairs format
  109. 27 def to_a
  110. 22932 Array(each)
  111. end
  112. # headers as string
  113. 27 def to_s
  114. 1902 @headers.to_s
  115. end
  116. skipped # :nocov:
  117. skipped def inspect
  118. skipped "#<#{self.class}:#{object_id} " \
  119. skipped "#{to_hash.inspect}>"
  120. skipped end
  121. skipped # :nocov:
  122. # this is internal API and doesn't abide to other public API
  123. # guarantees, like downcasing strings.
  124. # Please do not use this outside of core!
  125. #
  126. 27 def key?(downcased_key)
  127. 64553 @headers.key?(downcased_key)
  128. end
  129. # returns the values for the +field+ header in array format.
  130. # This method is more internal, and for this reason doesn't try
  131. # to "correct" the user input, i.e. it doesn't downcase the key.
  132. #
  133. 27 def get(field)
  134. 279 @headers[field] || EMPTY
  135. end
  136. 27 private
  137. 27 def array_value(value)
  138. 103366 Array(value)
  139. end
  140. 27 def downcased(field)
  141. 204095 String(field).downcase
  142. end
  143. end
  144. end

lib/httpx/io.rb

100.0% lines covered

5 relevant lines. 5 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "socket"
  3. 27 require "httpx/io/udp"
  4. 27 require "httpx/io/tcp"
  5. 27 require "httpx/io/unix"
  6. begin
  7. 27 require "httpx/io/ssl"
  8. rescue LoadError
  9. end

lib/httpx/io/ssl.rb

96.55% lines covered

87 relevant lines. 84 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "openssl"
  3. 27 module HTTPX
  4. 27 TLSError = OpenSSL::SSL::SSLError
  5. 27 class SSL < TCP
  6. # rubocop:disable Style/MutableConstant
  7. 27 TLS_OPTIONS = { alpn_protocols: %w[h2 http/1.1].freeze }
  8. # https://github.com/jruby/jruby-openssl/issues/284
  9. # TODO: remove when dropping support for jruby-openssl < 0.15.4
  10. 27 TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby" && JOpenSSL::VERSION < "0.15.4"
  11. # rubocop:enable Style/MutableConstant
  12. 27 TLS_OPTIONS.freeze
  13. 27 attr_writer :ssl_session
  14. 27 def initialize(_, _, options)
  15. 3003 super
  16. 3003 ctx_options = TLS_OPTIONS.merge(options.ssl)
  17. 3003 @sni_hostname = ctx_options.delete(:hostname) || @hostname
  18. 3003 if @keep_open && @io.is_a?(OpenSSL::SSL::SSLSocket)
  19. # externally initiated ssl socket
  20. 22 @ctx = @io.context
  21. 22 @state = :negotiated
  22. else
  23. 2981 @ctx = OpenSSL::SSL::SSLContext.new
  24. 2981 @ctx.set_params(ctx_options) unless ctx_options.empty?
  25. 2981 unless @ctx.session_cache_mode.nil? # a dummy method on JRuby
  26. 2646 @ctx.session_cache_mode =
  27. OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
  28. end
  29. 2981 yield(self) if block_given?
  30. end
  31. 3003 @verify_hostname = @ctx.verify_hostname
  32. end
  33. 27 if OpenSSL::SSL::SSLContext.method_defined?(:session_new_cb=)
  34. 26 def session_new_cb(&pr)
  35. 7627 @ctx.session_new_cb = proc { |_, sess| pr.call(sess) }
  36. end
  37. else
  38. # session_new_cb not implemented under JRuby
  39. 1 def session_new_cb; end
  40. end
  41. 27 def protocol
  42. 2881 @io.alpn_protocol || super
  43. rescue StandardError
  44. 8 super
  45. end
  46. 27 if RUBY_ENGINE == "jruby"
  47. # in jruby, alpn_protocol may return ""
  48. # https://github.com/jruby/jruby-openssl/issues/287
  49. 1 def protocol
  50. 366 proto = @io.alpn_protocol
  51. 365 return super if proto.nil? || proto.empty?
  52. 364 proto
  53. rescue StandardError
  54. 1 super
  55. end
  56. end
  57. 27 def can_verify_peer?
  58. 13 @ctx.verify_mode == OpenSSL::SSL::VERIFY_PEER
  59. end
  60. 27 def verify_hostname(host)
  61. 15 return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
  62. 15 return false if !@io.respond_to?(:peer_cert) || @io.peer_cert.nil?
  63. 15 OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
  64. end
  65. 27 def connected?
  66. 12638 @state == :negotiated
  67. end
  68. 27 def expired?
  69. super || ssl_session_expired?
  70. end
  71. 27 def ssl_session_expired?
  72. 3206 @ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (@ssl_session.time.to_f + @ssl_session.timeout)
  73. end
  74. 27 def connect
  75. 12694 return if @state == :negotiated
  76. 12694 unless @state == :connected
  77. 7088 super
  78. 7056 return unless @state == :connected
  79. end
  80. 8723 unless @io.is_a?(OpenSSL::SSL::SSLSocket)
  81. 3206 if (hostname_is_ip = (@ip == @sni_hostname))
  82. # IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
  83. 32 @sni_hostname = @ip.to_string
  84. # IP addresses in SNI is not valid per RFC 6066, section 3.
  85. 32 @ctx.verify_hostname = false
  86. end
  87. 3206 @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
  88. 3206 @io.hostname = @sni_hostname unless hostname_is_ip
  89. 3206 @io.session = @ssl_session unless ssl_session_expired?
  90. 3206 @io.sync_close = true
  91. end
  92. 8723 try_ssl_connect
  93. end
  94. 27 def try_ssl_connect
  95. 8723 ret = @io.connect_nonblock(exception: false)
  96. 8744 log(level: 3, color: :cyan) { "TLS CONNECT: #{ret}..." }
  97. 7956 case ret
  98. when :wait_readable
  99. 5541 @interests = :r
  100. 5541 return
  101. when :wait_writable
  102. @interests = :w
  103. return
  104. end
  105. 3159 @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && @verify_hostname
  106. 3158 transition(:negotiated)
  107. 3158 @interests = :w
  108. end
  109. 27 private
  110. 27 def transition(nextstate)
  111. 11056 case nextstate
  112. when :negotiated
  113. 3158 return unless @state == :connected
  114. when :closed
  115. 3080 return unless @state == :negotiated ||
  116. @state == :connected
  117. end
  118. 12504 do_transition(nextstate)
  119. end
  120. 27 def log_transition_state(nextstate)
  121. 70 return super unless nextstate == :negotiated
  122. 16 server_cert = @io.peer_cert
  123. 14 "#{super}\n\n" \
  124. 2 "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
  125. 2 "ALPN, server accepted to use #{protocol}\n" \
  126. "Server certificate:\n " \
  127. 2 "subject: #{server_cert.subject}\n " \
  128. 2 "start date: #{server_cert.not_before}\n " \
  129. 2 "expire date: #{server_cert.not_after}\n " \
  130. 2 "issuer: #{server_cert.issuer}\n " \
  131. "SSL certificate verify ok."
  132. end
  133. end
  134. end

lib/httpx/io/tcp.rb

91.15% lines covered

113 relevant lines. 103 lines covered and 10 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "ipaddr"
  4. 27 module HTTPX
  5. 27 class TCP
  6. 27 include Loggable
  7. 27 using URIExtensions
  8. 27 attr_reader :ip, :port, :addresses, :state, :interests
  9. 27 alias_method :host, :ip
  10. 27 def initialize(origin, addresses, options)
  11. 7098 @state = :idle
  12. 7098 @addresses = []
  13. 7098 @hostname = origin.host
  14. 7098 @options = options
  15. 7098 @fallback_protocol = @options.fallback_protocol
  16. 7098 @port = origin.port
  17. 7098 @interests = :w
  18. 7098 if @options.io
  19. 52 @io = case @options.io
  20. when Hash
  21. 16 @options.io[origin.authority]
  22. else
  23. 36 @options.io
  24. end
  25. 52 raise Error, "Given IO objects do not match the request authority" unless @io
  26. 52 _, _, _, @ip = @io.addr
  27. 52 @addresses << @ip
  28. 52 @keep_open = true
  29. 52 @state = :connected
  30. else
  31. 7046 add_addresses(addresses)
  32. end
  33. 7098 @ip_index = @addresses.size - 1
  34. end
  35. 27 def socket
  36. 195 @io
  37. end
  38. 27 def add_addresses(addrs)
  39. 7295 return if addrs.empty?
  40. 26074 addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
  41. 7295 ip_index = @ip_index || (@addresses.size - 1)
  42. 7295 if addrs.first.ipv6?
  43. # should be the next in line
  44. 259 @addresses = [*@addresses[0, ip_index], *addrs, *@addresses[ip_index..-1]]
  45. else
  46. 7036 @addresses.unshift(*addrs)
  47. 7036 @ip_index += addrs.size if @ip_index
  48. end
  49. end
  50. 27 def to_io
  51. 23230 @io.to_io
  52. end
  53. 27 def protocol
  54. 4458 @fallback_protocol
  55. end
  56. 27 def connect
  57. 26877 return unless closed?
  58. 26750 if !@io || @io.closed?
  59. 7762 transition(:idle)
  60. 7762 @io = build_socket
  61. end
  62. 26750 try_connect
  63. rescue Errno::ECONNREFUSED,
  64. Errno::EADDRNOTAVAIL,
  65. Errno::EHOSTUNREACH,
  66. SocketError,
  67. IOError => e
  68. 904 raise e if @ip_index <= 0
  69. 842 log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
  70. 818 @ip_index -= 1
  71. 830 @io = build_socket
  72. 830 retry
  73. rescue Errno::ETIMEDOUT => e
  74. raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index <= 0
  75. log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
  76. @ip_index -= 1
  77. @io = build_socket
  78. retry
  79. end
  80. 27 def try_connect
  81. 26750 ret = @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s), exception: false)
  82. 16388 log(level: 3, color: :cyan) { "TCP CONNECT: #{ret}..." }
  83. 14496 case ret
  84. when :wait_readable
  85. @interests = :r
  86. return
  87. when :wait_writable
  88. 8580 @interests = :w
  89. 8580 return
  90. end
  91. 7688 transition(:connected)
  92. 7688 @interests = :w
  93. rescue Errno::EALREADY
  94. 9578 @interests = :w
  95. end
  96. 27 private :try_connect
  97. 27 def read(size, buffer)
  98. 51136 ret = @io.read_nonblock(size, buffer, exception: false)
  99. 51136 if ret == :wait_readable
  100. 10716 buffer.clear
  101. 9912 return 0
  102. end
  103. 40420 return if ret.nil?
  104. 40484 log { "READ: #{buffer.bytesize} bytes..." }
  105. 40396 buffer.bytesize
  106. end
  107. 27 def write(buffer)
  108. 21368 siz = @io.write_nonblock(buffer, exception: false)
  109. 21339 return 0 if siz == :wait_writable
  110. 21323 return if siz.nil?
  111. 21418 log { "WRITE: #{siz} bytes..." }
  112. 21323 buffer.shift!(siz)
  113. 21323 siz
  114. end
  115. 27 def close
  116. 8468 return if @keep_open || closed?
  117. 861 begin
  118. 7492 @io.close
  119. ensure
  120. 7492 transition(:closed)
  121. end
  122. end
  123. 27 def connected?
  124. 18931 @state == :connected
  125. end
  126. 27 def closed?
  127. 35332 @state == :idle || @state == :closed
  128. end
  129. 27 def expired?
  130. # do not mess with external sockets
  131. return false if @options.io
  132. return true unless @addresses
  133. resolver_addresses = Resolver.nolookup_resolve(@hostname)
  134. (Array(resolver_addresses) & @addresses).empty?
  135. end
  136. skipped # :nocov:
  137. skipped def inspect
  138. skipped "#<#{self.class}:#{object_id} " \
  139. skipped "#{@ip}:#{@port} " \
  140. skipped "@state=#{@state} " \
  141. skipped "@hostname=#{@hostname} " \
  142. skipped "@addresses=#{@addresses} " \
  143. skipped "@state=#{@state}>"
  144. skipped end
  145. skipped # :nocov:
  146. 27 private
  147. 27 def build_socket
  148. 8592 @ip = @addresses[@ip_index]
  149. 8592 Socket.new(@ip.family, :STREAM, 0)
  150. end
  151. 27 def transition(nextstate)
  152. 12068 case nextstate
  153. # when :idle
  154. when :connected
  155. 4585 return unless @state == :idle
  156. when :closed
  157. 4412 return unless @state == :connected
  158. end
  159. 13617 do_transition(nextstate)
  160. end
  161. 27 def do_transition(nextstate)
  162. 26293 log(level: 1) { log_transition_state(nextstate) }
  163. 26121 @state = nextstate
  164. end
  165. 27 def log_transition_state(nextstate)
  166. 172 label = host
  167. 172 label = "#{label}(##{@io.fileno})" if nextstate == :connected
  168. 152 "#{label} #{@state} -> #{nextstate}"
  169. end
  170. end
  171. end

lib/httpx/io/udp.rb

100.0% lines covered

35 relevant lines. 35 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "ipaddr"
  3. 27 module HTTPX
  4. 27 class UDP
  5. 27 include Loggable
  6. 27 def initialize(ip, port, options)
  7. 384 @host = ip
  8. 384 @port = port
  9. 384 @io = UDPSocket.new(IPAddr.new(ip).family)
  10. 384 @options = options
  11. end
  12. 27 def to_io
  13. 1138 @io.to_io
  14. end
  15. 27 def connect; end
  16. 27 def connected?
  17. 384 true
  18. end
  19. 27 def close
  20. 384 @io.close
  21. end
  22. 27 if RUBY_ENGINE == "jruby"
  23. # In JRuby, sendmsg_nonblock is not implemented
  24. 1 def write(buffer)
  25. 66 siz = @io.send(buffer.to_s, 0, @host, @port)
  26. 66 log { "WRITE: #{siz} bytes..." }
  27. 66 buffer.shift!(siz)
  28. 66 siz
  29. end
  30. else
  31. 26 def write(buffer)
  32. 716 siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s), exception: false)
  33. 716 return 0 if siz == :wait_writable
  34. 716 return if siz.nil?
  35. 716 log { "WRITE: #{siz} bytes..." }
  36. 716 buffer.shift!(siz)
  37. 716 siz
  38. end
  39. end
  40. 27 def read(size, buffer)
  41. 1361 ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
  42. 1361 return 0 if ret == :wait_readable
  43. 710 return if ret.nil?
  44. 710 log { "READ: #{buffer.bytesize} bytes..." }
  45. 710 buffer.bytesize
  46. rescue IOError
  47. end
  48. end
  49. end

lib/httpx/io/unix.rb

94.29% lines covered

35 relevant lines. 33 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 class UNIX < TCP
  4. 27 using URIExtensions
  5. 27 attr_reader :path
  6. 27 alias_method :host, :path
  7. 27 def initialize(origin, path, options)
  8. 28 @addresses = []
  9. 28 @hostname = origin.host
  10. 28 @state = :idle
  11. 28 @options = options
  12. 28 @fallback_protocol = @options.fallback_protocol
  13. 28 if @options.io
  14. 14 @io = case @options.io
  15. when Hash
  16. 7 @options.io[origin.authority]
  17. else
  18. 7 @options.io
  19. end
  20. 14 raise Error, "Given IO objects do not match the request authority" unless @io
  21. 14 @path = @io.path
  22. 14 @keep_open = true
  23. 14 @state = :connected
  24. 14 elsif path
  25. 14 @path = path
  26. else
  27. raise Error, "No path given where to store the socket"
  28. end
  29. 28 @io ||= build_socket
  30. end
  31. 27 def connect
  32. 21 return unless closed?
  33. begin
  34. 21 if @io.closed?
  35. 7 transition(:idle)
  36. 7 @io = build_socket
  37. end
  38. 21 @io.connect_nonblock(Socket.sockaddr_un(@path))
  39. rescue Errno::EISCONN
  40. end
  41. 14 transition(:connected)
  42. rescue Errno::EINPROGRESS,
  43. Errno::EALREADY,
  44. ::IO::WaitReadable
  45. end
  46. 27 def expired?
  47. false
  48. end
  49. skipped # :nocov:
  50. skipped def inspect
  51. skipped "#<#{self.class}:#{object_id} @path=#{@path}) @state=#{@state})>"
  52. skipped end
  53. skipped # :nocov:
  54. 27 private
  55. 27 def build_socket
  56. 21 Socket.new(Socket::PF_UNIX, :STREAM, 0)
  57. end
  58. end
  59. end

lib/httpx/loggable.rb

100.0% lines covered

20 relevant lines. 20 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Loggable
  4. 27 COLORS = {
  5. black: 30,
  6. red: 31,
  7. green: 32,
  8. yellow: 33,
  9. blue: 34,
  10. magenta: 35,
  11. cyan: 36,
  12. white: 37,
  13. }.freeze
  14. 27 USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
  15. 27 def log(
  16. level: @options.debug_level,
  17. color: nil,
  18. debug_level: @options.debug_level,
  19. debug: @options.debug,
  20. &msg
  21. )
  22. 11203137 return unless debug_level >= level
  23. 226407 debug_stream = debug || ($stderr if USE_DEBUG_LOG)
  24. 226407 return unless debug_stream
  25. 2460 klass = self.class
  26. 4763 until (class_name = klass.name)
  27. 445 klass = klass.superclass
  28. end
  29. 2460 message = +"(pid:#{Process.pid} tid:#{Thread.current.object_id}, self:#{class_name}##{object_id}) "
  30. 2460 message << msg.call << "\n"
  31. 2460 message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
  32. 2460 debug_stream << message
  33. end
  34. 27 def log_exception(ex, level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug)
  35. 1214 log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
  36. end
  37. 27 def log_redact(text, should_redact = @options.debug_redact)
  38. 800 return text.to_s unless should_redact
  39. 114 "[REDACTED]"
  40. end
  41. end
  42. end

lib/httpx/options.rb

98.79% lines covered

165 relevant lines. 163 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "socket"
  3. 27 module HTTPX
  4. # Contains a set of options which are passed and shared across from session to its requests or
  5. # responses.
  6. 27 class Options
  7. 27 BUFFER_SIZE = 1 << 14
  8. 27 WINDOW_SIZE = 1 << 14 # 16K
  9. 27 MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
  10. 27 KEEP_ALIVE_TIMEOUT = 20
  11. 27 SETTINGS_TIMEOUT = 10
  12. 27 CLOSE_HANDSHAKE_TIMEOUT = 10
  13. 27 CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
  14. 27 REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
  15. # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
  16. 2 ip_address_families = begin
  17. 27 list = Socket.ip_address_list
  18. 110 if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? && !a.ipv6_unique_local? }
  19. [Socket::AF_INET6, Socket::AF_INET]
  20. else
  21. 27 [Socket::AF_INET]
  22. end
  23. rescue NotImplementedError
  24. [Socket::AF_INET]
  25. end.freeze
  26. 27 SET_TEMPORARY_NAME = ->(mod, pl = nil) do
  27. 9566 if mod.respond_to?(:set_temporary_name) # ruby 3.4 only
  28. 3597 name = mod.name || "#{mod.superclass.name}(plugin)"
  29. 3597 name = "#{name}/#{pl}" if pl
  30. 3597 mod.set_temporary_name(name)
  31. end
  32. end
  33. 2 DEFAULT_OPTIONS = {
  34. 25 :max_requests => Float::INFINITY,
  35. :debug => nil,
  36. 27 :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
  37. :debug_redact => ENV.key?("HTTPX_DEBUG_REDACT"),
  38. :ssl => EMPTY_HASH,
  39. :http2_settings => { settings_enable_push: 0 }.freeze,
  40. :fallback_protocol => "http/1.1",
  41. :supported_compression_formats => %w[gzip deflate],
  42. :decompress_response_body => true,
  43. :compress_request_body => true,
  44. :timeout => {
  45. connect_timeout: CONNECT_TIMEOUT,
  46. settings_timeout: SETTINGS_TIMEOUT,
  47. close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
  48. operation_timeout: OPERATION_TIMEOUT,
  49. keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
  50. read_timeout: READ_TIMEOUT,
  51. write_timeout: WRITE_TIMEOUT,
  52. request_timeout: REQUEST_TIMEOUT,
  53. },
  54. :headers_class => Class.new(Headers, &SET_TEMPORARY_NAME),
  55. :headers => {},
  56. :window_size => WINDOW_SIZE,
  57. :buffer_size => BUFFER_SIZE,
  58. :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
  59. :request_class => Class.new(Request, &SET_TEMPORARY_NAME),
  60. :response_class => Class.new(Response, &SET_TEMPORARY_NAME),
  61. :request_body_class => Class.new(Request::Body, &SET_TEMPORARY_NAME),
  62. :response_body_class => Class.new(Response::Body, &SET_TEMPORARY_NAME),
  63. :pool_class => Class.new(Pool, &SET_TEMPORARY_NAME),
  64. :connection_class => Class.new(Connection, &SET_TEMPORARY_NAME),
  65. :options_class => Class.new(self, &SET_TEMPORARY_NAME),
  66. :transport => nil,
  67. :addresses => nil,
  68. :persistent => false,
  69. 27 :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
  70. :resolver_options => { cache: true }.freeze,
  71. :pool_options => EMPTY_HASH,
  72. :ip_families => ip_address_families,
  73. :close_on_fork => false,
  74. }.freeze
  75. 27 class << self
  76. 27 def new(options = {})
  77. # let enhanced options go through
  78. 11901 return options if self == Options && options.class < self
  79. 9159 return options if options.is_a?(self)
  80. 4453 super
  81. end
  82. 27 def method_added(meth)
  83. 21069 super
  84. 21069 return unless meth =~ /^option_(.+)$/
  85. 9864 optname = Regexp.last_match(1).to_sym
  86. 9864 attr_reader(optname)
  87. end
  88. end
  89. # creates a new options instance from a given hash, which optionally define the following:
  90. #
  91. # :debug :: an object which log messages are written to (must respond to <tt><<</tt>)
  92. # :debug_level :: the log level of messages (can be 1, 2, or 3).
  93. # :debug_redact :: whether header/body payload should be redacted (defaults to <tt>false</tt>).
  94. # :ssl :: a hash of options which can be set as params of OpenSSL::SSL::SSLContext (see HTTPX::IO::SSL)
  95. # :http2_settings :: a hash of options to be passed to a HTTP2::Connection (ex: <tt>{ max_concurrent_streams: 2 }</tt>)
  96. # :fallback_protocol :: version of HTTP protocol to use by default in the absence of protocol negotiation
  97. # like ALPN (defaults to <tt>"http/1.1"</tt>)
  98. # :supported_compression_formats :: list of compressions supported by the transcoder layer (defaults to <tt>%w[gzip deflate]</tt>).
  99. # :decompress_response_body :: whether to auto-decompress response body (defaults to <tt>true</tt>).
  100. # :compress_request_body :: whether to auto-decompress response body (defaults to <tt>true</tt>)
  101. # :timeout :: hash of timeout configurations (supports <tt>:connect_timeout</tt>, <tt>:settings_timeout</tt>,
  102. # <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>, <tt>:read_timeout</tt>, <tt>:write_timeout</tt>
  103. # and <tt>:request_timeout</tt>
  104. # :headers :: hash of HTTP headers (ex: <tt>{ "x-custom-foo" => "bar" }</tt>)
  105. # :window_size :: number of bytes to read from a socket
  106. # :buffer_size :: internal read and write buffer size in bytes
  107. # :body_threshold_size :: maximum size in bytes of response payload that is buffered in memory.
  108. # :request_class :: class used to instantiate a request
  109. # :response_class :: class used to instantiate a response
  110. # :headers_class :: class used to instantiate headers
  111. # :request_body_class :: class used to instantiate a request body
  112. # :response_body_class :: class used to instantiate a response body
  113. # :connection_class :: class used to instantiate connections
  114. # :pool_class :: class used to instantiate the session connection pool
  115. # :options_class :: class used to instantiate options
  116. # :transport :: type of transport to use (set to "unix" for UNIX sockets)
  117. # :addresses :: bucket of peer addresses (can be a list of IP addresses, a hash of domain to list of adddresses;
  118. # paths should be used for UNIX sockets instead)
  119. # :io :: open socket, or domain/ip-to-socket hash, which requests should be sent to
  120. # :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
  121. # :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
  122. # using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class)
  123. # :resolver_options :: hash of options passed to the resolver. Accepted keys depend on the resolver type.
  124. # :pool_options :: hash of options passed to the connection pool (See Pool#initialize).
  125. # :ip_families :: which socket families are supported (system-dependent)
  126. # :origin :: HTTP origin to set on requests with relative path (ex: "https://api.serv.com")
  127. # :base_path :: path to prefix given relative paths with (ex: "/v2")
  128. # :max_concurrent_requests :: max number of requests which can be set concurrently
  129. # :max_requests :: max number of requests which can be made on socket before it reconnects.
  130. # :close_on_fork :: whether the session automatically closes when the process is fork (defaults to <tt>false</tt>).
  131. # it only works if the session is persistent (and ruby 3.1 or higher is used).
  132. #
  133. # This list of options are enhanced with each loaded plugin, see the plugin docs for details.
  134. 27 def initialize(options = {})
  135. 4453 do_initialize(options)
  136. 4437 freeze
  137. end
  138. 27 def freeze
  139. 11891 @origin.freeze
  140. 11891 @base_path.freeze
  141. 11891 @timeout.freeze
  142. 11891 @headers.freeze
  143. 11891 @addresses.freeze
  144. 11891 @supported_compression_formats.freeze
  145. 11891 @ssl.freeze
  146. 11891 @http2_settings.freeze
  147. 11891 @pool_options.freeze
  148. 11891 @resolver_options.freeze
  149. 11891 @ip_families.freeze
  150. 11891 super
  151. end
  152. 27 def option_origin(value)
  153. 608 URI(value)
  154. end
  155. 27 def option_base_path(value)
  156. 32 String(value)
  157. end
  158. 27 def option_headers(value)
  159. 7972 headers_class.new(value)
  160. end
  161. 27 def option_timeout(value)
  162. 8582 Hash[value]
  163. end
  164. 27 def option_supported_compression_formats(value)
  165. 7364 Array(value).map(&:to_s)
  166. end
  167. 27 def option_transport(value)
  168. 49 transport = value.to_s
  169. 49 raise TypeError, "#{transport} is an unsupported transport type" unless %w[unix].include?(transport)
  170. 49 transport
  171. end
  172. 27 def option_addresses(value)
  173. 43 Array(value)
  174. end
  175. 27 def option_ip_families(value)
  176. 7340 Array(value)
  177. end
  178. # number options
  179. 27 %i[
  180. max_concurrent_requests max_requests window_size buffer_size
  181. body_threshold_size debug_level
  182. ].each do |option|
  183. 162 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  184. 6 # converts +v+ into an Integer before setting the +#{option}+ option.
  185. 6 def option_#{option}(value) # def option_max_requests(v)
  186. value = Integer(value) unless value.infinite?
  187. 6 raise TypeError, ":#{option} must be positive" unless value.positive? # raise TypeError, ":max_requests must be positive" unless value.positive?
  188. value
  189. end
  190. OUT
  191. end
  192. # hashable options
  193. 27 %i[ssl http2_settings resolver_options pool_options].each do |option|
  194. 108 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  195. 4 # converts +v+ into an Hash before setting the +#{option}+ option.
  196. 4 def option_#{option}(value) # def option_ssl(v)
  197. Hash[value]
  198. end
  199. OUT
  200. end
  201. 27 %i[
  202. request_class response_class headers_class request_body_class
  203. response_body_class connection_class options_class
  204. pool_class
  205. io fallback_protocol debug debug_redact resolver_class
  206. compress_request_body decompress_response_body
  207. persistent close_on_fork
  208. ].each do |method_name|
  209. 459 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  210. 17 # sets +v+ as the value of the +#{method_name}+ option
  211. 17 def option_#{method_name}(v); v; end # def option_smth(v); v; end
  212. OUT
  213. end
  214. 27 REQUEST_BODY_IVARS = %i[@headers].freeze
  215. 27 def ==(other)
  216. 1967 super || options_equals?(other)
  217. end
  218. 27 def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
  219. # headers and other request options do not play a role, as they are
  220. # relevant only for the request.
  221. 461 ivars = instance_variables - ignore_ivars
  222. 461 other_ivars = other.instance_variables - ignore_ivars
  223. 461 return false if ivars.size != other_ivars.size
  224. 444 return false if ivars.sort != other_ivars.sort
  225. 444 ivars.all? do |ivar|
  226. 11586 instance_variable_get(ivar) == other.instance_variable_get(ivar)
  227. end
  228. end
  229. 27 def merge(other)
  230. 35995 ivar_map = nil
  231. 35995 other_ivars = case other
  232. when Hash
  233. 43023 ivar_map = other.keys.to_h { |k| [:"@#{k}", k] }
  234. 25020 ivar_map.keys
  235. else
  236. 10975 other.instance_variables
  237. end
  238. 35995 return self if other_ivars.empty?
  239. 314301 return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == access_option(other, ivar, ivar_map) }
  240. 13229 opts = dup
  241. 13229 other_ivars.each do |ivar|
  242. 99278 v = access_option(other, ivar, ivar_map)
  243. 99278 unless v
  244. 9242 opts.instance_variable_set(ivar, v)
  245. 9242 next
  246. end
  247. 90036 v = opts.__send__(:"option_#{ivar[1..-1]}", v)
  248. 90020 orig_v = instance_variable_get(ivar)
  249. 90020 v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
  250. 90020 opts.instance_variable_set(ivar, v)
  251. end
  252. 13213 opts
  253. end
  254. 27 def to_hash
  255. 3463 instance_variables.each_with_object({}) do |ivar, hs|
  256. 87115 hs[ivar[1..-1].to_sym] = instance_variable_get(ivar)
  257. end
  258. end
  259. 27 def extend_with_plugin_classes(pl)
  260. 7411 if defined?(pl::RequestMethods) || defined?(pl::RequestClassMethods)
  261. 2145 @request_class = @request_class.dup
  262. 2145 SET_TEMPORARY_NAME[@request_class, pl]
  263. 2145 @request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
  264. 2145 @request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
  265. end
  266. 7411 if defined?(pl::ResponseMethods) || defined?(pl::ResponseClassMethods)
  267. 2375 @response_class = @response_class.dup
  268. 2375 SET_TEMPORARY_NAME[@response_class, pl]
  269. 2375 @response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
  270. 2375 @response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
  271. end
  272. 7411 if defined?(pl::HeadersMethods) || defined?(pl::HeadersClassMethods)
  273. 152 @headers_class = @headers_class.dup
  274. 152 SET_TEMPORARY_NAME[@headers_class, pl]
  275. 152 @headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
  276. 152 @headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
  277. end
  278. 7411 if defined?(pl::RequestBodyMethods) || defined?(pl::RequestBodyClassMethods)
  279. 330 @request_body_class = @request_body_class.dup
  280. 330 SET_TEMPORARY_NAME[@request_body_class, pl]
  281. 330 @request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
  282. 330 @request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
  283. end
  284. 7411 if defined?(pl::ResponseBodyMethods) || defined?(pl::ResponseBodyClassMethods)
  285. 776 @response_body_class = @response_body_class.dup
  286. 776 SET_TEMPORARY_NAME[@response_body_class, pl]
  287. 776 @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
  288. 776 @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
  289. end
  290. 7411 if defined?(pl::PoolMethods)
  291. 606 @pool_class = @pool_class.dup
  292. 606 SET_TEMPORARY_NAME[@pool_class, pl]
  293. 606 @pool_class.__send__(:include, pl::PoolMethods)
  294. end
  295. 7411 if defined?(pl::ConnectionMethods)
  296. 2966 @connection_class = @connection_class.dup
  297. 2966 SET_TEMPORARY_NAME[@connection_class, pl]
  298. 2966 @connection_class.__send__(:include, pl::ConnectionMethods)
  299. end
  300. 7411 return unless defined?(pl::OptionsMethods)
  301. 2997 @options_class = @options_class.dup
  302. 2997 @options_class.__send__(:include, pl::OptionsMethods)
  303. end
  304. 27 private
  305. 27 def do_initialize(options = {})
  306. 4453 defaults = DEFAULT_OPTIONS.merge(options)
  307. 4453 defaults.each do |k, v|
  308. 138933 next if v.nil?
  309. 125574 option_method_name = :"option_#{k}"
  310. 125574 raise Error, "unknown option: #{k}" unless respond_to?(option_method_name)
  311. 125566 value = __send__(option_method_name, v)
  312. 125558 instance_variable_set(:"@#{k}", value)
  313. end
  314. end
  315. 27 def access_option(obj, k, ivar_map)
  316. 351407 case obj
  317. when Hash
  318. 29117 obj[ivar_map[k]]
  319. else
  320. 362423 obj.instance_variable_get(k)
  321. end
  322. end
  323. end
  324. end

lib/httpx/parser/http1.rb

100.0% lines covered

109 relevant lines. 109 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Parser
  4. 27 class Error < Error; end
  5. 27 class HTTP1
  6. 27 VERSIONS = %w[1.0 1.1].freeze
  7. 27 attr_reader :status_code, :http_version, :headers
  8. 27 def initialize(observer)
  9. 4607 @observer = observer
  10. 4607 @state = :idle
  11. 4607 @buffer = "".b
  12. 4607 @headers = {}
  13. end
  14. 27 def <<(chunk)
  15. 7366 @buffer << chunk
  16. 7366 parse
  17. end
  18. 27 def reset!
  19. 9419 @state = :idle
  20. 9419 @headers = {}
  21. 9419 @content_length = nil
  22. 9419 @_has_trailers = nil
  23. end
  24. 27 def upgrade?
  25. 4621 @upgrade
  26. end
  27. 27 def upgrade_data
  28. 32 @buffer
  29. end
  30. 27 private
  31. 27 def parse
  32. 7366 loop do
  33. 15642 state = @state
  34. 14002 case @state
  35. when :idle
  36. 4974 parse_headline
  37. when :headers, :trailers
  38. 5063 parse_headers
  39. when :data
  40. 5603 parse_data
  41. end
  42. 11445 return if @buffer.empty? || state == @state
  43. end
  44. end
  45. 27 def parse_headline
  46. 4974 idx = @buffer.index("\n")
  47. 4974 return unless idx
  48. 4974 (m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
  49. raise(Error, "wrong head line format")
  50. 4966 version, code, _ = m.captures
  51. 4966 raise(Error, "unsupported HTTP version (HTTP/#{version})") unless version && VERSIONS.include?(version)
  52. 4958 @http_version = version.split(".").map(&:to_i)
  53. 4958 @status_code = code.to_i
  54. 4958 raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
  55. 4950 @buffer = @buffer.byteslice((idx + 1)..-1)
  56. 4950 nextstate(:headers)
  57. end
  58. 27 def parse_headers
  59. 5065 headers = @headers
  60. 5065 buffer = @buffer
  61. 39229 while (idx = buffer.index("\n"))
  62. # @type var line: String
  63. 38645 line = buffer.byteslice(0..idx)
  64. 38645 raise Error, "wrong header format" if line.start_with?("\s", "\t")
  65. 38637 line.lstrip!
  66. 38637 buffer = @buffer = buffer.byteslice((idx + 1)..-1)
  67. 38637 if line.empty?
  68. 4438 case @state
  69. when :headers
  70. 4934 prepare_data(headers)
  71. 4934 @observer.on_headers(headers)
  72. 4307 return unless @state == :headers
  73. # state might have been reset
  74. # in the :headers callback
  75. 4234 nextstate(:data)
  76. 4234 headers.clear
  77. when :trailers
  78. 16 @observer.on_trailers(headers)
  79. 16 headers.clear
  80. 16 nextstate(:complete)
  81. end
  82. 4242 return
  83. end
  84. 33687 separator_index = line.index(":")
  85. 33687 raise Error, "wrong header format" unless separator_index
  86. # @type var key: String
  87. 33679 key = line.byteslice(0..(separator_index - 1))
  88. 33679 key.rstrip! # was lstripped previously!
  89. # @type var value: String
  90. 33679 value = line.byteslice((separator_index + 1)..-1)
  91. 33679 value.strip!
  92. 33679 raise Error, "wrong header format" if value.nil?
  93. 33679 (headers[key.downcase] ||= []) << value
  94. end
  95. end
  96. 27 def parse_data
  97. 5603 if @buffer.respond_to?(:each)
  98. 200 @buffer.each do |chunk|
  99. 237 @observer.on_data(chunk)
  100. end
  101. 5402 elsif @content_length
  102. # @type var data: String
  103. 5363 data = @buffer.byteslice(0, @content_length)
  104. 5363 @buffer = @buffer.byteslice(@content_length..-1) || "".b
  105. 4794 @content_length -= data.bytesize
  106. 5363 @observer.on_data(data)
  107. 5341 data.clear
  108. else
  109. 40 @observer.on_data(@buffer)
  110. 40 @buffer.clear
  111. end
  112. 5573 return unless no_more_data?
  113. 4084 @buffer = @buffer.to_s
  114. 4084 if @_has_trailers
  115. 16 nextstate(:trailers)
  116. else
  117. 4068 nextstate(:complete)
  118. end
  119. end
  120. 27 def prepare_data(headers)
  121. 4934 @upgrade = headers.key?("upgrade")
  122. 4934 @_has_trailers = headers.key?("trailer")
  123. 4934 if (tr_encodings = headers["transfer-encoding"])
  124. 115 tr_encodings.reverse_each do |tr_encoding|
  125. 115 tr_encoding.split(/ *, */).each do |encoding|
  126. 101 case encoding
  127. when "chunked"
  128. 115 @buffer = Transcoder::Chunker::Decoder.new(@buffer, @_has_trailers)
  129. end
  130. end
  131. end
  132. else
  133. 4819 @content_length = headers["content-length"][0].to_i if headers.key?("content-length")
  134. end
  135. end
  136. 27 def no_more_data?
  137. 5573 if @content_length
  138. 5341 @content_length <= 0
  139. 231 elsif @buffer.respond_to?(:finished?)
  140. 192 @buffer.finished?
  141. else
  142. 40 false
  143. end
  144. end
  145. 27 def nextstate(state)
  146. 13284 @state = state
  147. 11920 case state
  148. when :headers
  149. 4950 @observer.on_start
  150. when :complete
  151. 4084 @observer.on_complete
  152. 584 reset!
  153. 584 nextstate(:idle) unless @buffer.empty?
  154. end
  155. end
  156. end
  157. end
  158. end

lib/httpx/plugins/auth.rb

100.0% lines covered

9 relevant lines. 9 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds a shim +authorization+ method to the session, which will fill
  6. # the HTTP Authorization header, and another, +bearer_auth+, which fill the "Bearer " prefix
  7. # in its value.
  8. #
  9. # https://gitlab.com/os85/httpx/wikis/Auth#auth
  10. #
  11. 8 module Auth
  12. 8 module InstanceMethods
  13. 8 def authorization(token)
  14. 144 with(headers: { "authorization" => token })
  15. end
  16. 8 def bearer_auth(token)
  17. 16 authorization("Bearer #{token}")
  18. end
  19. end
  20. end
  21. 8 register_plugin :auth, Auth
  22. end
  23. end

lib/httpx/plugins/auth/basic.rb

100.0% lines covered

10 relevant lines. 10 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 9 require "httpx/base64"
  3. 9 module HTTPX
  4. 9 module Plugins
  5. 9 module Authentication
  6. 9 class Basic
  7. 9 def initialize(user, password, **)
  8. 274 @user = user
  9. 274 @password = password
  10. end
  11. 9 def authenticate(*)
  12. 255 "Basic #{Base64.strict_encode64("#{@user}:#{@password}")}"
  13. end
  14. end
  15. end
  16. end
  17. end

lib/httpx/plugins/auth/digest.rb

100.0% lines covered

61 relevant lines. 61 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "time"
  3. 8 require "securerandom"
  4. 8 require "digest"
  5. 8 module HTTPX
  6. 8 module Plugins
  7. 8 module Authentication
  8. 8 class Digest
  9. 8 def initialize(user, password, hashed: false, **)
  10. 176 @user = user
  11. 176 @password = password
  12. 176 @nonce = 0
  13. 176 @hashed = hashed
  14. end
  15. 8 def can_authenticate?(authenticate)
  16. 160 authenticate && /Digest .*/.match?(authenticate)
  17. end
  18. 8 def authenticate(request, authenticate)
  19. 160 "Digest #{generate_header(request.verb, request.path, authenticate)}"
  20. end
  21. 8 private
  22. 8 def generate_header(meth, uri, authenticate)
  23. # discard first token, it's Digest
  24. 160 auth_info = authenticate[/^(\w+) (.*)/, 2]
  25. 160 params = auth_info.split(/ *, */)
  26. 832 .to_h { |val| val.split("=", 2) }
  27. 832 .transform_values { |v| v.delete("\"") }
  28. 160 nonce = params["nonce"]
  29. 160 nc = next_nonce
  30. # verify qop
  31. 160 qop = params["qop"]
  32. 160 if params["algorithm"] =~ /(.*?)(-sess)?$/
  33. 144 alg = Regexp.last_match(1)
  34. 144 algorithm = ::Digest.const_get(alg)
  35. 144 raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
  36. 144 sess = Regexp.last_match(2)
  37. else
  38. 16 algorithm = ::Digest::MD5
  39. end
  40. 160 if qop || sess
  41. 160 cnonce = make_cnonce
  42. 160 nc = format("%<nonce>08x", nonce: nc)
  43. end
  44. 160 a1 = if sess
  45. 4 [
  46. 32 (@hashed ? @password : algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}")),
  47. nonce,
  48. cnonce,
  49. 3 ].join ":"
  50. else
  51. 128 @hashed ? @password : "#{@user}:#{params["realm"]}:#{@password}"
  52. end
  53. 160 ha1 = algorithm.hexdigest(a1)
  54. 160 ha2 = algorithm.hexdigest("#{meth}:#{uri}")
  55. 160 request_digest = [ha1, nonce]
  56. 160 request_digest.push(nc, cnonce, qop) if qop
  57. 160 request_digest << ha2
  58. 160 request_digest = request_digest.join(":")
  59. 40 header = [
  60. 140 %(username="#{@user}"),
  61. 20 %(nonce="#{nonce}"),
  62. 20 %(uri="#{uri}"),
  63. 20 %(response="#{algorithm.hexdigest(request_digest)}"),
  64. ]
  65. 160 header << %(realm="#{params["realm"]}") if params.key?("realm")
  66. 160 header << %(algorithm=#{params["algorithm"]}) if params.key?("algorithm")
  67. 160 header << %(cnonce="#{cnonce}") if cnonce
  68. 160 header << %(nc=#{nc})
  69. 160 header << %(qop=#{qop}) if qop
  70. 160 header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
  71. 160 header.join ", "
  72. end
  73. 8 def make_cnonce
  74. 180 ::Digest::MD5.hexdigest [
  75. Time.now.to_i,
  76. Process.pid,
  77. SecureRandom.random_number(2**32),
  78. ].join ":"
  79. end
  80. 8 def next_nonce
  81. 140 @nonce += 1
  82. end
  83. end
  84. end
  85. end
  86. end

lib/httpx/plugins/auth/ntlm.rb

100.0% lines covered

19 relevant lines. 19 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 3 require "httpx/base64"
  3. 3 require "ntlm"
  4. 3 module HTTPX
  5. 3 module Plugins
  6. 3 module Authentication
  7. 3 class Ntlm
  8. 3 def initialize(user, password, domain: nil)
  9. 4 @user = user
  10. 4 @password = password
  11. 4 @domain = domain
  12. end
  13. 3 def can_authenticate?(authenticate)
  14. 2 authenticate && /NTLM .*/.match?(authenticate)
  15. end
  16. 3 def negotiate
  17. 4 "NTLM #{NTLM.negotiate(domain: @domain).to_base64}"
  18. end
  19. 3 def authenticate(_req, www)
  20. 2 challenge = www[/NTLM (.*)/, 1]
  21. 2 challenge = Base64.decode64(challenge)
  22. 2 ntlm_challenge = NTLM.authenticate(challenge, @user, @domain, @password).to_base64
  23. 2 "NTLM #{ntlm_challenge}"
  24. end
  25. end
  26. end
  27. end
  28. end

lib/httpx/plugins/auth/socks5.rb

100.0% lines covered

11 relevant lines. 11 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 module Plugins
  4. 10 module Authentication
  5. 10 class Socks5
  6. 10 def initialize(user, password, **)
  7. 48 @user = user
  8. 48 @password = password
  9. end
  10. 10 def can_authenticate?(*)
  11. 48 @user && @password
  12. end
  13. 10 def authenticate(*)
  14. 48 [0x01, @user.bytesize, @user, @password.bytesize, @password].pack("CCA*CA*")
  15. end
  16. end
  17. end
  18. end
  19. end

lib/httpx/plugins/aws_sdk_authentication.rb

100.0% lines covered

43 relevant lines. 43 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin applies AWS Sigv4 to requests, using the AWS SDK credentials and configuration.
  6. #
  7. # It requires the "aws-sdk-core" gem.
  8. #
  9. 8 module AwsSdkAuthentication
  10. # Mock configuration, to be used only when resolving credentials
  11. 8 class Configuration
  12. 8 attr_reader :profile
  13. 8 def initialize(profile)
  14. 32 @profile = profile
  15. end
  16. 8 def respond_to_missing?(*)
  17. 16 true
  18. end
  19. 8 def method_missing(*); end
  20. end
  21. #
  22. # encapsulates access to an AWS SDK credentials store.
  23. #
  24. 8 class Credentials
  25. 8 def initialize(aws_credentials)
  26. 16 @aws_credentials = aws_credentials
  27. end
  28. 8 def username
  29. 16 @aws_credentials.access_key_id
  30. end
  31. 8 def password
  32. 16 @aws_credentials.secret_access_key
  33. end
  34. 8 def security_token
  35. 16 @aws_credentials.session_token
  36. end
  37. end
  38. 8 class << self
  39. 8 def load_dependencies(_klass)
  40. 16 require "aws-sdk-core"
  41. end
  42. 8 def configure(klass)
  43. 16 klass.plugin(:aws_sigv4)
  44. end
  45. 8 def extra_options(options)
  46. 16 options.merge(max_concurrent_requests: 1)
  47. end
  48. 8 def credentials(profile)
  49. 16 mock_configuration = Configuration.new(profile)
  50. 16 Credentials.new(Aws::CredentialProviderChain.new(mock_configuration).resolve)
  51. end
  52. 8 def region(profile)
  53. # https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb#L62
  54. 16 keys = %w[AWS_REGION AMAZON_REGION AWS_DEFAULT_REGION]
  55. 16 env_region = ENV.values_at(*keys).compact.first
  56. 16 env_region = nil if env_region == ""
  57. 16 cfg_region = Aws.shared_config.region(profile: profile)
  58. 16 env_region || cfg_region
  59. end
  60. end
  61. # adds support for the following options:
  62. #
  63. # :aws_profile :: AWS account profile to retrieve credentials from.
  64. 8 module OptionsMethods
  65. 8 def option_aws_profile(value)
  66. 80 String(value)
  67. end
  68. end
  69. 8 module InstanceMethods
  70. #
  71. # aws_authentication
  72. # aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
  73. # aws_authentication()
  74. #
  75. 8 def aws_sdk_authentication(
  76. credentials: AwsSdkAuthentication.credentials(@options.aws_profile),
  77. region: AwsSdkAuthentication.region(@options.aws_profile),
  78. **options
  79. )
  80. 16 aws_sigv4_authentication(
  81. credentials: credentials,
  82. region: region,
  83. provider_prefix: "aws",
  84. header_provider_field: "amz",
  85. **options
  86. )
  87. end
  88. 8 alias_method :aws_auth, :aws_sdk_authentication
  89. end
  90. end
  91. 8 register_plugin :aws_sdk_authentication, AwsSdkAuthentication
  92. end
  93. end

lib/httpx/plugins/aws_sigv4.rb

100.0% lines covered

121 relevant lines. 121 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds AWS Sigv4 authentication.
  6. #
  7. # https://docs.aws.amazon.com/IAM/latest/UserGuide/signing-elements.html
  8. #
  9. # https://gitlab.com/os85/httpx/wikis/AWS-SigV4
  10. #
  11. 8 module AWSSigV4
  12. 8 Credentials = Struct.new(:username, :password, :security_token)
  13. # Signs requests using the AWS sigv4 signing.
  14. 8 class Signer
  15. 8 def initialize(
  16. service:,
  17. region:,
  18. credentials: nil,
  19. username: nil,
  20. password: nil,
  21. security_token: nil,
  22. provider_prefix: "aws",
  23. header_provider_field: "amz",
  24. unsigned_headers: [],
  25. apply_checksum_header: true,
  26. algorithm: "SHA256"
  27. )
  28. 152 @credentials = credentials || Credentials.new(username, password, security_token)
  29. 152 @service = service
  30. 152 @region = region
  31. 152 @unsigned_headers = Set.new(unsigned_headers.map(&:downcase))
  32. 152 @unsigned_headers << "authorization"
  33. 152 @unsigned_headers << "x-amzn-trace-id"
  34. 152 @unsigned_headers << "expect"
  35. 152 @apply_checksum_header = apply_checksum_header
  36. 152 @provider_prefix = provider_prefix
  37. 152 @header_provider_field = header_provider_field
  38. 152 @algorithm = algorithm
  39. end
  40. 8 def sign!(request)
  41. 152 lower_provider_prefix = "#{@provider_prefix}4"
  42. 152 upper_provider_prefix = lower_provider_prefix.upcase
  43. 152 downcased_algorithm = @algorithm.downcase
  44. 152 datetime = (request.headers["x-#{@header_provider_field}-date"] ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ"))
  45. 152 date = datetime[0, 8]
  46. 152 content_hashed = request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] || hexdigest(request.body)
  47. 144 request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] ||= content_hashed if @apply_checksum_header
  48. 144 request.headers["x-#{@header_provider_field}-security-token"] ||= @credentials.security_token if @credentials.security_token
  49. 144 signature_headers = request.headers.each.reject do |k, _|
  50. 984 @unsigned_headers.include?(k)
  51. end
  52. # aws sigv4 needs to declare the host, regardless of protocol version
  53. 144 signature_headers << ["host", request.authority] unless request.headers.key?("host")
  54. 144 signature_headers.sort_by!(&:first)
  55. 144 signed_headers = signature_headers.map(&:first).join(";")
  56. 144 canonical_headers = signature_headers.map do |k, v|
  57. # eliminate whitespace between value fields, unless it's a quoted value
  58. 847 "#{k}:#{v.start_with?("\"") && v.end_with?("\"") ? v : v.gsub(/\s+/, " ").strip}\n"
  59. end.join
  60. # canonical request
  61. 144 creq = "#{request.verb}" \
  62. 18 "\n#{request.canonical_path}" \
  63. 18 "\n#{request.canonical_query}" \
  64. 18 "\n#{canonical_headers}" \
  65. 18 "\n#{signed_headers}" \
  66. 18 "\n#{content_hashed}"
  67. 144 credential_scope = "#{date}" \
  68. 18 "/#{@region}" \
  69. 18 "/#{@service}" \
  70. 18 "/#{lower_provider_prefix}_request"
  71. 144 algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
  72. # string to sign
  73. 144 sts = "#{algo_line}" \
  74. 18 "\n#{datetime}" \
  75. 18 "\n#{credential_scope}" \
  76. 18 "\n#{OpenSSL::Digest.new(@algorithm).hexdigest(creq)}"
  77. # signature
  78. 144 k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
  79. 144 k_region = hmac(k_date, @region)
  80. 144 k_service = hmac(k_region, @service)
  81. 144 k_credentials = hmac(k_service, "#{lower_provider_prefix}_request")
  82. 144 sig = hexhmac(k_credentials, sts)
  83. 144 credential = "#{@credentials.username}/#{credential_scope}"
  84. # apply signature
  85. 126 request.headers["authorization"] =
  86. 18 "#{algo_line} " \
  87. 18 "Credential=#{credential}, " \
  88. 18 "SignedHeaders=#{signed_headers}, " \
  89. 18 "Signature=#{sig}"
  90. end
  91. 8 private
  92. 8 def hexdigest(value)
  93. 144 digest = OpenSSL::Digest.new(@algorithm)
  94. 144 if value.respond_to?(:read)
  95. 32 if value.respond_to?(:to_path)
  96. # files, pathnames
  97. 8 digest.file(value.to_path).hexdigest
  98. else
  99. # gzipped request bodies
  100. 24 raise Error, "request body must be rewindable" unless value.respond_to?(:rewind)
  101. 24 buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  102. 2 begin
  103. 24 IO.copy_stream(value, buffer)
  104. 24 buffer.flush
  105. 24 digest.file(buffer.to_path).hexdigest
  106. ensure
  107. 24 value.rewind
  108. 24 buffer.close
  109. 24 buffer.unlink
  110. end
  111. end
  112. else
  113. # error on endless generators
  114. 112 raise Error, "hexdigest for endless enumerators is not supported" if value.unbounded_body?
  115. 104 mb_buffer = value.each.with_object("".b) do |chunk, b|
  116. 56 b << chunk
  117. 56 break if b.bytesize >= 1024 * 1024
  118. end
  119. 104 digest.hexdigest(mb_buffer)
  120. end
  121. end
  122. 8 def hmac(key, value)
  123. 576 OpenSSL::HMAC.digest(OpenSSL::Digest.new(@algorithm), key, value)
  124. end
  125. 8 def hexhmac(key, value)
  126. 144 OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new(@algorithm), key, value)
  127. end
  128. end
  129. 8 class << self
  130. 8 def load_dependencies(*)
  131. 152 require "set"
  132. 152 require "digest/sha2"
  133. 152 require "cgi/escape"
  134. end
  135. 8 def configure(klass)
  136. 152 klass.plugin(:expect)
  137. end
  138. end
  139. # adds support for the following options:
  140. #
  141. # :sigv4_signer :: instance of HTTPX::Plugins::AWSSigV4 used to sign requests.
  142. 8 module OptionsMethods
  143. 8 def option_sigv4_signer(value)
  144. 320 value.is_a?(Signer) ? value : Signer.new(value)
  145. end
  146. end
  147. 8 module InstanceMethods
  148. 8 def aws_sigv4_authentication(**options)
  149. 152 with(sigv4_signer: Signer.new(**options))
  150. end
  151. 8 def build_request(*)
  152. 152 request = super
  153. 152 return request if request.headers.key?("authorization")
  154. 152 signer = request.options.sigv4_signer
  155. 152 return request unless signer
  156. 152 signer.sign!(request)
  157. 144 request
  158. end
  159. end
  160. 8 module RequestMethods
  161. 8 def canonical_path
  162. 144 path = uri.path.dup
  163. 144 path << "/" if path.empty?
  164. 176 path.gsub(%r{[^/]+}) { |part| CGI.escape(part.encode("UTF-8")).gsub("+", "%20").gsub("%7E", "~") }
  165. end
  166. 8 def canonical_query
  167. 176 params = query.split("&")
  168. # params = params.map { |p| p.match(/=/) ? p : p + '=' }
  169. # From: https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html#create-canonical-request
  170. # Sort the parameter names by character code point in ascending order.
  171. # Parameters with duplicate names should be sorted by value.
  172. #
  173. # Default sort <=> in JRuby will swap members
  174. # occasionally when <=> is 0 (considered still sorted), but this
  175. # causes our normalized query string to not match the sent querystring.
  176. # When names match, we then sort by their values. When values also
  177. # match then we sort by their original order
  178. 176 params.each.with_index.sort do |a, b|
  179. 64 a, a_offset = a
  180. 64 b, b_offset = b
  181. 64 a_name, a_value = a.split("=", 2)
  182. 64 b_name, b_value = b.split("=", 2)
  183. 64 if a_name == b_name
  184. 32 if a_value == b_value
  185. 16 a_offset <=> b_offset
  186. else
  187. 16 a_value <=> b_value
  188. end
  189. else
  190. 32 a_name <=> b_name
  191. end
  192. end.map(&:first).join("&")
  193. end
  194. end
  195. end
  196. 8 register_plugin :aws_sigv4, AWSSigV4
  197. end
  198. end

lib/httpx/plugins/basic_auth.rb

100.0% lines covered

12 relevant lines. 12 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds helper methods to implement HTTP Basic Auth (https://datatracker.ietf.org/doc/html/rfc7617)
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Auth#basic-auth
  8. #
  9. 8 module BasicAuth
  10. 8 class << self
  11. 8 def load_dependencies(_klass)
  12. 112 require_relative "auth/basic"
  13. end
  14. 8 def configure(klass)
  15. 112 klass.plugin(:auth)
  16. end
  17. end
  18. 8 module InstanceMethods
  19. 8 def basic_auth(user, password)
  20. 128 authorization(Authentication::Basic.new(user, password).authenticate)
  21. end
  22. end
  23. end
  24. 8 register_plugin :basic_auth, BasicAuth
  25. end
  26. end

lib/httpx/plugins/brotli.rb

100.0% lines covered

25 relevant lines. 25 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. 6 module Brotli
  5. 6 class Deflater < Transcoder::Deflater
  6. 6 def deflate(chunk)
  7. 24 return unless chunk
  8. 12 ::Brotli.deflate(chunk)
  9. end
  10. end
  11. 6 module RequestBodyClassMethods
  12. 6 def initialize_deflater_body(body, encoding)
  13. 24 return Brotli.encode(body) if encoding == "br"
  14. 12 super
  15. end
  16. end
  17. 6 module ResponseBodyClassMethods
  18. 6 def initialize_inflater_by_encoding(encoding, response, **kwargs)
  19. 24 return Brotli.decode(response, **kwargs) if encoding == "br"
  20. 12 super
  21. end
  22. end
  23. 6 module_function
  24. 6 def load_dependencies(*)
  25. 24 require "brotli"
  26. end
  27. 6 def self.extra_options(options)
  28. 24 options.merge(supported_compression_formats: %w[br] + options.supported_compression_formats)
  29. end
  30. 6 def encode(body)
  31. 12 Deflater.new(body)
  32. end
  33. 6 def decode(_response, **)
  34. 12 ::Brotli.method(:inflate)
  35. end
  36. end
  37. 6 register_plugin :brotli, Brotli
  38. end
  39. end

lib/httpx/plugins/callbacks.rb

100.0% lines covered

55 relevant lines. 55 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Plugins
  4. #
  5. # This plugin adds suppoort for callbacks around the request/response lifecycle.
  6. #
  7. # https://gitlab.com/os85/httpx/-/wikis/Events
  8. #
  9. 27 module Callbacks
  10. 27 CALLBACKS = %i[
  11. connection_opened connection_closed
  12. request_error
  13. request_started request_body_chunk request_completed
  14. response_started response_body_chunk response_completed
  15. ].freeze
  16. # connection closed user-space errors happen after errors can be surfaced to requests,
  17. # so they need to pierce through the scheduler, which is only possible by simulating an
  18. # interrupt.
  19. 27 class CallbackError < Exception; end # rubocop:disable Lint/InheritException
  20. 27 module InstanceMethods
  21. 27 include HTTPX::Callbacks
  22. 27 CALLBACKS.each do |meth|
  23. 243 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  24. 9 def on_#{meth}(&blk) # def on_connection_opened(&blk)
  25. 9 on(:#{meth}, &blk) # on(:connection_opened, &blk)
  26. self # self
  27. end # end
  28. MOD
  29. end
  30. 27 private
  31. 27 def branch(options, &blk)
  32. 16 super(options).tap do |sess|
  33. 16 CALLBACKS.each do |cb|
  34. 144 next unless callbacks_for?(cb)
  35. 16 sess.callbacks(cb).concat(callbacks(cb))
  36. end
  37. 16 sess.wrap(&blk) if blk
  38. end
  39. end
  40. 27 def do_init_connection(connection, selector)
  41. 225 super
  42. 225 connection.on(:open) do
  43. 195 next unless connection.current_session == self
  44. 195 emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
  45. end
  46. 225 connection.on(:close) do
  47. 211 next unless connection.current_session == self
  48. 211 emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
  49. end
  50. 225 connection
  51. end
  52. 27 def set_request_callbacks(request)
  53. 227 super
  54. 227 request.on(:headers) do
  55. 179 emit_or_callback_error(:request_started, request)
  56. end
  57. 227 request.on(:body_chunk) do |chunk|
  58. 16 emit_or_callback_error(:request_body_chunk, request, chunk)
  59. end
  60. 227 request.on(:done) do
  61. 163 emit_or_callback_error(:request_completed, request)
  62. end
  63. 227 request.on(:response_started) do |res|
  64. 179 if res.is_a?(Response)
  65. 147 emit_or_callback_error(:response_started, request, res)
  66. 131 res.on(:chunk_received) do |chunk|
  67. 168 emit_or_callback_error(:response_body_chunk, request, res, chunk)
  68. end
  69. else
  70. 32 emit_or_callback_error(:request_error, request, res.error)
  71. end
  72. end
  73. 227 request.on(:response) do |res|
  74. 131 emit_or_callback_error(:response_completed, request, res)
  75. end
  76. end
  77. 27 def emit_or_callback_error(*args)
  78. 1226 emit(*args)
  79. rescue StandardError => e
  80. 128 ex = CallbackError.new(e.message)
  81. 128 ex.set_backtrace(e.backtrace)
  82. 128 raise ex
  83. end
  84. 27 def receive_requests(*)
  85. 227 super
  86. rescue CallbackError => e
  87. 120 raise e.cause
  88. end
  89. 27 def close(*)
  90. 225 super
  91. rescue CallbackError => e
  92. 8 raise e.cause
  93. end
  94. end
  95. end
  96. 27 register_plugin :callbacks, Callbacks
  97. end
  98. end

lib/httpx/plugins/circuit_breaker.rb

100.0% lines covered

66 relevant lines. 66 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin implements a circuit breaker around connection errors.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Circuit-Breaker
  8. #
  9. 8 module CircuitBreaker
  10. 8 using URIExtensions
  11. 8 def self.load_dependencies(*)
  12. 56 require_relative "circuit_breaker/circuit"
  13. 56 require_relative "circuit_breaker/circuit_store"
  14. end
  15. 8 def self.extra_options(options)
  16. 56 options.merge(
  17. circuit_breaker_max_attempts: 3,
  18. circuit_breaker_reset_attempts_in: 60,
  19. circuit_breaker_break_in: 60,
  20. circuit_breaker_half_open_drip_rate: 1
  21. )
  22. end
  23. 8 module InstanceMethods
  24. 8 include HTTPX::Callbacks
  25. 8 def initialize(*)
  26. 56 super
  27. 56 @circuit_store = CircuitStore.new(@options)
  28. end
  29. 8 %i[circuit_open].each do |meth|
  30. 8 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  31. 1 def on_#{meth}(&blk) # def on_circuit_open(&blk)
  32. 1 on(:#{meth}, &blk) # on(:circuit_open, &blk)
  33. self # self
  34. end # end
  35. MOD
  36. end
  37. 8 private
  38. 8 def send_requests(*requests)
  39. # @type var short_circuit_responses: Array[response]
  40. 224 short_circuit_responses = []
  41. # run all requests through the circuit breaker, see if the circuit is
  42. # open for any of them.
  43. 224 real_requests = requests.each_with_index.with_object([]) do |(req, idx), real_reqs|
  44. 224 short_circuit_response = @circuit_store.try_respond(req)
  45. 224 if short_circuit_response.nil?
  46. 176 real_reqs << req
  47. 176 next
  48. end
  49. 42 short_circuit_responses[idx] = short_circuit_response
  50. end
  51. # run requests for the remainder
  52. 224 unless real_requests.empty?
  53. 176 responses = super(*real_requests)
  54. 176 real_requests.each_with_index do |request, idx|
  55. 154 short_circuit_responses[requests.index(request)] = responses[idx]
  56. end
  57. end
  58. 224 short_circuit_responses
  59. end
  60. 8 def set_request_callbacks(request)
  61. 224 super
  62. 224 request.on(:response) do |response|
  63. 176 emit(:circuit_open, request) if try_circuit_open(request, response)
  64. end
  65. end
  66. 8 def try_circuit_open(request, response)
  67. 176 if response.is_a?(ErrorResponse)
  68. 112 case response.error
  69. when RequestTimeoutError
  70. 80 @circuit_store.try_open(request.uri, response)
  71. else
  72. 48 @circuit_store.try_open(request.origin, response)
  73. end
  74. 48 elsif (break_on = request.options.circuit_breaker_break_on) && break_on.call(response)
  75. 16 @circuit_store.try_open(request.uri, response)
  76. else
  77. 32 @circuit_store.try_close(request.uri)
  78. 12 nil
  79. end
  80. end
  81. end
  82. # adds support for the following options:
  83. #
  84. # :circuit_breaker_max_attempts :: the number of attempts the circuit allows, before it is opened (defaults to <tt>3</tt>).
  85. # :circuit_breaker_reset_attempts_in :: the time a circuit stays open at most, before it resets (defaults to <tt>60</tt>).
  86. # :circuit_breaker_break_on :: callable defining an alternative rule for a response to break
  87. # (i.e. <tt>->(res) { res.status == 429 } </tt>)
  88. # :circuit_breaker_break_in :: the time that must elapse before an open circuit can transit to the half-open state
  89. # (defaults to <tt><60</tt>).
  90. # :circuit_breaker_half_open_drip_rate :: the rate of requests a circuit allows to be performed when in an half-open state
  91. # (defaults to <tt>1</tt>).
  92. 8 module OptionsMethods
  93. 8 def option_circuit_breaker_max_attempts(value)
  94. 112 attempts = Integer(value)
  95. 112 raise TypeError, ":circuit_breaker_max_attempts must be positive" unless attempts.positive?
  96. 112 attempts
  97. end
  98. 8 def option_circuit_breaker_reset_attempts_in(value)
  99. 64 timeout = Float(value)
  100. 64 raise TypeError, ":circuit_breaker_reset_attempts_in must be positive" unless timeout.positive?
  101. 64 timeout
  102. end
  103. 8 def option_circuit_breaker_break_in(value)
  104. 88 timeout = Float(value)
  105. 88 raise TypeError, ":circuit_breaker_break_in must be positive" unless timeout.positive?
  106. 88 timeout
  107. end
  108. 8 def option_circuit_breaker_half_open_drip_rate(value)
  109. 88 ratio = Float(value)
  110. 88 raise TypeError, ":circuit_breaker_half_open_drip_rate must be a number between 0 and 1" unless (0..1).cover?(ratio)
  111. 88 ratio
  112. end
  113. 8 def option_circuit_breaker_break_on(value)
  114. 16 raise TypeError, ":circuit_breaker_break_on must be called with the response" unless value.respond_to?(:call)
  115. 16 value
  116. end
  117. end
  118. end
  119. 8 register_plugin :circuit_breaker, CircuitBreaker
  120. end
  121. end

lib/httpx/plugins/circuit_breaker/circuit.rb

100.0% lines covered

47 relevant lines. 47 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins::CircuitBreaker
  4. #
  5. # A circuit is assigned to a given absoolute url or origin.
  6. #
  7. # It sets +max_attempts+, the number of attempts the circuit allows, before it is opened.
  8. # It sets +reset_attempts_in+, the time a circuit stays open at most, before it resets.
  9. # It sets +break_in+, the time that must elapse before an open circuit can transit to the half-open state.
  10. # It sets +circuit_breaker_half_open_drip_rate+, the rate of requests a circuit allows to be performed when in an half-open state.
  11. #
  12. 8 class Circuit
  13. 8 def initialize(max_attempts, reset_attempts_in, break_in, circuit_breaker_half_open_drip_rate)
  14. 56 @max_attempts = max_attempts
  15. 56 @reset_attempts_in = reset_attempts_in
  16. 56 @break_in = break_in
  17. 56 @circuit_breaker_half_open_drip_rate = circuit_breaker_half_open_drip_rate
  18. 56 @attempts = 0
  19. 56 total_real_attempts = @max_attempts * @circuit_breaker_half_open_drip_rate
  20. 56 @drip_factor = (@max_attempts / total_real_attempts).round
  21. 56 @state = :closed
  22. end
  23. 8 def respond
  24. 224 try_close
  25. 196 case @state
  26. when :closed
  27. 51 nil
  28. when :half_open
  29. 49 @attempts += 1
  30. # do real requests while drip rate valid
  31. 56 if (@real_attempts % @drip_factor).zero?
  32. 35 @real_attempts += 1
  33. 35 return
  34. end
  35. 16 @response
  36. when :open
  37. 32 @response
  38. end
  39. end
  40. 8 def try_open(response)
  41. 126 case @state
  42. when :closed
  43. 120 now = Utils.now
  44. 120 if @attempts.positive?
  45. # reset if error happened long ago
  46. 48 @attempts = 0 if now - @attempted_at > @reset_attempts_in
  47. else
  48. 72 @attempted_at = now
  49. end
  50. 105 @attempts += 1
  51. 120 return unless @attempts >= @max_attempts
  52. 64 @state = :open
  53. 64 @opened_at = now
  54. 64 @response = response
  55. when :half_open
  56. # open immediately
  57. 24 @state = :open
  58. 24 @attempted_at = @opened_at = Utils.now
  59. 24 @response = response
  60. end
  61. end
  62. 8 def try_close
  63. 224 case @state
  64. when :closed
  65. 51 nil
  66. when :half_open
  67. # do not close circuit unless attempts exhausted
  68. 48 return unless @attempts >= @max_attempts
  69. # reset!
  70. 16 @attempts = 0
  71. 16 @opened_at = @attempted_at = @response = nil
  72. 16 @state = :closed
  73. when :open
  74. 72 if Utils.elapsed_time(@opened_at) > @break_in
  75. 40 @state = :half_open
  76. 40 @attempts = 0
  77. 40 @real_attempts = 0
  78. end
  79. end
  80. end
  81. end
  82. end
  83. end

lib/httpx/plugins/circuit_breaker/circuit_store.rb

100.0% lines covered

23 relevant lines. 23 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX::Plugins::CircuitBreaker
  3. 8 using HTTPX::URIExtensions
  4. 8 class CircuitStore
  5. 8 def initialize(options)
  6. 56 @circuits = Hash.new do |h, k|
  7. 49 h[k] = Circuit.new(
  8. options.circuit_breaker_max_attempts,
  9. options.circuit_breaker_reset_attempts_in,
  10. options.circuit_breaker_break_in,
  11. options.circuit_breaker_half_open_drip_rate
  12. )
  13. end
  14. 56 @circuits_mutex = Thread::Mutex.new
  15. end
  16. 8 def try_open(uri, response)
  17. 288 circuit = @circuits_mutex.synchronize { get_circuit_for_uri(uri) }
  18. 144 circuit.try_open(response)
  19. end
  20. 8 def try_close(uri)
  21. 32 circuit = @circuits_mutex.synchronize do
  22. 32 return unless @circuits.key?(uri.origin) || @circuits.key?(uri.to_s)
  23. 32 get_circuit_for_uri(uri)
  24. end
  25. 32 circuit.try_close
  26. end
  27. # if circuit is open, it'll respond with the stored response.
  28. # if not, nil.
  29. 8 def try_respond(request)
  30. 448 circuit = @circuits_mutex.synchronize { get_circuit_for_uri(request.uri) }
  31. 224 circuit.respond
  32. end
  33. 8 private
  34. 8 def get_circuit_for_uri(uri)
  35. 400 if uri.respond_to?(:origin) && @circuits.key?(uri.origin)
  36. 288 @circuits[uri.origin]
  37. else
  38. 112 @circuits[uri.to_s]
  39. end
  40. end
  41. end
  42. end

lib/httpx/plugins/content_digest.rb

100.0% lines covered

102 relevant lines. 102 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds `Content-Digest` headers to requests
  6. # and can validate these headers on responses
  7. #
  8. # https://datatracker.ietf.org/doc/html/rfc9530
  9. #
  10. 8 module ContentDigest
  11. 8 class Error < HTTPX::Error; end
  12. # Error raised on response "content-digest" header validation.
  13. 8 class ValidationError < Error
  14. 8 attr_reader :response
  15. 8 def initialize(message, response)
  16. 48 super(message)
  17. 48 @response = response
  18. end
  19. end
  20. 8 class MissingContentDigestError < ValidationError; end
  21. 8 class InvalidContentDigestError < ValidationError; end
  22. 2 SUPPORTED_ALGORITHMS = {
  23. 6 "sha-256" => OpenSSL::Digest::SHA256,
  24. "sha-512" => OpenSSL::Digest::SHA512,
  25. }.freeze
  26. 8 class << self
  27. 8 def extra_options(options)
  28. 208 options.merge(encode_content_digest: true, validate_content_digest: false, content_digest_algorithm: "sha-256")
  29. end
  30. end
  31. # add support for the following options:
  32. #
  33. # :content_digest_algorithm :: the digest algorithm to use. Currently supports `sha-256` and `sha-512`. (defaults to `sha-256`)
  34. # :encode_content_digest :: whether a <tt>Content-Digest</tt> header should be computed for the request;
  35. # can also be a callable object (i.e. <tt>->(req) { ... }</tt>, defaults to <tt>true</tt>)
  36. # :validate_content_digest :: whether a <tt>Content-Digest</tt> header in the response should be validated;
  37. # can also be a callable object (i.e. <tt>->(res) { ... }</tt>, defaults to <tt>false</tt>)
  38. 8 module OptionsMethods
  39. 8 def option_content_digest_algorithm(value)
  40. 216 raise TypeError, ":content_digest_algorithm must be one of 'sha-256', 'sha-512'" unless SUPPORTED_ALGORITHMS.key?(value)
  41. 216 value
  42. end
  43. 8 def option_encode_content_digest(value)
  44. 208 value
  45. end
  46. 8 def option_validate_content_digest(value)
  47. 144 value
  48. end
  49. end
  50. 8 module ResponseBodyMethods
  51. 8 attr_reader :content_digest_buffer
  52. 8 def initialize(response, options)
  53. 176 super
  54. 176 return unless response.headers.key?("content-digest")
  55. 128 should_validate = options.validate_content_digest
  56. 128 should_validate = should_validate.call(response) if should_validate.respond_to?(:call)
  57. 128 return unless should_validate
  58. 112 @content_digest_buffer = Response::Buffer.new(
  59. threshold_size: @options.body_threshold_size,
  60. bytesize: @length,
  61. encoding: @encoding
  62. )
  63. end
  64. 8 def write(chunk)
  65. 291 @content_digest_buffer.write(chunk) if @content_digest_buffer
  66. 291 super
  67. end
  68. 8 def close
  69. 112 if @content_digest_buffer
  70. 112 @content_digest_buffer.close
  71. 112 @content_digest_buffer = nil
  72. end
  73. 112 super
  74. end
  75. end
  76. 8 module InstanceMethods
  77. 8 def build_request(*)
  78. 224 request = super
  79. 224 return request if request.empty?
  80. 48 return request if request.headers.key?("content-digest")
  81. 48 perform_encoding = @options.encode_content_digest
  82. 48 perform_encoding = perform_encoding.call(request) if perform_encoding.respond_to?(:call)
  83. 48 return request unless perform_encoding
  84. 40 digest = base64digest(request.body)
  85. 40 request.headers.add("content-digest", "#{@options.content_digest_algorithm}=:#{digest}:")
  86. 40 request
  87. end
  88. 8 private
  89. 8 def fetch_response(request, _, _)
  90. 595 response = super
  91. 595 return response unless response.is_a?(Response)
  92. 176 perform_validation = @options.validate_content_digest
  93. 176 perform_validation = perform_validation.call(response) if perform_validation.respond_to?(:call)
  94. 176 validate_content_digest(response) if perform_validation
  95. 128 response
  96. rescue ValidationError => e
  97. 48 ErrorResponse.new(request, e)
  98. end
  99. 8 def validate_content_digest(response)
  100. 128 content_digest_header = response.headers["content-digest"]
  101. 128 raise MissingContentDigestError.new("response is missing a `content-digest` header", response) unless content_digest_header
  102. 112 digests = extract_content_digests(content_digest_header)
  103. 112 included_algorithms = SUPPORTED_ALGORITHMS.keys & digests.keys
  104. 112 raise MissingContentDigestError.new("unsupported algorithms: #{digests.keys.join(", ")}", response) if included_algorithms.empty?
  105. 112 content_buffer = response.body.content_digest_buffer
  106. 112 included_algorithms.each do |algorithm|
  107. 112 digest = SUPPORTED_ALGORITHMS.fetch(algorithm).new
  108. 112 digest_received = digests[algorithm]
  109. 14 digest_computed =
  110. 111 if content_buffer.respond_to?(:to_path)
  111. 16 content_buffer.flush
  112. 16 digest.file(content_buffer.to_path).base64digest
  113. else
  114. 96 digest.base64digest(content_buffer.to_s)
  115. end
  116. 4 raise InvalidContentDigestError.new("#{algorithm} digest does not match content",
  117. 111 response) unless digest_received == digest_computed
  118. end
  119. end
  120. 8 def extract_content_digests(header)
  121. 112 header.split(",").to_h do |entry|
  122. 128 algorithm, digest = entry.split("=", 2)
  123. 128 raise Error, "#{entry} is an invalid digest format" unless algorithm && digest
  124. 128 [algorithm, digest.byteslice(1..-2)]
  125. end
  126. end
  127. 8 def base64digest(body)
  128. 40 digest = SUPPORTED_ALGORITHMS.fetch(@options.content_digest_algorithm).new
  129. 40 if body.respond_to?(:read)
  130. 32 if body.respond_to?(:to_path)
  131. 8 digest.file(body.to_path).base64digest
  132. else
  133. 24 raise ContentDigestError, "request body must be rewindable" unless body.respond_to?(:rewind)
  134. 24 buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  135. 2 begin
  136. 24 IO.copy_stream(body, buffer)
  137. 24 buffer.flush
  138. 24 digest.file(buffer.to_path).base64digest
  139. ensure
  140. 24 body.rewind
  141. 24 buffer.close
  142. 24 buffer.unlink
  143. end
  144. end
  145. else
  146. 8 raise ContentDigestError, "base64digest for endless enumerators is not supported" if body.unbounded_body?
  147. 8 buffer = "".b
  148. 16 body.each { |chunk| buffer << chunk }
  149. 8 digest.base64digest(buffer)
  150. end
  151. end
  152. end
  153. end
  154. 8 register_plugin :content_digest, ContentDigest
  155. end
  156. end

lib/httpx/plugins/cookies.rb

100.0% lines covered

52 relevant lines. 52 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "forwardable"
  3. 8 module HTTPX
  4. 8 module Plugins
  5. #
  6. # This plugin implements a persistent cookie jar for the duration of a session.
  7. #
  8. # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
  9. #
  10. # https://gitlab.com/os85/httpx/wikis/Cookies
  11. #
  12. 8 module Cookies
  13. 8 def self.load_dependencies(*)
  14. 144 require "httpx/plugins/cookies/jar"
  15. 144 require "httpx/plugins/cookies/cookie"
  16. 144 require "httpx/plugins/cookies/set_cookie_parser"
  17. end
  18. 8 module InstanceMethods
  19. 8 extend Forwardable
  20. 8 def_delegator :@options, :cookies
  21. 8 def initialize(options = {}, &blk)
  22. 288 super({ cookies: Jar.new }.merge(options), &blk)
  23. end
  24. 8 def wrap
  25. 16 return super unless block_given?
  26. 16 super do |session|
  27. 16 old_cookies_jar = @options.cookies.dup
  28. 1 begin
  29. 16 yield session
  30. ensure
  31. 16 @options = @options.merge(cookies: old_cookies_jar)
  32. end
  33. end
  34. end
  35. 8 def build_request(*)
  36. 320 request = super
  37. 320 request.headers.set_cookie(request.options.cookies[request.uri])
  38. 320 request
  39. end
  40. 8 private
  41. 8 def set_request_callbacks(request)
  42. 320 super
  43. 320 request.on(:response) do |response|
  44. 320 next unless response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
  45. 64 log { "cookies: set-cookie is over #{Cookie::MAX_LENGTH}" } if set_cookie.bytesize > Cookie::MAX_LENGTH
  46. 64 @options.cookies.parse(set_cookie)
  47. end
  48. end
  49. end
  50. 8 module HeadersMethods
  51. 8 def set_cookie(cookies)
  52. 320 return if cookies.empty?
  53. 272 header_value = cookies.sort.join("; ")
  54. 272 add("cookie", header_value)
  55. end
  56. end
  57. # adds support for the following options:
  58. #
  59. # :cookies :: cookie jar for the session (can be a Hash, an Array, an instance of HTTPX::Plugins::Cookies::CookieJar)
  60. 8 module OptionsMethods
  61. 8 def option_headers(*)
  62. 320 value = super
  63. 320 merge_cookie_in_jar(value.delete("cookie"), @cookies) if defined?(@cookies) && value.key?("cookie")
  64. 320 value
  65. end
  66. 8 def option_cookies(value)
  67. 480 jar = value.is_a?(Jar) ? value : Jar.new(value)
  68. 480 merge_cookie_in_jar(@headers.delete("cookie"), jar) if defined?(@headers) && @headers.key?("cookie")
  69. 480 jar
  70. end
  71. 8 private
  72. 8 def merge_cookie_in_jar(cookies, jar)
  73. 16 cookies.each do |ck|
  74. 16 ck.split(/ *; */).each do |cookie|
  75. 32 name, value = cookie.split("=", 2)
  76. 32 jar.add(Cookie.new(name, value))
  77. end
  78. end
  79. end
  80. end
  81. end
  82. 8 register_plugin :cookies, Cookies
  83. end
  84. end

lib/httpx/plugins/cookies/cookie.rb

100.0% lines covered

77 relevant lines. 77 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins::Cookies
  4. # The HTTP Cookie.
  5. #
  6. # Contains the single cookie info: name, value and attributes.
  7. 8 class Cookie
  8. 8 include Comparable
  9. # Maximum number of bytes per cookie (RFC 6265 6.1 requires 4096 at
  10. # least)
  11. 8 MAX_LENGTH = 4096
  12. 8 attr_reader :domain, :path, :name, :value, :created_at
  13. 8 def path=(path)
  14. 184 path = String(path)
  15. 184 @path = path.start_with?("/") ? path : "/"
  16. end
  17. # See #domain.
  18. 8 def domain=(domain)
  19. 40 domain = String(domain)
  20. 40 if domain.start_with?(".")
  21. 16 @for_domain = true
  22. 16 domain = domain[1..-1]
  23. end
  24. 40 return if domain.empty?
  25. 40 @domain_name = DomainName.new(domain)
  26. # RFC 6265 5.3 5.
  27. 40 @for_domain = false if @domain_name.domain.nil? # a public suffix or IP address
  28. 40 @domain = @domain_name.hostname
  29. end
  30. # Compares the cookie with another. When there are many cookies with
  31. # the same name for a URL, the value of the smallest must be used.
  32. 8 def <=>(other)
  33. # RFC 6265 5.4
  34. # Precedence: 1. longer path 2. older creation
  35. 699 (@name <=> other.name).nonzero? ||
  36. 60 (other.path.length <=> @path.length).nonzero? ||
  37. 35 (@created_at <=> other.created_at).nonzero? || 0
  38. end
  39. 8 class << self
  40. 8 def new(cookie, *args)
  41. 504 return cookie if cookie.is_a?(self)
  42. 504 super
  43. end
  44. # Tests if +target_path+ is under +base_path+ as described in RFC
  45. # 6265 5.1.4. +base_path+ must be an absolute path.
  46. # +target_path+ may be empty, in which case it is treated as the
  47. # root path.
  48. #
  49. # e.g.
  50. #
  51. # path_match?('/admin/', '/admin/index') == true
  52. # path_match?('/admin/', '/Admin/index') == false
  53. # path_match?('/admin/', '/admin/') == true
  54. # path_match?('/admin/', '/admin') == false
  55. #
  56. # path_match?('/admin', '/admin') == true
  57. # path_match?('/admin', '/Admin') == false
  58. # path_match?('/admin', '/admins') == false
  59. # path_match?('/admin', '/admin/') == true
  60. # path_match?('/admin', '/admin/index') == true
  61. 8 def path_match?(base_path, target_path)
  62. 1352 base_path.start_with?("/") || (return false)
  63. # RFC 6265 5.1.4
  64. 1352 bsize = base_path.size
  65. 1352 tsize = target_path.size
  66. 1352 return bsize == 1 if tsize.zero? # treat empty target_path as "/"
  67. 1352 return false unless target_path.start_with?(base_path)
  68. 1344 return true if bsize == tsize || base_path.end_with?("/")
  69. 16 target_path[bsize] == "/"
  70. end
  71. end
  72. 8 def initialize(arg, *attrs)
  73. 504 @created_at = Time.now
  74. 504 if attrs.empty?
  75. 24 attr_hash = Hash.try_convert(arg)
  76. else
  77. 480 @name = arg
  78. 480 @value, attr_hash = attrs
  79. 480 attr_hash = Hash.try_convert(attr_hash)
  80. end
  81. 33 attr_hash.each do |key, val|
  82. 312 key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)
  83. 273 case key
  84. when :domain, :path
  85. 201 __send__(:"#{key}=", val)
  86. else
  87. 88 instance_variable_set(:"@#{key}", val)
  88. end
  89. 503 end if attr_hash
  90. 504 @path ||= "/"
  91. 504 raise ArgumentError, "name must be specified" if @name.nil?
  92. 504 @name = @name.to_s
  93. end
  94. 8 def expires
  95. 760 @expires || (@created_at && @max_age ? @created_at + @max_age : nil)
  96. end
  97. 8 def expired?(time = Time.now)
  98. 728 return false unless expires
  99. 32 expires <= time
  100. end
  101. # Returns a string for use in the Cookie header, i.e. `name=value`
  102. # or `name="value"`.
  103. 8 def cookie_value
  104. 483 "#{@name}=#{Scanner.quote(@value.to_s)}"
  105. end
  106. 8 alias_method :to_s, :cookie_value
  107. # Tests if it is OK to send this cookie to a given `uri`. A
  108. # RuntimeError is raised if the cookie's domain is unknown.
  109. 8 def valid_for_uri?(uri)
  110. 712 uri = URI(uri)
  111. # RFC 6265 5.4
  112. 712 return false if @secure && uri.scheme != "https"
  113. 704 acceptable_from_uri?(uri) && Cookie.path_match?(@path, uri.path)
  114. end
  115. 8 private
  116. # Tests if it is OK to accept this cookie if it is sent from a given
  117. # URI/URL, `uri`.
  118. 8 def acceptable_from_uri?(uri)
  119. 736 uri = URI(uri)
  120. 736 host = DomainName.new(uri.host)
  121. # RFC 6265 5.3
  122. 736 if host.hostname == @domain
  123. 16 true
  124. 719 elsif @for_domain # !host-only-flag
  125. 32 host.cookie_domain?(@domain_name)
  126. else
  127. 688 @domain.nil?
  128. end
  129. end
  130. 8 module Scanner
  131. 8 RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
  132. 8 module_function
  133. 8 def quote(s)
  134. 552 return s unless s.match(RE_BAD_CHAR)
  135. 8 "\"#{s.gsub(/([\\"])/, "\\\\\\1")}\""
  136. end
  137. end
  138. end
  139. end
  140. end

lib/httpx/plugins/cookies/jar.rb

100.0% lines covered

46 relevant lines. 46 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins::Cookies
  4. # The Cookie Jar
  5. #
  6. # It holds a bunch of cookies.
  7. 8 class Jar
  8. 8 using URIExtensions
  9. 8 include Enumerable
  10. 8 def initialize_dup(orig)
  11. 216 super
  12. 216 @cookies = orig.instance_variable_get(:@cookies).dup
  13. end
  14. 8 def initialize(cookies = nil)
  15. 536 @cookies = []
  16. 120 cookies.each do |elem|
  17. 176 cookie = case elem
  18. when Cookie
  19. 16 elem
  20. when Array
  21. 144 Cookie.new(*elem)
  22. else
  23. 16 Cookie.new(elem)
  24. end
  25. 176 @cookies << cookie
  26. 535 end if cookies
  27. end
  28. 8 def parse(set_cookie)
  29. 144 SetCookieParser.call(set_cookie) do |name, value, attrs|
  30. 208 add(Cookie.new(name, value, attrs))
  31. end
  32. end
  33. 8 def add(cookie, path = nil)
  34. 456 c = cookie.dup
  35. 456 c.path = path if path && c.path == "/"
  36. # If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
  37. # as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
  38. 864 @cookies.delete_if { |ck| ck.name == c.name && ck.domain == c.domain && ck.path == c.path }
  39. 456 @cookies << c
  40. end
  41. 8 def [](uri)
  42. 472 each(uri).sort
  43. end
  44. 8 def each(uri = nil, &blk)
  45. 1184 return enum_for(__method__, uri) unless blk
  46. 680 return @cookies.each(&blk) unless uri
  47. 472 now = Time.now
  48. 472 tpath = uri.path
  49. 472 @cookies.delete_if do |cookie|
  50. 728 if cookie.expired?(now)
  51. 16 true
  52. else
  53. 712 yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
  54. 712 false
  55. end
  56. end
  57. end
  58. 8 def merge(other)
  59. 200 cookies_dup = dup
  60. 200 other.each do |elem|
  61. 216 cookie = case elem
  62. when Cookie
  63. 200 elem
  64. when Array
  65. 8 Cookie.new(*elem)
  66. else
  67. 8 Cookie.new(elem)
  68. end
  69. 216 cookies_dup.add(cookie)
  70. end
  71. 200 cookies_dup
  72. end
  73. end
  74. end
  75. end

lib/httpx/plugins/cookies/set_cookie_parser.rb

100.0% lines covered

72 relevant lines. 72 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "strscan"
  3. 8 require "time"
  4. 8 module HTTPX
  5. 8 module Plugins::Cookies
  6. 8 module SetCookieParser
  7. # Whitespace.
  8. 8 RE_WSP = /[ \t]+/.freeze
  9. # A pattern that matches a cookie name or attribute name which may
  10. # be empty, capturing trailing whitespace.
  11. 8 RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
  12. 8 RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
  13. # A pattern that matches the comma in a (typically date) value.
  14. 8 RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
  15. 8 module_function
  16. 8 def scan_dquoted(scanner)
  17. 16 s = +""
  18. 22 until scanner.eos?
  19. 64 break if scanner.skip(/"/)
  20. 48 if scanner.skip(/\\/)
  21. 16 s << scanner.getch
  22. 31 elsif scanner.scan(/[^"\\]+/)
  23. 32 s << scanner.matched
  24. end
  25. end
  26. 16 s
  27. end
  28. 8 def scan_value(scanner, comma_as_separator = false)
  29. 440 value = +""
  30. 498 until scanner.eos?
  31. 760 if scanner.scan(/[^,;"]+/)
  32. 432 value << scanner.matched
  33. 327 elsif scanner.skip(/"/)
  34. # RFC 6265 2.2
  35. # A cookie-value may be DQUOTE'd.
  36. 16 value << scan_dquoted(scanner)
  37. 311 elsif scanner.check(/;/)
  38. 232 break
  39. 79 elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
  40. 64 break
  41. else
  42. 16 value << scanner.getch
  43. end
  44. end
  45. 440 value.rstrip!
  46. 440 value
  47. end
  48. 8 def scan_name_value(scanner, comma_as_separator = false)
  49. 440 name = scanner.scan(RE_NAME)
  50. 440 name.rstrip! if name
  51. 440 if scanner.skip(/=/)
  52. 432 value = scan_value(scanner, comma_as_separator)
  53. else
  54. 8 scan_value(scanner, comma_as_separator)
  55. 8 value = nil
  56. end
  57. 440 [name, value]
  58. end
  59. 8 def call(set_cookie)
  60. 144 scanner = StringScanner.new(set_cookie)
  61. # RFC 6265 4.1.1 & 5.2
  62. 170 until scanner.eos?
  63. 208 start = scanner.pos
  64. 208 len = nil
  65. 208 scanner.skip(RE_WSP)
  66. 208 name, value = scan_name_value(scanner, true)
  67. 208 value = nil if name && name.empty?
  68. 208 attrs = {}
  69. 237 until scanner.eos?
  70. 296 if scanner.skip(/,/)
  71. # The comma is used as separator for concatenating multiple
  72. # values of a header.
  73. 64 len = (scanner.pos - 1) - start
  74. 64 break
  75. 231 elsif scanner.skip(/;/)
  76. 232 scanner.skip(RE_WSP)
  77. 232 aname, avalue = scan_name_value(scanner, true)
  78. 232 next if (aname.nil? || aname.empty?) || value.nil?
  79. 232 aname.downcase!
  80. 203 case aname
  81. when "expires"
  82. 16 next unless avalue
  83. # RFC 6265 5.2.1
  84. 16 (avalue = Time.parse(avalue)) || next
  85. when "max-age"
  86. 8 next unless avalue
  87. # RFC 6265 5.2.2
  88. 8 next unless /\A-?\d+\z/.match?(avalue)
  89. 8 avalue = Integer(avalue)
  90. when "domain"
  91. # RFC 6265 5.2.3
  92. # An empty value SHOULD be ignored.
  93. 24 next if avalue.nil? || avalue.empty?
  94. when "path"
  95. # RFC 6265 5.2.4
  96. # A relative path must be ignored rather than normalizing it
  97. # to "/".
  98. 176 next unless avalue && avalue.start_with?("/")
  99. when "secure", "httponly"
  100. # RFC 6265 5.2.5, 5.2.6
  101. 7 avalue = true
  102. end
  103. 203 attrs[aname] = avalue
  104. end
  105. end
  106. 208 len ||= scanner.pos - start
  107. 208 next if len > Cookie::MAX_LENGTH
  108. 208 yield(name, value, attrs) if name && !name.empty? && value
  109. end
  110. end
  111. end
  112. end
  113. end

lib/httpx/plugins/digest_auth.rb

100.0% lines covered

29 relevant lines. 29 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds helper methods to implement HTTP Digest Auth (https://datatracker.ietf.org/doc/html/rfc7616)
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
  8. #
  9. 8 module DigestAuth
  10. 8 DigestError = Class.new(Error)
  11. 8 class << self
  12. 8 def extra_options(options)
  13. 160 options.merge(max_concurrent_requests: 1)
  14. end
  15. 8 def load_dependencies(*)
  16. 160 require_relative "auth/digest"
  17. end
  18. end
  19. # adds support for the following options:
  20. #
  21. # :digest :: instance of HTTPX::Plugins::Authentication::Digest, used to authenticate requests in the session.
  22. 8 module OptionsMethods
  23. 8 def option_digest(value)
  24. 320 raise TypeError, ":digest must be a #{Authentication::Digest}" unless value.is_a?(Authentication::Digest)
  25. 320 value
  26. end
  27. end
  28. 8 module InstanceMethods
  29. 8 def digest_auth(user, password, hashed: false)
  30. 160 with(digest: Authentication::Digest.new(user, password, hashed: hashed))
  31. end
  32. 8 private
  33. 8 def send_requests(*requests)
  34. 192 requests.flat_map do |request|
  35. 192 digest = request.options.digest
  36. 192 next super(request) unless digest
  37. 320 probe_response = wrap { super(request).first }
  38. 160 return probe_response unless probe_response.is_a?(Response)
  39. 160 if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
  40. 144 request.transition(:idle)
  41. 126 request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
  42. 144 super(request)
  43. else
  44. 16 probe_response
  45. end
  46. end
  47. end
  48. end
  49. end
  50. 8 register_plugin :digest_auth, DigestAuth
  51. end
  52. end

lib/httpx/plugins/expect.rb

100.0% lines covered

56 relevant lines. 56 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin makes all HTTP/1.1 requests with a body send the "Expect: 100-continue".
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Expect#expect
  8. #
  9. 8 module Expect
  10. 8 EXPECT_TIMEOUT = 2
  11. 8 class << self
  12. 8 def no_expect_store
  13. 176 @no_expect_store ||= []
  14. end
  15. 8 def extra_options(options)
  16. 216 options.merge(expect_timeout: EXPECT_TIMEOUT)
  17. end
  18. end
  19. # adds support for the following options:
  20. #
  21. # :expect_timeout :: time (in seconds) to wait for a 100-expect response,
  22. # before retrying without the Expect header (defaults to <tt>2</tt>).
  23. # :expect_threshold_size :: min threshold (in bytes) of the request payload to enable the 100-continue negotiation on.
  24. 8 module OptionsMethods
  25. 8 def option_expect_timeout(value)
  26. 384 seconds = Float(value)
  27. 384 raise TypeError, ":expect_timeout must be positive" unless seconds.positive?
  28. 384 seconds
  29. end
  30. 8 def option_expect_threshold_size(value)
  31. 16 bytes = Integer(value)
  32. 16 raise TypeError, ":expect_threshold_size must be positive" unless bytes.positive?
  33. 16 bytes
  34. end
  35. end
  36. 8 module RequestMethods
  37. 8 def initialize(*)
  38. 248 super
  39. 248 return if @body.empty?
  40. 168 threshold = @options.expect_threshold_size
  41. 168 return if threshold && !@body.unbounded_body? && @body.bytesize < threshold
  42. 152 return if Expect.no_expect_store.include?(origin)
  43. 126 @headers["expect"] = "100-continue"
  44. end
  45. 8 def response=(response)
  46. 184 if response.is_a?(Response) &&
  47. response.status == 100 &&
  48. !@headers.key?("expect") &&
  49. 3 (@state == :body || @state == :done)
  50. # if we're past this point, this means that we just received a 100-Continue response,
  51. # but the request doesn't have the expect flag, and is already flushing (or flushed) the body.
  52. #
  53. # this means that expect was deactivated for this request too soon, i.e. response took longer.
  54. #
  55. # so we have to reactivate it again.
  56. 7 @headers["expect"] = "100-continue"
  57. 8 @informational_status = 100
  58. 8 Expect.no_expect_store.delete(origin)
  59. end
  60. 184 super
  61. end
  62. end
  63. 8 module ConnectionMethods
  64. 8 def send_request_to_parser(request)
  65. 112 super
  66. 112 return unless request.headers["expect"] == "100-continue"
  67. 80 expect_timeout = request.options.expect_timeout
  68. 80 return if expect_timeout.nil? || expect_timeout.infinite?
  69. 80 set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
  70. # expect timeout expired
  71. 16 if request.state == :expect && !request.expects?
  72. 16 Expect.no_expect_store << request.origin
  73. 16 request.headers.delete("expect")
  74. 16 consume
  75. end
  76. end
  77. end
  78. end
  79. 8 module InstanceMethods
  80. 8 def fetch_response(request, selector, options)
  81. 426 response = super
  82. 426 return unless response
  83. 112 if response.is_a?(Response) && response.status == 417 && request.headers.key?("expect")
  84. 16 response.close
  85. 16 request.headers.delete("expect")
  86. 16 request.transition(:idle)
  87. 16 send_request(request, selector, options)
  88. 14 return
  89. end
  90. 96 response
  91. end
  92. end
  93. end
  94. 8 register_plugin :expect, Expect
  95. end
  96. end

lib/httpx/plugins/follow_redirects.rb

100.0% lines covered

108 relevant lines. 108 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 15 module HTTPX
  3. 15 InsecureRedirectError = Class.new(Error)
  4. 15 module Plugins
  5. #
  6. # This plugin adds support for automatically following redirect (status 30X) responses.
  7. #
  8. # It has a default upper bound of followed redirects (see *MAX_REDIRECTS* and the *max_redirects* option),
  9. # after which it will return the last redirect response. It will **not** raise an exception.
  10. #
  11. # It doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
  12. #
  13. # It doesn't propagate authorization related headers to requests redirecting to different origins
  14. # (see *allow_auth_to_other_origins*) to override.
  15. #
  16. # It allows customization of when to redirect via the *redirect_on* callback option).
  17. #
  18. # https://gitlab.com/os85/httpx/wikis/Follow-Redirects
  19. #
  20. 15 module FollowRedirects
  21. 15 MAX_REDIRECTS = 3
  22. 15 REDIRECT_STATUS = (300..399).freeze
  23. 15 REQUEST_BODY_HEADERS = %w[transfer-encoding content-encoding content-type content-length content-language content-md5 trailer].freeze
  24. 15 using URIExtensions
  25. # adds support for the following options:
  26. #
  27. # :max_redirects :: max number of times a request will be redirected (defaults to <tt>3</tt>).
  28. # :follow_insecure_redirects :: whether redirects to an "http://" URI, when coming from an "https//", are allowed
  29. # (defaults to <tt>false</tt>).
  30. # :allow_auth_to_other_origins :: whether auth-related headers, such as "Authorization", are propagated on redirection
  31. # (defaults to <tt>false</tt>).
  32. # :redirect_on :: optional callback which receives the redirect location and can halt the redirect chain if it returns <tt>false</tt>.
  33. 15 module OptionsMethods
  34. 15 def option_max_redirects(value)
  35. 462 num = Integer(value)
  36. 462 raise TypeError, ":max_redirects must be positive" if num.negative?
  37. 462 num
  38. end
  39. 15 def option_follow_insecure_redirects(value)
  40. 24 value
  41. end
  42. 15 def option_allow_auth_to_other_origins(value)
  43. 24 value
  44. end
  45. 15 def option_redirect_on(value)
  46. 48 raise TypeError, ":redirect_on must be callable" unless value.respond_to?(:call)
  47. 48 value
  48. end
  49. end
  50. 15 module InstanceMethods
  51. # returns a session with the *max_redirects* option set to +n+
  52. 15 def max_redirects(n)
  53. 48 with(max_redirects: n.to_i)
  54. end
  55. 15 private
  56. 15 def fetch_response(request, selector, options)
  57. 4076888 redirect_request = request.redirect_request
  58. 4076888 response = super(redirect_request, selector, options)
  59. 4076888 return unless response
  60. 568 max_redirects = redirect_request.max_redirects
  61. 568 return response unless response.is_a?(Response)
  62. 552 return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
  63. 361 return response unless max_redirects.positive?
  64. 329 redirect_uri = __get_location_from_response(response)
  65. 329 if options.redirect_on
  66. 32 redirect_allowed = options.redirect_on.call(redirect_uri)
  67. 32 return response unless redirect_allowed
  68. end
  69. # build redirect request
  70. 313 request_body = redirect_request.body
  71. 313 redirect_method = "GET"
  72. 313 redirect_params = {}
  73. 313 if response.status == 305 && options.respond_to?(:proxy)
  74. 8 request_body.rewind
  75. # The requested resource MUST be accessed through the proxy given by
  76. # the Location field. The Location field gives the URI of the proxy.
  77. 8 redirect_options = options.merge(headers: redirect_request.headers,
  78. proxy: { uri: redirect_uri },
  79. max_redirects: max_redirects - 1)
  80. 7 redirect_params[:body] = request_body
  81. 8 redirect_uri = redirect_request.uri
  82. 8 options = redirect_options
  83. else
  84. 305 redirect_headers = redirect_request_headers(redirect_request.uri, redirect_uri, request.headers, options)
  85. 305 redirect_opts = Hash[options]
  86. 269 redirect_params[:max_redirects] = max_redirects - 1
  87. 305 unless request_body.empty?
  88. 24 if response.status == 307
  89. # The method and the body of the original request are reused to perform the redirected request.
  90. 8 redirect_method = redirect_request.verb
  91. 8 request_body.rewind
  92. 7 redirect_params[:body] = request_body
  93. else
  94. # redirects are **ALWAYS** GET, so remove body-related headers
  95. 16 REQUEST_BODY_HEADERS.each do |h|
  96. 112 redirect_headers.delete(h)
  97. end
  98. 14 redirect_params[:body] = nil
  99. end
  100. end
  101. 305 options = options.class.new(redirect_opts.merge(headers: redirect_headers.to_h))
  102. end
  103. 313 redirect_uri = Utils.to_uri(redirect_uri)
  104. 313 if !options.follow_insecure_redirects &&
  105. response.uri.scheme == "https" &&
  106. redirect_uri.scheme == "http"
  107. 8 error = InsecureRedirectError.new(redirect_uri.to_s)
  108. 8 error.set_backtrace(caller)
  109. 7 return ErrorResponse.new(request, error)
  110. end
  111. 305 retry_request = build_request(redirect_method, redirect_uri, redirect_params, options)
  112. 305 request.redirect_request = retry_request
  113. 305 redirect_after = response.headers["retry-after"]
  114. 305 if redirect_after
  115. # Servers send the "Retry-After" header field to indicate how long the
  116. # user agent ought to wait before making a follow-up request.
  117. # When sent with any 3xx (Redirection) response, Retry-After indicates
  118. # the minimum time that the user agent is asked to wait before issuing
  119. # the redirected request.
  120. #
  121. 31 redirect_after = Utils.parse_retry_after(redirect_after)
  122. 31 retry_start = Utils.now
  123. 31 log { "redirecting after #{redirect_after} secs..." }
  124. 31 selector.after(redirect_after) do
  125. 31 if (response = request.response)
  126. 15 response.finish!
  127. 15 retry_request.response = response
  128. # request has terminated abruptly meanwhile
  129. 15 retry_request.emit(:response, response)
  130. else
  131. 16 log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
  132. 16 send_request(retry_request, selector, options)
  133. end
  134. end
  135. else
  136. 274 send_request(retry_request, selector, options)
  137. end
  138. 115 nil
  139. end
  140. # :nodoc:
  141. 15 def redirect_request_headers(original_uri, redirect_uri, headers, options)
  142. 305 headers = headers.dup
  143. 305 return headers if options.allow_auth_to_other_origins
  144. 297 return headers unless headers.key?("authorization")
  145. 8 return headers if original_uri.origin == redirect_uri.origin
  146. 8 headers.delete("authorization")
  147. 8 headers
  148. end
  149. # :nodoc:
  150. 15 def __get_location_from_response(response)
  151. # @type var location_uri: http_uri
  152. 329 location_uri = URI(response.headers["location"])
  153. 329 location_uri = response.uri.merge(location_uri) if location_uri.relative?
  154. 329 location_uri
  155. end
  156. end
  157. 15 module RequestMethods
  158. # returns the top-most original HTTPX::Request from the redirect chain
  159. 15 attr_accessor :root_request
  160. # returns the follow-up redirect request, or itself
  161. 15 def redirect_request
  162. 4076888 @redirect_request || self
  163. end
  164. # sets the follow-up redirect request
  165. 15 def redirect_request=(req)
  166. 305 @redirect_request = req
  167. 305 req.root_request = @root_request || self
  168. 305 @response = nil
  169. end
  170. 15 def response
  171. 4078689 return super unless @redirect_request && @response.nil?
  172. 73 @redirect_request.response
  173. end
  174. 15 def max_redirects
  175. 568 @options.max_redirects || MAX_REDIRECTS
  176. end
  177. end
  178. 15 module ConnectionMethods
  179. 15 private
  180. 15 def set_request_request_timeout(request)
  181. 541 return unless request.root_request.nil?
  182. 257 super
  183. end
  184. end
  185. end
  186. 15 register_plugin :follow_redirects, FollowRedirects
  187. end
  188. end

lib/httpx/plugins/grpc.rb

100.0% lines covered

133 relevant lines. 133 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 GRPCError = Class.new(Error) do
  4. 6 attr_reader :status, :details, :metadata
  5. 6 def initialize(status, details, metadata)
  6. 24 @status = status
  7. 24 @details = details
  8. 24 @metadata = metadata
  9. 24 super("GRPC error, code=#{status}, details=#{details}, metadata=#{metadata}")
  10. end
  11. end
  12. 6 module Plugins
  13. #
  14. # This plugin adds DSL to build GRPC interfaces.
  15. #
  16. # https://gitlab.com/os85/httpx/wikis/GRPC
  17. #
  18. 6 module GRPC
  19. 6 unless String.method_defined?(:underscore)
  20. 6 module StringExtensions
  21. 6 refine String do
  22. 6 def underscore
  23. 312 s = dup # Avoid mutating the argument, as it might be frozen.
  24. 312 s.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
  25. 312 s.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
  26. 312 s.tr!("-", "_")
  27. 312 s.downcase!
  28. 312 s
  29. end
  30. end
  31. end
  32. 6 using StringExtensions
  33. end
  34. 6 DEADLINE = 60
  35. 6 MARSHAL_METHOD = :encode
  36. 6 UNMARSHAL_METHOD = :decode
  37. 6 HEADERS = {
  38. "content-type" => "application/grpc",
  39. "te" => "trailers",
  40. "accept" => "application/grpc",
  41. # metadata fits here
  42. # ex "foo-bin" => base64("bar")
  43. }.freeze
  44. 6 class << self
  45. 6 def load_dependencies(*)
  46. 138 require "stringio"
  47. 138 require "httpx/plugins/grpc/grpc_encoding"
  48. 138 require "httpx/plugins/grpc/message"
  49. 138 require "httpx/plugins/grpc/call"
  50. end
  51. 6 def configure(klass)
  52. 138 klass.plugin(:persistent)
  53. 138 klass.plugin(:stream)
  54. end
  55. 6 def extra_options(options)
  56. 138 options.merge(
  57. fallback_protocol: "h2",
  58. grpc_rpcs: {}.freeze,
  59. grpc_compression: false,
  60. grpc_deadline: DEADLINE
  61. )
  62. end
  63. end
  64. 6 module OptionsMethods
  65. 6 def option_grpc_service(value)
  66. 120 String(value)
  67. end
  68. 6 def option_grpc_compression(value)
  69. 162 case value
  70. when true, false
  71. 138 value
  72. else
  73. 24 value.to_s
  74. end
  75. end
  76. 6 def option_grpc_rpcs(value)
  77. 1116 Hash[value]
  78. end
  79. 6 def option_grpc_deadline(value)
  80. 804 raise TypeError, ":grpc_deadline must be positive" unless value.positive?
  81. 804 value
  82. end
  83. 6 def option_call_credentials(value)
  84. 18 raise TypeError, ":call_credentials must respond to #call" unless value.respond_to?(:call)
  85. 18 value
  86. end
  87. end
  88. 6 module ResponseMethods
  89. 6 attr_reader :trailing_metadata
  90. 6 def merge_headers(trailers)
  91. 114 @trailing_metadata = Hash[trailers]
  92. 114 super
  93. end
  94. end
  95. 6 module RequestBodyMethods
  96. 6 def initialize(*, **)
  97. 126 super
  98. 126 if (compression = @headers["grpc-encoding"])
  99. 12 deflater_body = self.class.initialize_deflater_body(@body, compression)
  100. 12 @body = Transcoder::GRPCEncoding.encode(deflater_body || @body, compressed: !deflater_body.nil?)
  101. else
  102. 114 @body = Transcoder::GRPCEncoding.encode(@body, compressed: false)
  103. end
  104. end
  105. end
  106. 6 module InstanceMethods
  107. 6 def with_channel_credentials(ca_path, key = nil, cert = nil, **ssl_opts)
  108. # @type var ssl_params: ::Hash[::Symbol, untyped]
  109. 72 ssl_params = {
  110. **ssl_opts,
  111. ca_file: ca_path,
  112. }
  113. 72 if key
  114. 72 key = File.read(key) if File.file?(key)
  115. 72 ssl_params[:key] = OpenSSL::PKey.read(key)
  116. end
  117. 72 if cert
  118. 72 cert = File.read(cert) if File.file?(cert)
  119. 72 ssl_params[:cert] = OpenSSL::X509::Certificate.new(cert)
  120. end
  121. 72 with(ssl: ssl_params)
  122. end
  123. 6 def rpc(rpc_name, input, output, **opts)
  124. 312 rpc_name = rpc_name.to_s
  125. 312 raise Error, "rpc #{rpc_name} already defined" if @options.grpc_rpcs.key?(rpc_name)
  126. rpc_opts = {
  127. 312 deadline: @options.grpc_deadline,
  128. }.merge(opts)
  129. 312 local_rpc_name = rpc_name.underscore
  130. 312 session_class = Class.new(self.class) do
  131. # define rpc method with ruby style name
  132. 312 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  133. def #{local_rpc_name}(input, **opts) # def grpc_action(input, **opts)
  134. rpc_execute("#{local_rpc_name}", input, **opts) # rpc_execute("grpc_action", input, **opts)
  135. end # end
  136. OUT
  137. # define rpc method with original name
  138. 312 unless local_rpc_name == rpc_name
  139. 12 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  140. def #{rpc_name}(input, **opts) # def grpcAction(input, **opts)
  141. rpc_execute("#{local_rpc_name}", input, **opts) # rpc_execute("grpc_action", input, **opts)
  142. end # end
  143. OUT
  144. end
  145. end
  146. 312 session_class.new(@options.merge(
  147. grpc_rpcs: @options.grpc_rpcs.merge(
  148. local_rpc_name => [rpc_name, input, output, rpc_opts]
  149. ).freeze
  150. ))
  151. end
  152. 6 def build_stub(origin, service: nil, compression: false)
  153. 138 scheme = @options.ssl.empty? ? "http" : "https"
  154. 138 origin = URI.parse("#{scheme}://#{origin}")
  155. 138 session = self
  156. 138 if service && service.respond_to?(:rpc_descs)
  157. # it's a grpc generic service
  158. 60 service.rpc_descs.each do |rpc_name, rpc_desc|
  159. rpc_opts = {
  160. 300 marshal_method: rpc_desc.marshal_method,
  161. unmarshal_method: rpc_desc.unmarshal_method,
  162. }
  163. 300 input = rpc_desc.input
  164. 300 input = input.type if input.respond_to?(:type)
  165. 300 output = rpc_desc.output
  166. 300 if output.respond_to?(:type)
  167. 120 rpc_opts[:stream] = true
  168. 120 output = output.type
  169. end
  170. 300 session = session.rpc(rpc_name, input, output, **rpc_opts)
  171. end
  172. 60 service = service.service_name
  173. end
  174. 138 session.with(origin: origin, grpc_service: service, grpc_compression: compression)
  175. end
  176. 6 def execute(rpc_method, input,
  177. deadline: DEADLINE,
  178. metadata: nil,
  179. **opts)
  180. 126 grpc_request = build_grpc_request(rpc_method, input, deadline: deadline, metadata: metadata, **opts)
  181. 126 response = request(grpc_request, **opts)
  182. 126 response.raise_for_status unless opts[:stream]
  183. 114 GRPC::Call.new(response)
  184. end
  185. 6 private
  186. 6 def rpc_execute(rpc_name, input, **opts)
  187. 60 rpc_name, input_enc, output_enc, rpc_opts = @options.grpc_rpcs[rpc_name]
  188. 60 exec_opts = rpc_opts.merge(opts)
  189. 60 marshal_method ||= exec_opts.delete(:marshal_method) || MARSHAL_METHOD
  190. 60 unmarshal_method ||= exec_opts.delete(:unmarshal_method) || UNMARSHAL_METHOD
  191. 60 messages = if input.respond_to?(:each)
  192. 24 Enumerator.new do |y|
  193. 24 input.each do |message|
  194. 48 y << input_enc.__send__(marshal_method, message)
  195. end
  196. end
  197. else
  198. 36 input_enc.__send__(marshal_method, input)
  199. end
  200. 60 call = execute(rpc_name, messages, **exec_opts)
  201. 60 call.decoder = output_enc.method(unmarshal_method)
  202. 60 call
  203. end
  204. 6 def build_grpc_request(rpc_method, input, deadline:, metadata: nil, **)
  205. 126 uri = @options.origin.dup
  206. 126 rpc_method = "/#{rpc_method}" unless rpc_method.start_with?("/")
  207. 126 rpc_method = "/#{@options.grpc_service}#{rpc_method}" if @options.grpc_service
  208. 126 uri.path = rpc_method
  209. 126 headers = HEADERS.merge(
  210. "grpc-accept-encoding" => ["identity", *@options.supported_compression_formats]
  211. )
  212. 126 unless deadline == Float::INFINITY
  213. # convert to milliseconds
  214. 126 deadline = (deadline * 1000.0).to_i
  215. 126 headers["grpc-timeout"] = "#{deadline}m"
  216. end
  217. 126 headers = headers.merge(metadata.transform_keys(&:to_s)) if metadata
  218. # prepare compressor
  219. 126 compression = @options.grpc_compression == true ? "gzip" : @options.grpc_compression
  220. 126 headers["grpc-encoding"] = compression if compression
  221. 126 headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
  222. 126 build_request("POST", uri, headers: headers, body: input)
  223. end
  224. end
  225. end
  226. 6 register_plugin :grpc, GRPC
  227. end
  228. end

lib/httpx/plugins/grpc/call.rb

90.91% lines covered

33 relevant lines. 30 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. 6 module GRPC
  5. # Encapsulates call information
  6. 6 class Call
  7. 6 attr_writer :decoder
  8. 6 def initialize(response)
  9. 114 @response = response
  10. 156 @decoder = ->(z) { z }
  11. 114 @consumed = false
  12. 114 @grpc_response = nil
  13. end
  14. 6 def inspect
  15. "#{self.class}(#{grpc_response})"
  16. end
  17. 6 def to_s
  18. 66 grpc_response.to_s
  19. end
  20. 6 def metadata
  21. response.headers
  22. end
  23. 6 def trailing_metadata
  24. 72 return unless @consumed
  25. 48 @response.trailing_metadata
  26. end
  27. 6 private
  28. 6 def grpc_response
  29. 186 @grpc_response ||= if @response.respond_to?(:each)
  30. 24 Enumerator.new do |y|
  31. 24 Message.stream(@response).each do |message|
  32. 48 y << @decoder.call(message)
  33. end
  34. 24 @consumed = true
  35. end
  36. else
  37. 90 @consumed = true
  38. 90 @decoder.call(Message.unary(@response))
  39. end
  40. end
  41. 6 def respond_to_missing?(meth, *args, &blk)
  42. 24 grpc_response.respond_to?(meth, *args) || super
  43. end
  44. 6 def method_missing(meth, *args, &blk)
  45. 48 return grpc_response.__send__(meth, *args, &blk) if grpc_response.respond_to?(meth)
  46. super
  47. end
  48. end
  49. end
  50. end
  51. end

lib/httpx/plugins/grpc/grpc_encoding.rb

97.87% lines covered

47 relevant lines. 46 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Transcoder
  4. 6 module GRPCEncoding
  5. 6 class Deflater
  6. 6 extend Forwardable
  7. 6 attr_reader :content_type
  8. 6 def initialize(body, compressed:)
  9. 126 @content_type = body.content_type
  10. 126 @body = BodyReader.new(body)
  11. 126 @compressed = compressed
  12. end
  13. 6 def bytesize
  14. 450 return @body.bytesize if @body.respond_to?(:bytesize)
  15. Float::INFINITY
  16. end
  17. 6 def read(length = nil, outbuf = nil)
  18. 276 buf = @body.read(length, outbuf)
  19. 252 return unless buf
  20. 138 compressed_flag = @compressed ? 1 : 0
  21. 138 buf = outbuf if outbuf
  22. 138 buf = buf.b if buf.frozen?
  23. 138 buf.prepend([compressed_flag, buf.bytesize].pack("CL>"))
  24. 138 buf
  25. end
  26. end
  27. 6 class Inflater
  28. 6 def initialize(response)
  29. 90 @response = response
  30. 90 @grpc_encodings = nil
  31. end
  32. 6 def call(message, &blk)
  33. 114 data = "".b
  34. 114 until message.empty?
  35. 114 compressed, size = message.unpack("CL>")
  36. 114 encoded_data = message.byteslice(5..size + 5 - 1)
  37. 114 if compressed == 1
  38. 12 grpc_encodings.reverse_each do |encoding|
  39. 12 decoder = @response.body.class.initialize_inflater_by_encoding(encoding, @response, bytesize: encoded_data.bytesize)
  40. 12 encoded_data = decoder.call(encoded_data)
  41. 12 blk.call(encoded_data) if blk
  42. 12 data << encoded_data
  43. end
  44. else
  45. 102 blk.call(encoded_data) if blk
  46. 102 data << encoded_data
  47. end
  48. 114 message = message.byteslice((size + 5)..-1)
  49. end
  50. 114 data
  51. end
  52. 6 private
  53. 6 def grpc_encodings
  54. 12 @grpc_encodings ||= @response.headers.get("grpc-encoding")
  55. end
  56. end
  57. 6 def self.encode(*args, **kwargs)
  58. 126 Deflater.new(*args, **kwargs)
  59. end
  60. 6 def self.decode(response)
  61. 90 Inflater.new(response)
  62. end
  63. end
  64. end
  65. end

lib/httpx/plugins/grpc/message.rb

95.83% lines covered

24 relevant lines. 23 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. 6 module GRPC
  5. # Encoding module for GRPC responses
  6. #
  7. # Can encode and decode grpc messages.
  8. 6 module Message
  9. 6 module_function
  10. # decodes a unary grpc response
  11. 6 def unary(response)
  12. 90 verify_status(response)
  13. 66 decoder = Transcoder::GRPCEncoding.decode(response)
  14. 66 decoder.call(response.to_s)
  15. end
  16. # lazy decodes a grpc stream response
  17. 6 def stream(response, &block)
  18. 48 return enum_for(__method__, response) unless block
  19. 24 decoder = Transcoder::GRPCEncoding.decode(response)
  20. 24 response.each do |frame|
  21. 48 decoder.call(frame, &block)
  22. end
  23. 24 verify_status(response)
  24. end
  25. 6 def cancel(request)
  26. request.emit(:refuse, :client_cancellation)
  27. end
  28. # interprets the grpc call trailing metadata, and raises an
  29. # exception in case of error code
  30. 6 def verify_status(response)
  31. # return standard errors if need be
  32. 114 response.raise_for_status
  33. 114 status = Integer(response.headers["grpc-status"])
  34. 114 message = response.headers["grpc-message"]
  35. 114 return if status.zero?
  36. 24 response.close
  37. 24 raise GRPCError.new(status, message, response.trailing_metadata)
  38. end
  39. end
  40. end
  41. end
  42. end

lib/httpx/plugins/h2c.rb

94.92% lines covered

59 relevant lines. 56 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
  6. # (https://datatracker.ietf.org/doc/html/rfc7540#section-3.2)
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2c
  9. #
  10. 8 module H2C
  11. 8 VALID_H2C_VERBS = %w[GET OPTIONS HEAD].freeze
  12. 8 class << self
  13. 8 def load_dependencies(klass)
  14. 16 klass.plugin(:upgrade)
  15. end
  16. 8 def call(connection, request, response)
  17. 16 connection.upgrade_to_h2c(request, response)
  18. end
  19. 8 def extra_options(options)
  20. 16 options.merge(max_concurrent_requests: 1, upgrade_handlers: options.upgrade_handlers.merge("h2c" => self))
  21. end
  22. end
  23. 8 class H2CParser < Connection::HTTP2
  24. 8 def upgrade(request, response)
  25. # skip checks, it is assumed that this is the first
  26. # request in the connection
  27. 16 stream = @connection.upgrade
  28. # on_settings
  29. 16 handle_stream(stream, request)
  30. 14 @streams[request] = stream
  31. # clean up data left behind in the buffer, if the server started
  32. # sending frames
  33. 16 data = response.read
  34. 16 @connection << data
  35. end
  36. end
  37. 8 module RequestMethods
  38. 8 def valid_h2c_verb?
  39. 16 VALID_H2C_VERBS.include?(@verb)
  40. end
  41. end
  42. 8 module ConnectionMethods
  43. 8 using URIExtensions
  44. 8 def initialize(*)
  45. 16 super
  46. 16 @h2c_handshake = false
  47. end
  48. 8 def send(request)
  49. 56 return super if @h2c_handshake
  50. 16 return super unless request.valid_h2c_verb? && request.scheme == "http"
  51. 16 return super if @upgrade_protocol == "h2c"
  52. 16 @h2c_handshake = true
  53. # build upgrade request
  54. 16 request.headers.add("connection", "upgrade")
  55. 16 request.headers.add("connection", "http2-settings")
  56. 14 request.headers["upgrade"] = "h2c"
  57. 14 request.headers["http2-settings"] = ::HTTP2::Client.settings_header(request.options.http2_settings)
  58. 16 super
  59. end
  60. 8 def upgrade_to_h2c(request, response)
  61. 16 prev_parser = @parser
  62. 16 if prev_parser
  63. 16 prev_parser.reset
  64. 14 @inflight -= prev_parser.requests.size
  65. end
  66. 16 @parser = H2CParser.new(@write_buffer, @options)
  67. 16 set_parser_callbacks(@parser)
  68. 14 @inflight += 1
  69. 16 @parser.upgrade(request, response)
  70. 16 @upgrade_protocol = "h2c"
  71. 16 prev_parser.requests.each do |req|
  72. 16 req.transition(:idle)
  73. 16 send(req)
  74. end
  75. end
  76. 8 private
  77. 8 def send_request_to_parser(request)
  78. 56 super
  79. 56 return unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
  80. 16 max_concurrent_requests = parser.max_concurrent_requests
  81. 16 return if max_concurrent_requests == 1
  82. parser.max_concurrent_requests = 1
  83. request.once(:response) do
  84. parser.max_concurrent_requests = max_concurrent_requests
  85. end
  86. end
  87. end
  88. end
  89. 8 register_plugin(:h2c, H2C)
  90. end
  91. end

lib/httpx/plugins/ntlm_auth.rb

100.0% lines covered

30 relevant lines. 30 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 3 module HTTPX
  3. 3 module Plugins
  4. #
  5. # https://gitlab.com/os85/httpx/wikis/Auth#ntlm-auth
  6. #
  7. 3 module NTLMAuth
  8. 3 class << self
  9. 3 def load_dependencies(_klass)
  10. 2 require_relative "auth/ntlm"
  11. end
  12. 3 def extra_options(options)
  13. 2 options.merge(max_concurrent_requests: 1)
  14. end
  15. end
  16. 3 module OptionsMethods
  17. 3 def option_ntlm(value)
  18. 8 raise TypeError, ":ntlm must be a #{Authentication::Ntlm}" unless value.is_a?(Authentication::Ntlm)
  19. 8 value
  20. end
  21. end
  22. 3 module InstanceMethods
  23. 3 def ntlm_auth(user, password, domain = nil)
  24. 4 with(ntlm: Authentication::Ntlm.new(user, password, domain: domain))
  25. end
  26. 3 private
  27. 3 def send_requests(*requests)
  28. 8 requests.flat_map do |request|
  29. 8 ntlm = request.options.ntlm
  30. 8 if ntlm
  31. 4 request.headers["authorization"] = ntlm.negotiate
  32. 8 probe_response = wrap { super(request).first }
  33. 4 return probe_response unless probe_response.is_a?(Response)
  34. 4 if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
  35. 2 request.transition(:idle)
  36. 2 request.headers["authorization"] = ntlm.authenticate(request, probe_response.headers["www-authenticate"])
  37. 2 super(request)
  38. else
  39. 2 probe_response
  40. end
  41. else
  42. 4 super(request)
  43. end
  44. end
  45. end
  46. end
  47. end
  48. 3 register_plugin :ntlm_auth, NTLMAuth
  49. end
  50. end

lib/httpx/plugins/oauth.rb

100.0% lines covered

87 relevant lines. 87 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # https://gitlab.com/os85/httpx/wikis/OAuth
  6. #
  7. 8 module OAuth
  8. 8 class << self
  9. 8 def load_dependencies(_klass)
  10. 144 require_relative "auth/basic"
  11. end
  12. end
  13. 8 SUPPORTED_GRANT_TYPES = %w[client_credentials refresh_token].freeze
  14. 8 SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
  15. 8 class OAuthSession
  16. 8 attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope
  17. 8 def initialize(
  18. issuer:,
  19. client_id:,
  20. client_secret:,
  21. access_token: nil,
  22. refresh_token: nil,
  23. scope: nil,
  24. token_endpoint: nil,
  25. response_type: nil,
  26. grant_type: nil,
  27. token_endpoint_auth_method: nil
  28. )
  29. 128 @issuer = URI(issuer)
  30. 128 @client_id = client_id
  31. 128 @client_secret = client_secret
  32. 128 @token_endpoint = URI(token_endpoint) if token_endpoint
  33. 128 @response_type = response_type
  34. 128 @scope = case scope
  35. when String
  36. 48 scope.split
  37. when Array
  38. 32 scope
  39. end
  40. 128 @access_token = access_token
  41. 128 @refresh_token = refresh_token
  42. 128 @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
  43. 128 @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
  44. 128 unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
  45. 16 raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
  46. end
  47. 112 return if SUPPORTED_GRANT_TYPES.include?(@grant_type)
  48. 16 raise Error, "#{@grant_type} is not a supported grant type"
  49. end
  50. 8 def token_endpoint
  51. 112 @token_endpoint || "#{@issuer}/token"
  52. end
  53. 8 def token_endpoint_auth_method
  54. 160 @token_endpoint_auth_method || "client_secret_basic"
  55. end
  56. 8 def load(http)
  57. 48 return if @grant_type && @scope
  58. 16 metadata = http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json
  59. 16 @token_endpoint = metadata["token_endpoint"]
  60. 16 @scope = metadata["scopes_supported"]
  61. 64 @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
  62. 16 @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
  63. 16 SUPPORTED_AUTH_METHODS.include?(am)
  64. end
  65. 6 nil
  66. end
  67. 8 def merge(other)
  68. 96 obj = dup
  69. 84 case other
  70. when OAuthSession
  71. 48 other.instance_variables.each do |ivar|
  72. 432 val = other.instance_variable_get(ivar)
  73. 432 next unless val
  74. 336 obj.instance_variable_set(ivar, val)
  75. end
  76. when Hash
  77. 48 other.each do |k, v|
  78. 96 obj.instance_variable_set(:"@#{k}", v) if obj.instance_variable_defined?(:"@#{k}")
  79. end
  80. end
  81. 96 obj
  82. end
  83. end
  84. 8 module OptionsMethods
  85. 8 def option_oauth_session(value)
  86. 266 case value
  87. when Hash
  88. 16 OAuthSession.new(**value)
  89. when OAuthSession
  90. 272 value
  91. else
  92. 16 raise TypeError, ":oauth_session must be a #{OAuthSession}"
  93. end
  94. end
  95. end
  96. 8 module InstanceMethods
  97. 8 def oauth_auth(**args)
  98. 112 with(oauth_session: OAuthSession.new(**args))
  99. end
  100. 8 def with_access_token
  101. 48 oauth_session = @options.oauth_session
  102. 48 oauth_session.load(self)
  103. 48 grant_type = oauth_session.grant_type
  104. 48 headers = {}
  105. 48 form_post = { "grant_type" => grant_type, "scope" => Array(oauth_session.scope).join(" ") }.compact
  106. # auth
  107. 42 case oauth_session.token_endpoint_auth_method
  108. when "client_secret_post"
  109. 14 form_post["client_id"] = oauth_session.client_id
  110. 14 form_post["client_secret"] = oauth_session.client_secret
  111. when "client_secret_basic"
  112. 28 headers["authorization"] = Authentication::Basic.new(oauth_session.client_id, oauth_session.client_secret).authenticate
  113. end
  114. 42 case grant_type
  115. when "client_credentials"
  116. # do nothing
  117. when "refresh_token"
  118. 14 form_post["refresh_token"] = oauth_session.refresh_token
  119. end
  120. 48 token_request = build_request("POST", oauth_session.token_endpoint, headers: headers, form: form_post)
  121. 48 token_request.headers.delete("authorization") unless oauth_session.token_endpoint_auth_method == "client_secret_basic"
  122. 48 token_response = request(token_request)
  123. 48 token_response.raise_for_status
  124. 48 payload = token_response.json
  125. 48 access_token = payload["access_token"]
  126. 48 refresh_token = payload["refresh_token"]
  127. 48 with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
  128. end
  129. 8 def build_request(*)
  130. 128 request = super
  131. 128 return request if request.headers.key?("authorization")
  132. 96 oauth_session = @options.oauth_session
  133. 96 return request unless oauth_session && oauth_session.access_token
  134. 56 request.headers["authorization"] = "Bearer #{oauth_session.access_token}"
  135. 64 request
  136. end
  137. end
  138. end
  139. 8 register_plugin :oauth, OAuth
  140. end
  141. end

lib/httpx/plugins/persistent.rb

100.0% lines covered

28 relevant lines. 28 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 17 module HTTPX
  3. 17 module Plugins
  4. # This plugin implements a session that persists connections over the duration of the process.
  5. #
  6. # This will improve connection reuse in a long-running process.
  7. #
  8. # One important caveat to note is, although this session might not close connections,
  9. # other sessions from the same process that don't have this plugin turned on might.
  10. #
  11. # This session will still be able to work with it, as if, when expecting a connection
  12. # terminated by a different session, it will just retry on a new one and keep it open.
  13. #
  14. # This plugin is also not recommendable when connecting to >9000 (like, a lot) different origins.
  15. # So when you use this, make sure that you don't fall into this trap.
  16. #
  17. # https://gitlab.com/os85/httpx/wikis/Persistent
  18. #
  19. 17 module Persistent
  20. 17 def self.load_dependencies(klass)
  21. 499 max_retries = if klass.default_options.respond_to?(:max_retries)
  22. 8 [klass.default_options.max_retries, 1].max
  23. else
  24. 491 1
  25. end
  26. 499 klass.plugin(:retries, max_retries: max_retries)
  27. end
  28. 17 def self.extra_options(options)
  29. 499 options.merge(persistent: true)
  30. end
  31. 17 module InstanceMethods
  32. 17 private
  33. 17 def repeatable_request?(request, _)
  34. 519 super || begin
  35. 196 response = request.response
  36. 196 return false unless response && response.is_a?(ErrorResponse)
  37. 28 error = response.error
  38. 308 Retries::RECONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
  39. end
  40. end
  41. 17 def retryable_error?(ex)
  42. 77 super &&
  43. # under the persistent plugin rules, requests are only retried for connection related errors,
  44. # which do not include request timeout related errors. This only gets overriden if the end user
  45. # manually changed +:max_retries+ to something else, which means it is aware of the
  46. # consequences.
  47. 63 (!ex.is_a?(RequestTimeoutError) || @options.max_retries != 1)
  48. end
  49. 17 def get_current_selector
  50. 491 super(&nil) || begin
  51. 475 return unless block_given?
  52. 475 default = yield
  53. 475 set_current_selector(default)
  54. 475 default
  55. end
  56. end
  57. end
  58. end
  59. 17 register_plugin :persistent, Persistent
  60. end
  61. end

lib/httpx/plugins/proxy.rb

98.01% lines covered

151 relevant lines. 148 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 class HTTPProxyError < ConnectionError; end
  4. 10 module Plugins
  5. #
  6. # This plugin adds support for proxies. It ships with support for:
  7. #
  8. # * HTTP proxies
  9. # * HTTPS proxies
  10. # * Socks4/4a proxies
  11. # * Socks5 proxies
  12. #
  13. # https://gitlab.com/os85/httpx/wikis/Proxy
  14. #
  15. 10 module Proxy
  16. 10 Error = HTTPProxyError
  17. 10 PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
  18. 10 class << self
  19. 10 def configure(klass)
  20. 337 klass.plugin(:"proxy/http")
  21. 337 klass.plugin(:"proxy/socks4")
  22. 337 klass.plugin(:"proxy/socks5")
  23. end
  24. 10 def extra_options(options)
  25. 337 options.merge(supported_proxy_protocols: [])
  26. end
  27. end
  28. 10 class Parameters
  29. 10 attr_reader :uri, :username, :password, :scheme, :no_proxy
  30. 10 def initialize(uri: nil, scheme: nil, username: nil, password: nil, no_proxy: nil, **extra)
  31. 371 @no_proxy = Array(no_proxy) if no_proxy
  32. 371 @uris = Array(uri)
  33. 371 uri = @uris.first
  34. 371 @username = username
  35. 371 @password = password
  36. 371 @ns = 0
  37. 371 if uri
  38. 331 @uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
  39. 331 @username ||= @uri.user
  40. 331 @password ||= @uri.password
  41. end
  42. 371 @scheme = scheme
  43. 371 return unless @uri && @username && @password
  44. 210 @authenticator = nil
  45. 210 @scheme ||= infer_default_auth_scheme(@uri)
  46. 210 return unless @scheme
  47. 162 @authenticator = load_authenticator(@scheme, @username, @password, **extra)
  48. end
  49. 10 def shift
  50. # TODO: this operation must be synchronized
  51. 105 @ns += 1
  52. 120 @uri = @uris[@ns]
  53. 120 return unless @uri
  54. 16 @uri = URI(@uri) unless @uri.is_a?(URI::Generic)
  55. 16 scheme = infer_default_auth_scheme(@uri)
  56. 16 return unless scheme != @scheme
  57. 16 @scheme = scheme
  58. 16 @username = username || @uri.user
  59. 16 @password = password || @uri.password
  60. 16 @authenticator = load_authenticator(scheme, @username, @password)
  61. end
  62. 10 def can_authenticate?(*args)
  63. 184 return false unless @authenticator
  64. 64 @authenticator.can_authenticate?(*args)
  65. end
  66. 10 def authenticate(*args)
  67. 159 return unless @authenticator
  68. 159 @authenticator.authenticate(*args)
  69. end
  70. 10 def ==(other)
  71. 384 case other
  72. when Parameters
  73. 396 @uri == other.uri &&
  74. @username == other.username &&
  75. @password == other.password &&
  76. @scheme == other.scheme
  77. when URI::Generic, String
  78. 24 proxy_uri = @uri.dup
  79. 24 proxy_uri.user = @username
  80. 24 proxy_uri.password = @password
  81. 24 other_uri = other.is_a?(URI::Generic) ? other : URI.parse(other)
  82. 24 proxy_uri == other_uri
  83. else
  84. 16 super
  85. end
  86. end
  87. 10 private
  88. 10 def infer_default_auth_scheme(uri)
  89. 185 case uri.scheme
  90. when "socks5"
  91. 48 uri.scheme
  92. when "http", "https"
  93. 101 "basic"
  94. end
  95. end
  96. 10 def load_authenticator(scheme, username, password, **extra)
  97. 178 auth_scheme = scheme.to_s.capitalize
  98. 178 require_relative "auth/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
  99. 178 Authentication.const_get(auth_scheme).new(username, password, **extra)
  100. end
  101. end
  102. # adds support for the following options:
  103. #
  104. # :proxy :: proxy options defining *:uri*, *:username*, *:password* or
  105. # *:scheme* (i.e. <tt>{ uri: "http://proxy" }</tt>)
  106. 10 module OptionsMethods
  107. 10 def option_proxy(value)
  108. 672 value.is_a?(Parameters) ? value : Parameters.new(**Hash[value])
  109. end
  110. 10 def option_supported_proxy_protocols(value)
  111. 1699 raise TypeError, ":supported_proxy_protocols must be an Array" unless value.is_a?(Array)
  112. 1699 value.map(&:to_s)
  113. end
  114. end
  115. 10 module InstanceMethods
  116. 10 def find_connection(request_uri, selector, options)
  117. 417 return super unless options.respond_to?(:proxy)
  118. 417 if (next_proxy = request_uri.find_proxy)
  119. 4 return super(request_uri, selector, options.merge(proxy: Parameters.new(uri: next_proxy)))
  120. end
  121. 413 proxy = options.proxy
  122. 413 return super unless proxy
  123. 403 next_proxy = proxy.uri
  124. 403 raise Error, "Failed to connect to proxy" unless next_proxy
  125. 1 raise Error,
  126. 387 "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
  127. 379 if (no_proxy = proxy.no_proxy)
  128. 16 no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
  129. # TODO: setting proxy to nil leaks the connection object in the pool
  130. 16 return super(request_uri, selector, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(request_uri.host, next_proxy.host,
  131. next_proxy.port, no_proxy)
  132. end
  133. 371 super(request_uri, selector, options.merge(proxy: proxy))
  134. end
  135. 10 private
  136. 10 def fetch_response(request, selector, options)
  137. 1555 response = super
  138. 1555 if response.is_a?(ErrorResponse) && proxy_error?(request, response, options)
  139. 120 options.proxy.shift
  140. # return last error response if no more proxies to try
  141. 120 return response if options.proxy.uri.nil?
  142. 16 log { "failed connecting to proxy, trying next..." }
  143. 16 request.transition(:idle)
  144. 16 send_request(request, selector, options)
  145. 14 return
  146. end
  147. 1435 response
  148. end
  149. 10 def proxy_error?(_request, response, options)
  150. 169 return false unless options.proxy
  151. 168 error = response.error
  152. 147 case error
  153. when NativeResolveError
  154. 16 proxy_uri = URI(options.proxy.uri)
  155. 16 peer = error.connection.peer
  156. # failed resolving proxy domain
  157. 16 peer.host == proxy_uri.host && peer.port == proxy_uri.port
  158. when ResolveError
  159. proxy_uri = URI(options.proxy.uri)
  160. error.message.end_with?(proxy_uri.to_s)
  161. when *PROXY_ERRORS
  162. # timeout errors connecting to proxy
  163. 152 true
  164. else
  165. false
  166. end
  167. end
  168. end
  169. 10 module ConnectionMethods
  170. 10 using URIExtensions
  171. 10 def initialize(*)
  172. 384 super
  173. 384 return unless @options.proxy
  174. # redefining the connection origin as the proxy's URI,
  175. # as this will be used as the tcp peer ip.
  176. 366 @proxy_uri = URI(@options.proxy.uri)
  177. end
  178. 10 def peer
  179. 954 @proxy_uri || super
  180. end
  181. 10 def connecting?
  182. 4532 return super unless @options.proxy
  183. 4369 super || @state == :connecting || @state == :connected
  184. end
  185. 10 def call
  186. 1102 super
  187. 1102 return unless @options.proxy
  188. 969 case @state
  189. when :connecting
  190. 291 consume
  191. end
  192. end
  193. 10 def reset
  194. 384 return super unless @options.proxy
  195. 367 @state = :open
  196. 367 super
  197. # emit(:close)
  198. end
  199. 10 private
  200. 10 def initialize_type(uri, options)
  201. 384 return super unless options.proxy
  202. 366 "tcp"
  203. end
  204. 10 def connect
  205. 1042 return super unless @options.proxy
  206. 892 case @state
  207. when :idle
  208. 712 transition(:connecting)
  209. when :connected
  210. 296 transition(:open)
  211. end
  212. end
  213. 10 def handle_transition(nextstate)
  214. 2117 return super unless @options.proxy
  215. 1792 case nextstate
  216. when :closing
  217. # this is a hack so that we can use the super method
  218. # and it'll think that the current state is open
  219. 375 @state = :open if @state == :connecting
  220. end
  221. 2030 super
  222. end
  223. end
  224. end
  225. 10 register_plugin :proxy, Proxy
  226. end
  227. 10 class ProxySSL < SSL
  228. 10 def initialize(tcp, request_uri, options)
  229. 89 @io = tcp.to_io
  230. 89 super(request_uri, tcp.addresses, options)
  231. 89 @hostname = request_uri.host
  232. 89 @state = :connected
  233. end
  234. end
  235. end

lib/httpx/plugins/proxy/http.rb

100.0% lines covered

104 relevant lines. 104 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 module Plugins
  4. 10 module Proxy
  5. 10 module HTTP
  6. 10 class << self
  7. 10 def extra_options(options)
  8. 337 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[http])
  9. end
  10. end
  11. 10 module InstanceMethods
  12. 10 def with_proxy_basic_auth(opts)
  13. 8 with(proxy: opts.merge(scheme: "basic"))
  14. end
  15. 10 def with_proxy_digest_auth(opts)
  16. 24 with(proxy: opts.merge(scheme: "digest"))
  17. end
  18. 10 def with_proxy_ntlm_auth(opts)
  19. 8 with(proxy: opts.merge(scheme: "ntlm"))
  20. end
  21. 10 def fetch_response(request, selector, options)
  22. 1555 response = super
  23. 1555 if response &&
  24. response.is_a?(Response) &&
  25. response.status == 407 &&
  26. !request.headers.key?("proxy-authorization") &&
  27. response.headers.key?("proxy-authenticate") && options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
  28. 8 request.transition(:idle)
  29. 7 request.headers["proxy-authorization"] =
  30. options.proxy.authenticate(request, response.headers["proxy-authenticate"])
  31. 8 send_request(request, selector, options)
  32. 7 return
  33. end
  34. 1547 response
  35. end
  36. end
  37. 10 module ConnectionMethods
  38. 10 def connecting?
  39. 4532 super || @state == :connecting || @state == :connected
  40. end
  41. 10 private
  42. 10 def handle_transition(nextstate)
  43. 2356 return super unless @options.proxy && @options.proxy.uri.scheme == "http"
  44. 1020 case nextstate
  45. when :connecting
  46. 304 return unless @state == :idle
  47. 304 @io.connect
  48. 304 return unless @io.connected?
  49. 152 @parser || begin
  50. 144 @parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
  51. 144 parser = @parser
  52. 144 parser.extend(ProxyParser)
  53. 144 parser.on(:response, &method(:__http_on_connect))
  54. 144 parser.on(:close) do |force|
  55. 57 next unless @parser
  56. 8 if force
  57. 8 reset
  58. 8 emit(:terminate)
  59. end
  60. end
  61. 144 parser.on(:reset) do
  62. 16 if parser.empty?
  63. 8 reset
  64. else
  65. 8 transition(:closing)
  66. 8 transition(:closed)
  67. 8 parser.reset if @parser
  68. 8 transition(:idle)
  69. 8 transition(:connecting)
  70. end
  71. end
  72. 144 __http_proxy_connect(parser)
  73. end
  74. 152 return if @state == :connected
  75. when :connected
  76. 136 return unless @state == :idle || @state == :connecting
  77. 121 case @state
  78. when :connecting
  79. 49 parser = @parser
  80. 49 @parser = nil
  81. 49 parser.close
  82. when :idle
  83. 87 @parser.callbacks.clear
  84. 87 set_parser_callbacks(@parser)
  85. end
  86. end
  87. 906 super
  88. end
  89. 10 def __http_proxy_connect(parser)
  90. 144 req = @pending.first
  91. 144 if req && req.uri.scheme == "https"
  92. # if the first request after CONNECT is to an https address, it is assumed that
  93. # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
  94. # and therefore, will share the connection.
  95. #
  96. 57 connect_request = ConnectRequest.new(req.uri, @options)
  97. 51 @inflight += 1
  98. 57 parser.send(connect_request)
  99. else
  100. 87 handle_transition(:connected)
  101. end
  102. end
  103. 10 def __http_on_connect(request, response)
  104. 58 @inflight -= 1
  105. 65 if response.is_a?(Response) && response.status == 200
  106. 49 req = @pending.first
  107. 49 request_uri = req.uri
  108. 49 @io = ProxySSL.new(@io, request_uri, @options)
  109. 49 transition(:connected)
  110. 49 throw(:called)
  111. 15 elsif response.is_a?(Response) &&
  112. response.status == 407 &&
  113. !request.headers.key?("proxy-authorization") &&
  114. @options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
  115. 8 request.transition(:idle)
  116. 7 request.headers["proxy-authorization"] = @options.proxy.authenticate(request, response.headers["proxy-authenticate"])
  117. 8 @parser.send(request)
  118. 7 @inflight += 1
  119. else
  120. 8 pending = @pending + @parser.pending
  121. 21 while (req = pending.shift)
  122. 8 response.finish!
  123. 8 req.response = response
  124. 8 req.emit(:response, response)
  125. end
  126. 8 reset
  127. end
  128. end
  129. end
  130. 10 module ProxyParser
  131. 10 def join_headline(request)
  132. 144 return super if request.verb == "CONNECT"
  133. 70 "#{request.verb} #{request.uri} HTTP/#{@version.join(".")}"
  134. end
  135. 10 def set_protocol_headers(request)
  136. 152 extra_headers = super
  137. 152 proxy_params = @options.proxy
  138. 152 if proxy_params.scheme == "basic"
  139. # opt for basic auth
  140. 85 extra_headers["proxy-authorization"] = proxy_params.authenticate(extra_headers)
  141. end
  142. 152 extra_headers["proxy-connection"] = extra_headers.delete("connection") if extra_headers.key?("connection")
  143. 152 extra_headers
  144. end
  145. end
  146. 10 class ConnectRequest < Request
  147. 10 def initialize(uri, options)
  148. 57 super("CONNECT", uri, options)
  149. 57 @headers.delete("accept")
  150. end
  151. 10 def path
  152. 65 "#{@uri.hostname}:#{@uri.port}"
  153. end
  154. end
  155. end
  156. end
  157. 10 register_plugin :"proxy/http", Proxy::HTTP
  158. end
  159. end

lib/httpx/plugins/proxy/socks4.rb

97.47% lines covered

79 relevant lines. 77 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 require "resolv"
  3. 10 require "ipaddr"
  4. 10 module HTTPX
  5. 10 class Socks4Error < HTTPProxyError; end
  6. 10 module Plugins
  7. 10 module Proxy
  8. 10 module Socks4
  9. 10 VERSION = 4
  10. 10 CONNECT = 1
  11. 10 GRANTED = 0x5A
  12. 10 PROTOCOLS = %w[socks4 socks4a].freeze
  13. 10 Error = Socks4Error
  14. 10 class << self
  15. 10 def extra_options(options)
  16. 337 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + PROTOCOLS)
  17. end
  18. end
  19. 10 module ConnectionMethods
  20. 10 def interests
  21. 3843 if @state == :connecting
  22. return @write_buffer.empty? ? :r : :w
  23. end
  24. 3843 super
  25. end
  26. 10 private
  27. 10 def handle_transition(nextstate)
  28. 2420 return super unless @options.proxy && PROTOCOLS.include?(@options.proxy.uri.scheme)
  29. 383 case nextstate
  30. when :connecting
  31. 128 return unless @state == :idle
  32. 128 @io.connect
  33. 128 return unless @io.connected?
  34. 64 req = @pending.first
  35. 64 return unless req
  36. 64 request_uri = req.uri
  37. 64 @write_buffer << Packet.connect(@options.proxy, request_uri)
  38. 64 __socks4_proxy_connect
  39. when :connected
  40. 48 return unless @state == :connecting
  41. 48 @parser = nil
  42. end
  43. 373 log(level: 1) { "SOCKS4: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
  44. 373 super
  45. end
  46. 10 def __socks4_proxy_connect
  47. 64 @parser = SocksParser.new(@write_buffer, @options)
  48. 64 @parser.once(:packet, &method(:__socks4_on_packet))
  49. end
  50. 10 def __socks4_on_packet(packet)
  51. 64 _version, status, _port, _ip = packet.unpack("CCnN")
  52. 64 if status == GRANTED
  53. 48 req = @pending.first
  54. 48 request_uri = req.uri
  55. 48 @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
  56. 48 transition(:connected)
  57. 48 throw(:called)
  58. else
  59. 16 on_socks4_error("socks error: #{status}")
  60. end
  61. end
  62. 10 def on_socks4_error(message)
  63. 16 ex = Error.new(message)
  64. 16 ex.set_backtrace(caller)
  65. 16 on_error(ex)
  66. 16 throw(:called)
  67. end
  68. end
  69. 10 class SocksParser
  70. 10 include HTTPX::Callbacks
  71. 10 def initialize(buffer, options)
  72. 64 @buffer = buffer
  73. 64 @options = options
  74. end
  75. 10 def close; end
  76. 10 def consume(*); end
  77. 10 def empty?
  78. true
  79. end
  80. 10 def <<(packet)
  81. 64 emit(:packet, packet)
  82. end
  83. end
  84. 10 module Packet
  85. 10 module_function
  86. 10 def connect(parameters, uri)
  87. 64 packet = [VERSION, CONNECT, uri.port].pack("CCn")
  88. 56 case parameters.uri.scheme
  89. when "socks4"
  90. 48 socks_host = uri.host
  91. 5 begin
  92. 96 ip = IPAddr.new(socks_host)
  93. 48 packet << ip.hton
  94. rescue IPAddr::InvalidAddressError
  95. 48 socks_host = Resolv.getaddress(socks_host)
  96. 48 retry
  97. end
  98. 48 packet << [parameters.username].pack("Z*")
  99. when "socks4a"
  100. 16 packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
  101. end
  102. 64 packet
  103. end
  104. end
  105. end
  106. end
  107. 10 register_plugin :"proxy/socks4", Proxy::Socks4
  108. end
  109. end

lib/httpx/plugins/proxy/socks5.rb

99.12% lines covered

113 relevant lines. 112 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 class Socks5Error < HTTPProxyError; end
  4. 10 module Plugins
  5. 10 module Proxy
  6. 10 module Socks5
  7. 10 VERSION = 5
  8. 10 NOAUTH = 0
  9. 10 PASSWD = 2
  10. 10 NONE = 0xff
  11. 10 CONNECT = 1
  12. 10 IPV4 = 1
  13. 10 DOMAIN = 3
  14. 10 IPV6 = 4
  15. 10 SUCCESS = 0
  16. 10 Error = Socks5Error
  17. 10 class << self
  18. 10 def load_dependencies(*)
  19. 337 require_relative "../auth/socks5"
  20. end
  21. 10 def extra_options(options)
  22. 337 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[socks5])
  23. end
  24. end
  25. 10 module ConnectionMethods
  26. 10 def call
  27. 1102 super
  28. 1102 return unless @options.proxy && @options.proxy.uri.scheme == "socks5"
  29. 301 case @state
  30. when :connecting,
  31. :negotiating,
  32. :authenticating
  33. 146 consume
  34. end
  35. end
  36. 10 def connecting?
  37. 4532 super || @state == :authenticating || @state == :negotiating
  38. end
  39. 10 def interests
  40. 6363 if @state == :connecting || @state == :authenticating || @state == :negotiating
  41. 2310 return @write_buffer.empty? ? :r : :w
  42. end
  43. 3843 super
  44. end
  45. 10 private
  46. 10 def handle_transition(nextstate)
  47. 2708 return super unless @options.proxy && @options.proxy.uri.scheme == "socks5"
  48. 909 case nextstate
  49. when :connecting
  50. 288 return unless @state == :idle
  51. 288 @io.connect
  52. 288 return unless @io.connected?
  53. 144 @write_buffer << Packet.negotiate(@options.proxy)
  54. 144 __socks5_proxy_connect
  55. when :authenticating
  56. 48 return unless @state == :connecting
  57. 48 @write_buffer << Packet.authenticate(@options.proxy)
  58. when :negotiating
  59. 192 return unless @state == :connecting || @state == :authenticating
  60. 48 req = @pending.first
  61. 48 request_uri = req.uri
  62. 48 @write_buffer << Packet.connect(request_uri)
  63. when :connected
  64. 32 return unless @state == :negotiating
  65. 32 @parser = nil
  66. end
  67. 751 log(level: 1) { "SOCKS5: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
  68. 751 super
  69. end
  70. 10 def __socks5_proxy_connect
  71. 144 @parser = SocksParser.new(@write_buffer, @options)
  72. 144 @parser.on(:packet, &method(:__socks5_on_packet))
  73. 144 transition(:negotiating)
  74. end
  75. 10 def __socks5_on_packet(packet)
  76. 210 case @state
  77. when :connecting
  78. 144 version, method = packet.unpack("CC")
  79. 144 __socks5_check_version(version)
  80. 126 case method
  81. when PASSWD
  82. 48 transition(:authenticating)
  83. 18 nil
  84. when NONE
  85. 80 __on_socks5_error("no supported authorization methods")
  86. else
  87. 16 transition(:negotiating)
  88. end
  89. when :authenticating
  90. 48 _, status = packet.unpack("CC")
  91. 48 return transition(:negotiating) if status == SUCCESS
  92. 16 __on_socks5_error("socks authentication error: #{status}")
  93. when :negotiating
  94. 48 version, reply, = packet.unpack("CC")
  95. 48 __socks5_check_version(version)
  96. 48 __on_socks5_error("socks5 negotiation error: #{reply}") unless reply == SUCCESS
  97. 32 req = @pending.first
  98. 32 request_uri = req.uri
  99. 32 @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
  100. 32 transition(:connected)
  101. 32 throw(:called)
  102. end
  103. end
  104. 10 def __socks5_check_version(version)
  105. 192 __on_socks5_error("invalid SOCKS version (#{version})") if version != 5
  106. end
  107. 10 def __on_socks5_error(message)
  108. 112 ex = Error.new(message)
  109. 112 ex.set_backtrace(caller)
  110. 112 on_error(ex)
  111. 112 throw(:called)
  112. end
  113. end
  114. 10 class SocksParser
  115. 10 include HTTPX::Callbacks
  116. 10 def initialize(buffer, options)
  117. 144 @buffer = buffer
  118. 144 @options = options
  119. end
  120. 10 def close; end
  121. 10 def consume(*); end
  122. 10 def empty?
  123. true
  124. end
  125. 10 def <<(packet)
  126. 240 emit(:packet, packet)
  127. end
  128. end
  129. 10 module Packet
  130. 10 module_function
  131. 10 def negotiate(parameters)
  132. 144 methods = [NOAUTH]
  133. 144 methods << PASSWD if parameters.can_authenticate?
  134. 144 methods.unshift(methods.size)
  135. 144 methods.unshift(VERSION)
  136. 144 methods.pack("C*")
  137. end
  138. 10 def authenticate(parameters)
  139. 48 parameters.authenticate
  140. end
  141. 10 def connect(uri)
  142. 48 packet = [VERSION, CONNECT, 0].pack("C*")
  143. 5 begin
  144. 48 ip = IPAddr.new(uri.host)
  145. 16 ipcode = ip.ipv6? ? IPV6 : IPV4
  146. 16 packet << [ipcode].pack("C") << ip.hton
  147. rescue IPAddr::InvalidAddressError
  148. 32 packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
  149. end
  150. 48 packet << [uri.port].pack("n")
  151. 48 packet
  152. end
  153. end
  154. end
  155. end
  156. 10 register_plugin :"proxy/socks5", Proxy::Socks5
  157. end
  158. end

lib/httpx/plugins/proxy/ssh.rb

92.31% lines covered

52 relevant lines. 48 lines covered and 4 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "httpx/plugins/proxy"
  3. 6 module HTTPX
  4. 6 module Plugins
  5. 6 module Proxy
  6. 6 module SSH
  7. 6 class << self
  8. 6 def load_dependencies(*)
  9. 12 require "net/ssh/gateway"
  10. end
  11. end
  12. 6 module OptionsMethods
  13. 6 def option_proxy(value)
  14. 24 Hash[value]
  15. end
  16. end
  17. 6 module InstanceMethods
  18. 6 def request(*args, **options)
  19. 12 raise ArgumentError, "must perform at least one request" if args.empty?
  20. 12 requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
  21. 12 request = requests.first or return super
  22. 12 request_options = request.options
  23. 12 return super unless request_options.proxy
  24. 12 ssh_options = request_options.proxy
  25. 12 ssh_uris = ssh_options.delete(:uri)
  26. 12 ssh_uri = URI.parse(ssh_uris.shift)
  27. 12 return super unless ssh_uri.scheme == "ssh"
  28. 12 ssh_username = ssh_options.delete(:username)
  29. 12 ssh_options[:port] ||= ssh_uri.port || 22
  30. 12 if request_options.debug
  31. ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
  32. end
  33. 12 request_uri = URI(requests.first.uri)
  34. 12 @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
  35. begin
  36. 12 @_gateway.open(request_uri.host, request_uri.port) do |local_port|
  37. 12 io = build_gateway_socket(local_port, request_uri, request_options)
  38. 12 super(*args, **options.merge(io: io))
  39. end
  40. ensure
  41. 12 @_gateway.shutdown!
  42. end
  43. end
  44. 6 private
  45. 6 def build_gateway_socket(port, request_uri, options)
  46. 12 case request_uri.scheme
  47. when "https"
  48. 6 ctx = OpenSSL::SSL::SSLContext.new
  49. 6 ctx_options = SSL::TLS_OPTIONS.merge(options.ssl)
  50. 6 ctx.set_params(ctx_options) unless ctx_options.empty?
  51. 6 sock = TCPSocket.open("localhost", port)
  52. 6 io = OpenSSL::SSL::SSLSocket.new(sock, ctx)
  53. 6 io.hostname = request_uri.host
  54. 6 io.sync_close = true
  55. 6 io.connect
  56. 6 io.post_connection_check(request_uri.host) if ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
  57. 6 io
  58. when "http"
  59. 6 TCPSocket.open("localhost", port)
  60. else
  61. raise TypeError, "unexpected scheme: #{request_uri.scheme}"
  62. end
  63. end
  64. end
  65. 6 module ConnectionMethods
  66. # should not coalesce connections here, as the IP is the IP of the proxy
  67. 6 def coalescable?(*)
  68. return super unless @options.proxy
  69. false
  70. end
  71. end
  72. end
  73. end
  74. 6 register_plugin :"proxy/ssh", Proxy::SSH
  75. end
  76. end

lib/httpx/plugins/push_promise.rb

100.0% lines covered

41 relevant lines. 41 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for HTTP/2 Push responses.
  6. #
  7. # In order to benefit from this, requests are sent one at a time, so that
  8. # no push responses are received after corresponding request has been sent.
  9. #
  10. # https://gitlab.com/os85/httpx/wikis/Server-Push
  11. #
  12. 8 module PushPromise
  13. 8 def self.extra_options(options)
  14. 16 options.merge(http2_settings: { settings_enable_push: 1 },
  15. max_concurrent_requests: 1)
  16. end
  17. 8 module ResponseMethods
  18. 8 def pushed?
  19. 16 @__pushed
  20. end
  21. 8 def mark_as_pushed!
  22. 8 @__pushed = true
  23. end
  24. end
  25. 8 module InstanceMethods
  26. 8 private
  27. 8 def promise_headers
  28. 16 @promise_headers ||= {}
  29. end
  30. 8 def on_promise(parser, stream)
  31. 16 stream.on(:promise_headers) do |h|
  32. 16 __on_promise_request(parser, stream, h)
  33. end
  34. 16 stream.on(:headers) do |h|
  35. 8 __on_promise_response(parser, stream, h)
  36. end
  37. end
  38. 8 def __on_promise_request(parser, stream, h)
  39. 16 log(level: 1, color: :yellow) do
  40. skipped # :nocov:
  41. skipped h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
  42. skipped # :nocov:
  43. end
  44. 16 headers = @options.headers_class.new(h)
  45. 16 path = headers[":path"]
  46. 16 authority = headers[":authority"]
  47. 24 request = parser.pending.find { |r| r.authority == authority && r.path == path }
  48. 16 if request
  49. 8 request.merge_headers(headers)
  50. 7 promise_headers[stream] = request
  51. 8 parser.pending.delete(request)
  52. 7 parser.streams[request] = stream
  53. 8 request.transition(:done)
  54. else
  55. 8 stream.refuse
  56. end
  57. end
  58. 8 def __on_promise_response(parser, stream, h)
  59. 8 request = promise_headers.delete(stream)
  60. 8 return unless request
  61. 8 parser.__send__(:on_stream_headers, stream, request, h)
  62. 8 response = request.response
  63. 8 response.mark_as_pushed!
  64. 8 stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
  65. 8 stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
  66. end
  67. end
  68. end
  69. 8 register_plugin(:push_promise, PushPromise)
  70. end
  71. end

lib/httpx/plugins/query.rb

100.0% lines covered

15 relevant lines. 15 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for using the experimental QUERY HTTP method
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Query
  8. 8 module Query
  9. 8 def self.subplugins
  10. 3 {
  11. 20 retries: QueryRetries,
  12. }
  13. end
  14. 8 module InstanceMethods
  15. 8 def query(*uri, **options)
  16. 16 request("QUERY", uri, **options)
  17. end
  18. end
  19. 8 module QueryRetries
  20. 8 module InstanceMethods
  21. 8 private
  22. 8 def repeatable_request?(request, options)
  23. 24 super || request.verb == "QUERY"
  24. end
  25. end
  26. end
  27. end
  28. 8 register_plugin :query, Query
  29. end
  30. end

lib/httpx/plugins/rate_limiter.rb

100.0% lines covered

18 relevant lines. 18 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when the request:
  6. #
  7. # * is rate limited;
  8. # * when the server is unavailable (503);
  9. # * when a 3xx request comes with a "retry-after" value
  10. #
  11. # https://gitlab.com/os85/httpx/wikis/Rate-Limiter
  12. #
  13. 8 module RateLimiter
  14. 8 class << self
  15. 8 RATE_LIMIT_CODES = [429, 503].freeze
  16. 8 def configure(klass)
  17. 64 klass.plugin(:retries,
  18. retry_change_requests: true,
  19. 7 retry_on: method(:retry_on_rate_limited_response),
  20. retry_after: method(:retry_after_rate_limit))
  21. end
  22. 8 def retry_on_rate_limited_response(response)
  23. 128 return false unless response.is_a?(Response)
  24. 128 status = response.status
  25. 128 RATE_LIMIT_CODES.include?(status)
  26. end
  27. # Servers send the "Retry-After" header field to indicate how long the
  28. # user agent ought to wait before making a follow-up request. When
  29. # sent with a 503 (Service Unavailable) response, Retry-After indicates
  30. # how long the service is expected to be unavailable to the client.
  31. # When sent with any 3xx (Redirection) response, Retry-After indicates
  32. # the minimum time that the user agent is asked to wait before issuing
  33. # the redirected request.
  34. #
  35. 8 def retry_after_rate_limit(_, response)
  36. 64 return unless response.is_a?(Response)
  37. 64 retry_after = response.headers["retry-after"]
  38. 64 return unless retry_after
  39. 32 Utils.parse_retry_after(retry_after)
  40. end
  41. end
  42. end
  43. 8 register_plugin :rate_limiter, RateLimiter
  44. end
  45. end

lib/httpx/plugins/response_cache.rb

100.0% lines covered

143 relevant lines. 143 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when certain errors happen.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Response-Cache
  8. #
  9. 8 module ResponseCache
  10. 8 CACHEABLE_VERBS = %w[GET HEAD].freeze
  11. 8 CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
  12. 8 SUPPORTED_VARY_HEADERS = %w[accept accept-encoding accept-language cookie origin].sort.freeze
  13. 8 private_constant :CACHEABLE_VERBS
  14. 8 private_constant :CACHEABLE_STATUS_CODES
  15. 8 class << self
  16. 8 def load_dependencies(*)
  17. 224 require_relative "response_cache/store"
  18. 224 require_relative "response_cache/file_store"
  19. end
  20. # whether the +response+ can be stored in the response cache.
  21. # (i.e. has a cacheable body, does not contain directives prohibiting storage, etc...)
  22. 8 def cacheable_response?(response)
  23. 136 response.is_a?(Response) &&
  24. (
  25. 136 response.cache_control.nil? ||
  26. # TODO: !response.cache_control.include?("private") && is shared cache
  27. !response.cache_control.include?("no-store")
  28. ) &&
  29. CACHEABLE_STATUS_CODES.include?(response.status) &&
  30. # RFC 2616 13.4 - A response received with a status code of 200, 203, 206, 300, 301 or
  31. # 410 MAY be stored by a cache and used in reply to a subsequent
  32. # request, subject to the expiration mechanism, unless a cache-control
  33. # directive prohibits caching. However, a cache that does not support
  34. # the Range and Content-Range headers MUST NOT cache 206 (Partial
  35. # Content) responses.
  36. response.status != 206
  37. end
  38. # whether the +response+
  39. 8 def not_modified?(response)
  40. 168 response.is_a?(Response) && response.status == 304
  41. end
  42. 8 def extra_options(options)
  43. 224 options.merge(
  44. supported_vary_headers: SUPPORTED_VARY_HEADERS,
  45. response_cache_store: :store,
  46. )
  47. end
  48. end
  49. # adds support for the following options:
  50. #
  51. # :supported_vary_headers :: array of header values that will be considered for a "vary" header based cache validation
  52. # (defaults to {SUPPORTED_VARY_HEADERS}).
  53. # :response_cache_store :: object where cached responses are fetch from or stored in; defaults to <tt>:store</tt> (in-memory
  54. # cache), can be set to <tt>:file_store</tt> (file system cache store) as well, or any object which
  55. # abides by the Cache Store Interface
  56. #
  57. # The Cache Store Interface requires implementation of the following methods:
  58. #
  59. # * +#get(request) -> response or nil+
  60. # * +#set(request, response) -> void+
  61. # * +#clear() -> void+)
  62. #
  63. 8 module OptionsMethods
  64. 8 def option_response_cache_store(value)
  65. 329 case value
  66. when :store
  67. 240 Store.new
  68. when :file_store
  69. 16 FileStore.new
  70. else
  71. 120 value
  72. end
  73. end
  74. 8 def option_supported_vary_headers(value)
  75. 224 Array(value).sort
  76. end
  77. end
  78. 8 module InstanceMethods
  79. # wipes out all cached responses from the cache store.
  80. 8 def clear_response_cache
  81. 136 @options.response_cache_store.clear
  82. end
  83. 8 def build_request(*)
  84. 464 request = super
  85. 464 return request unless cacheable_request?(request)
  86. 448 prepare_cache(request)
  87. 448 request
  88. end
  89. 8 private
  90. 8 def send_request(request, *)
  91. 168 return request if request.response
  92. 152 super
  93. end
  94. 8 def fetch_response(request, *)
  95. 529 response = super
  96. 529 return unless response
  97. 168 if ResponseCache.not_modified?(response)
  98. 32 log { "returning cached response for #{request.uri}" }
  99. 32 response.copy_from_cached!
  100. 135 elsif request.cacheable_verb? && ResponseCache.cacheable_response?(response)
  101. 112 request.options.response_cache_store.set(request, response) unless response.cached?
  102. end
  103. 168 response
  104. end
  105. # will either assign a still-fresh cached response to +request+, or set up its HTTP
  106. # cache invalidation headers in case it's not fresh anymore.
  107. 8 def prepare_cache(request)
  108. 672 cached_response = request.options.response_cache_store.get(request)
  109. 672 return unless cached_response && match_by_vary?(request, cached_response)
  110. 304 cached_response.body.rewind
  111. 304 if cached_response.fresh?
  112. 64 cached_response = cached_response.dup
  113. 64 cached_response.mark_as_cached!
  114. 64 request.response = cached_response
  115. 64 request.emit(:response, cached_response)
  116. 56 return
  117. end
  118. 240 request.cached_response = cached_response
  119. 240 if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
  120. 32 request.headers.add("if-modified-since", last_modified)
  121. end
  122. 240 if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
  123. 176 request.headers.add("if-none-match", etag)
  124. end
  125. end
  126. 8 def cacheable_request?(request)
  127. 464 request.cacheable_verb? &&
  128. (
  129. 448 !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
  130. )
  131. end
  132. # whether the +response+ complies with the directives set by the +request+ "vary" header
  133. # (true when none is available).
  134. 8 def match_by_vary?(request, response)
  135. 304 vary = response.vary
  136. 304 return true unless vary
  137. 96 original_request = response.original_request
  138. 96 if vary == %w[*]
  139. 32 request.options.supported_vary_headers.each do |field|
  140. 160 return false unless request.headers[field] == original_request.headers[field]
  141. end
  142. 28 return true
  143. end
  144. 64 vary.all? do |field|
  145. 64 !original_request.headers.key?(field) || request.headers[field] == original_request.headers[field]
  146. end
  147. end
  148. end
  149. 8 module RequestMethods
  150. # points to a previously cached Response corresponding to this request.
  151. 8 attr_accessor :cached_response
  152. 8 def initialize(*)
  153. 624 super
  154. 624 @cached_response = nil
  155. end
  156. 8 def merge_headers(*)
  157. 296 super
  158. 296 @response_cache_key = nil
  159. end
  160. # returns whether this request is cacheable as per HTTP caching rules.
  161. 8 def cacheable_verb?
  162. 600 CACHEABLE_VERBS.include?(@verb)
  163. end
  164. # returns a unique cache key as a String identifying this request
  165. 8 def response_cache_key
  166. 1296 @response_cache_key ||= begin
  167. 472 keys = [@verb, @uri]
  168. 472 @options.supported_vary_headers.each do |field|
  169. 2360 value = @headers[field]
  170. 2360 keys << value if value
  171. end
  172. 472 Digest::SHA1.hexdigest("httpx-response-cache-#{keys.join("-")}")
  173. end
  174. end
  175. end
  176. 8 module ResponseMethods
  177. 8 attr_writer :original_request
  178. 8 def initialize(*)
  179. 504 super
  180. 504 @cached = false
  181. end
  182. # a copy of the request this response was originally cached from
  183. 8 def original_request
  184. 96 @original_request || @request
  185. end
  186. # whether this Response was duplicated from a previously {RequestMethods#cached_response}.
  187. 8 def cached?
  188. 112 @cached
  189. end
  190. # sets this Response as being duplicated from a previously cached response.
  191. 8 def mark_as_cached!
  192. 64 @cached = true
  193. end
  194. # eager-copies the response headers and body from {RequestMethods#cached_response}.
  195. 8 def copy_from_cached!
  196. 32 cached_response = @request.cached_response
  197. 32 return unless cached_response
  198. # 304 responses do not have content-type, which are needed for decoding.
  199. 32 @headers = @headers.class.new(cached_response.headers.merge(@headers))
  200. 32 @body = cached_response.body.dup
  201. 32 @body.rewind
  202. end
  203. # A response is fresh if its age has not yet exceeded its freshness lifetime.
  204. # other (#cache_control} directives may influence the outcome, as per the rules
  205. # from the {rfc}[https://www.rfc-editor.org/rfc/rfc7234]
  206. 8 def fresh?
  207. 304 if cache_control
  208. 112 return false if cache_control.include?("no-cache")
  209. 80 return true if cache_control.include?("immutable")
  210. # check age: max-age
  211. 192 max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
  212. 192 max_age ||= cache_control.find { |directive| directive.start_with?("max-age") }
  213. 80 max_age = max_age[/age=(\d+)/, 1] if max_age
  214. 80 max_age = max_age.to_i if max_age
  215. 80 return max_age > age if max_age
  216. end
  217. # check age: expires
  218. 192 if @headers.key?("expires")
  219. 5 begin
  220. 48 expires = Time.httpdate(@headers["expires"])
  221. rescue ArgumentError
  222. 16 return false
  223. end
  224. 28 return (expires - Time.now).to_i.positive?
  225. end
  226. 144 false
  227. end
  228. # returns the "cache-control" directives as an Array of String(s).
  229. 8 def cache_control
  230. 848 return @cache_control if defined?(@cache_control)
  231. 48 @cache_control = begin
  232. 384 return unless @headers.key?("cache-control")
  233. 112 @headers["cache-control"].split(/ *, */)
  234. end
  235. end
  236. # returns the "vary" header value as an Array of (String) headers.
  237. 8 def vary
  238. 304 return @vary if defined?(@vary)
  239. 34 @vary = begin
  240. 272 return unless @headers.key?("vary")
  241. 64 @headers["vary"].split(/ *, */).map(&:downcase)
  242. end
  243. end
  244. 8 private
  245. # returns the value of the "age" header as an Integer (time since epoch).
  246. # if no "age" of header exists, it returns the number of seconds since {#date}.
  247. 8 def age
  248. 80 return @headers["age"].to_i if @headers.key?("age")
  249. 80 (Time.now - date).to_i
  250. end
  251. # returns the value of the "date" header as a Time object
  252. 8 def date
  253. 80 @date ||= Time.httpdate(@headers["date"])
  254. rescue NoMethodError, ArgumentError
  255. 16 Time.now
  256. end
  257. end
  258. end
  259. 8 register_plugin :response_cache, ResponseCache
  260. end
  261. end

lib/httpx/plugins/response_cache/file_store.rb

100.0% lines covered

72 relevant lines. 72 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "pathname"
  3. 8 module HTTPX::Plugins
  4. 8 module ResponseCache
  5. # Implementation of a file system based cache store.
  6. #
  7. # It stores cached responses in a file under a directory pointed by the +dir+
  8. # variable (defaults to the default temp directory from the OS), in a custom
  9. # format (similar but different from HTTP/1.1 request/response framing).
  10. 8 class FileStore
  11. 8 CRLF = HTTPX::Connection::HTTP1::CRLF
  12. 8 attr_reader :dir
  13. 8 def initialize(dir = Dir.tmpdir)
  14. 80 @dir = Pathname.new(dir).join("httpx-response-cache")
  15. 80 FileUtils.mkdir_p(@dir)
  16. end
  17. 8 def clear
  18. 64 FileUtils.rm_rf(@dir)
  19. end
  20. 8 def get(request)
  21. 296 path = file_path(request)
  22. 296 return unless File.exist?(path)
  23. 152 File.open(path, mode: File::RDONLY | File::BINARY) do |f|
  24. 152 f.flock(File::Constants::LOCK_SH)
  25. 152 read_from_file(request, f)
  26. end
  27. end
  28. 8 def set(request, response)
  29. 96 path = file_path(request)
  30. 96 file_exists = File.exist?(path)
  31. 96 mode = file_exists ? File::RDWR : File::CREAT | File::Constants::WRONLY
  32. 96 File.open(path, mode: mode | File::BINARY) do |f|
  33. 96 f.flock(File::Constants::LOCK_EX)
  34. 96 if file_exists
  35. 8 cached_response = read_from_file(request, f)
  36. 8 if cached_response
  37. 8 next if cached_response == request.cached_response
  38. 8 cached_response.close
  39. 8 f.truncate(0)
  40. 8 f.rewind
  41. end
  42. end
  43. # cache the request headers
  44. 96 f << request.verb << CRLF
  45. 96 f << request.uri << CRLF
  46. 96 request.headers.each do |field, value|
  47. 288 f << field << ":" << value << CRLF
  48. end
  49. 96 f << CRLF
  50. # cache the response
  51. 96 f << response.status << CRLF
  52. 96 f << response.version << CRLF
  53. 96 response.headers.each do |field, value|
  54. 264 f << field << ":" << value << CRLF
  55. end
  56. 96 f << CRLF
  57. 96 response.body.rewind
  58. 96 ::IO.copy_stream(response.body, f)
  59. end
  60. end
  61. 8 private
  62. 8 def file_path(request)
  63. 392 @dir.join(request.response_cache_key)
  64. end
  65. 8 def read_from_file(request, f)
  66. # if it's an empty file
  67. 160 return if f.eof?
  68. # read request data
  69. 160 verb = f.readline.delete_suffix!(CRLF)
  70. 160 uri = f.readline.delete_suffix!(CRLF)
  71. 160 request_headers = {}
  72. 700 while (line = f.readline) != CRLF
  73. 480 line.delete_suffix!(CRLF)
  74. 480 sep_index = line.index(":")
  75. 480 field = line.byteslice(0..(sep_index - 1))
  76. 480 value = line.byteslice((sep_index + 1)..-1)
  77. 420 request_headers[field] = value
  78. end
  79. 160 status = f.readline.delete_suffix!(CRLF)
  80. 160 version = f.readline.delete_suffix!(CRLF)
  81. 160 response_headers = {}
  82. 665 while (line = f.readline) != CRLF
  83. 440 line.delete_suffix!(CRLF)
  84. 440 sep_index = line.index(":")
  85. 440 field = line.byteslice(0..(sep_index - 1))
  86. 440 value = line.byteslice((sep_index + 1)..-1)
  87. 385 response_headers[field] = value
  88. end
  89. 160 original_request = request.options.request_class.new(verb, uri, request.options)
  90. 160 original_request.merge_headers(request_headers)
  91. 160 response = request.options.response_class.new(request, status, version, response_headers)
  92. 160 response.original_request = original_request
  93. 160 response.finish!
  94. 160 ::IO.copy_stream(f, response.body)
  95. 160 response
  96. end
  97. end
  98. end
  99. end

lib/httpx/plugins/response_cache/store.rb

100.0% lines covered

16 relevant lines. 16 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX::Plugins
  3. 8 module ResponseCache
  4. # Implementation of a thread-safe in-memory cache store.
  5. 8 class Store
  6. 8 def initialize
  7. 296 @store = {}
  8. 296 @store_mutex = Thread::Mutex.new
  9. end
  10. 8 def clear
  11. 144 @store_mutex.synchronize { @store.clear }
  12. end
  13. 8 def get(request)
  14. 448 @store_mutex.synchronize do
  15. 448 @store[request.response_cache_key]
  16. end
  17. end
  18. 8 def set(request, response)
  19. 200 @store_mutex.synchronize do
  20. 200 cached_response = @store[request.response_cache_key]
  21. 200 cached_response.close if cached_response
  22. 175 @store[request.response_cache_key] = response
  23. end
  24. end
  25. end
  26. end
  27. end

lib/httpx/plugins/retries.rb

96.91% lines covered

97 relevant lines. 94 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 17 module HTTPX
  3. 17 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when errors happen.
  6. #
  7. # It has a default max number of retries (see *MAX_RETRIES* and the *max_retries* option),
  8. # after which it will return the last response, error or not. It will **not** raise an exception.
  9. #
  10. # It does not retry which are not considered idempotent (see *retry_change_requests* to override).
  11. #
  12. # https://gitlab.com/os85/httpx/wikis/Retries
  13. #
  14. 17 module Retries
  15. 17 MAX_RETRIES = 3
  16. # TODO: pass max_retries in a configure/load block
  17. 17 IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
  18. # subset of retryable errors which are safe to retry when reconnecting
  19. 2 RECONNECTABLE_ERRORS = [
  20. 15 IOError,
  21. EOFError,
  22. Errno::ECONNRESET,
  23. Errno::ECONNABORTED,
  24. Errno::EPIPE,
  25. Errno::EINVAL,
  26. Errno::ETIMEDOUT,
  27. ConnectionError,
  28. TLSError,
  29. Connection::HTTP2::Error,
  30. ].freeze
  31. 17 RETRYABLE_ERRORS = (RECONNECTABLE_ERRORS + [
  32. Parser::Error,
  33. TimeoutError,
  34. ]).freeze
  35. 17 DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
  36. 17 if ENV.key?("HTTPX_NO_JITTER")
  37. 16 def self.extra_options(options)
  38. 773 options.merge(max_retries: MAX_RETRIES)
  39. end
  40. else
  41. 1 def self.extra_options(options)
  42. 2 options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
  43. end
  44. end
  45. # adds support for the following options:
  46. #
  47. # :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
  48. # :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
  49. # :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
  50. # :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
  51. # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
  52. # (i.e. <tt>->(res) { ... }</tt>).
  53. 17 module OptionsMethods
  54. 17 def option_retry_after(value)
  55. # return early if callable
  56. 208 unless value.respond_to?(:call)
  57. 96 value = Float(value)
  58. 96 raise TypeError, ":retry_after must be positive" unless value.positive?
  59. end
  60. 208 value
  61. end
  62. 17 def option_retry_jitter(value)
  63. # return early if callable
  64. 54 raise TypeError, ":retry_jitter must be callable" unless value.respond_to?(:call)
  65. 54 value
  66. end
  67. 17 def option_max_retries(value)
  68. 2331 num = Integer(value)
  69. 2331 raise TypeError, ":max_retries must be positive" unless num >= 0
  70. 2331 num
  71. end
  72. 17 def option_retry_change_requests(v)
  73. 128 v
  74. end
  75. 17 def option_retry_on(value)
  76. 286 raise TypeError, ":retry_on must be called with the response" unless value.respond_to?(:call)
  77. 286 value
  78. end
  79. end
  80. 17 module InstanceMethods
  81. # returns a `:retries` plugin enabled session with +n+ maximum retries per request setting.
  82. 17 def max_retries(n)
  83. 96 with(max_retries: n)
  84. end
  85. 17 private
  86. 17 def fetch_response(request, selector, options)
  87. 6620125 response = super
  88. 6620125 if response &&
  89. request.retries.positive? &&
  90. repeatable_request?(request, options) &&
  91. (
  92. 96 (
  93. 331 response.is_a?(ErrorResponse) && retryable_error?(response.error)
  94. ) ||
  95. (
  96. 235 options.retry_on && options.retry_on.call(response)
  97. )
  98. )
  99. 487 try_partial_retry(request, response)
  100. 487 log { "failed to get response, #{request.retries} tries to go..." }
  101. 487 request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
  102. 487 request.transition(:idle)
  103. 487 retry_after = options.retry_after
  104. 487 retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
  105. 487 if retry_after
  106. # apply jitter
  107. 96 if (jitter = request.options.retry_jitter)
  108. 16 retry_after = jitter.call(retry_after)
  109. end
  110. 96 retry_start = Utils.now
  111. 96 log { "retrying after #{retry_after} secs..." }
  112. 96 selector.after(retry_after) do
  113. 96 if (response = request.response)
  114. response.finish!
  115. # request has terminated abruptly meanwhile
  116. request.emit(:response, response)
  117. else
  118. 96 log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
  119. 96 send_request(request, selector, options)
  120. end
  121. end
  122. else
  123. 391 send_request(request, selector, options)
  124. end
  125. 431 return
  126. end
  127. 6619638 response
  128. end
  129. # returns whether +request+ can be retried.
  130. 17 def repeatable_request?(request, options)
  131. 1069 IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
  132. end
  133. # returns whether the +ex+ exception happend for a retriable request.
  134. 17 def retryable_error?(ex)
  135. 3549 RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
  136. end
  137. 17 def proxy_error?(request, response, _)
  138. 64 super && !request.retries.positive?
  139. end
  140. #
  141. # Attempt to set the request to perform a partial range request.
  142. # This happens if the peer server accepts byte-range requests, and
  143. # the last response contains some body payload.
  144. #
  145. 17 def try_partial_retry(request, response)
  146. 487 response = response.response if response.is_a?(ErrorResponse)
  147. 487 return unless response
  148. 236 unless response.headers.key?("accept-ranges") &&
  149. response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
  150. 16 (original_body = response.body)
  151. 220 response.body.close
  152. 194 return
  153. end
  154. 16 request.partial_response = response
  155. 16 size = original_body.bytesize
  156. 14 request.headers["range"] = "bytes=#{size}-"
  157. end
  158. end
  159. 17 module RequestMethods
  160. # number of retries left.
  161. 17 attr_accessor :retries
  162. # a response partially received before.
  163. 17 attr_writer :partial_response
  164. # initializes the request instance, sets the number of retries for the request.
  165. 17 def initialize(*args)
  166. 807 super
  167. 807 @retries = @options.max_retries
  168. end
  169. 17 def response=(response)
  170. 1310 if @partial_response
  171. 16 if response.is_a?(Response) && response.status == 206
  172. 16 response.from_partial_response(@partial_response)
  173. else
  174. @partial_response.close
  175. end
  176. 16 @partial_response = nil
  177. end
  178. 1310 super
  179. end
  180. end
  181. 17 module ResponseMethods
  182. 17 def from_partial_response(response)
  183. 16 @status = response.status
  184. 16 @headers = response.headers
  185. 16 @body = response.body
  186. end
  187. end
  188. end
  189. 17 register_plugin :retries, Retries
  190. end
  191. end

lib/httpx/plugins/ssrf_filter.rb

100.0% lines covered

61 relevant lines. 61 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 class ServerSideRequestForgeryError < Error; end
  4. 8 module Plugins
  5. #
  6. # This plugin adds support for preventing Server-Side Request Forgery attacks.
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Server-Side-Request-Forgery-Filter
  9. #
  10. 8 module SsrfFilter
  11. 8 module IPAddrExtensions
  12. 8 refine IPAddr do
  13. 8 def prefixlen
  14. 128 mask_addr = @mask_addr
  15. 128 raise "Invalid mask" if mask_addr.zero?
  16. 387 mask_addr >>= 1 while (mask_addr & 0x1).zero?
  17. 128 length = 0
  18. 381 while mask_addr & 0x1 == 0x1
  19. 1771 length += 1
  20. 1771 mask_addr >>= 1
  21. end
  22. 128 length
  23. end
  24. end
  25. end
  26. 8 using IPAddrExtensions
  27. # https://en.wikipedia.org/wiki/Reserved_IP_addresses
  28. 2 IPV4_BLACKLIST = [
  29. 6 IPAddr.new("0.0.0.0/8"), # Current network (only valid as source address)
  30. IPAddr.new("10.0.0.0/8"), # Private network
  31. IPAddr.new("100.64.0.0/10"), # Shared Address Space
  32. IPAddr.new("127.0.0.0/8"), # Loopback
  33. IPAddr.new("169.254.0.0/16"), # Link-local
  34. IPAddr.new("172.16.0.0/12"), # Private network
  35. IPAddr.new("192.0.0.0/24"), # IETF Protocol Assignments
  36. IPAddr.new("192.0.2.0/24"), # TEST-NET-1, documentation and examples
  37. IPAddr.new("192.88.99.0/24"), # IPv6 to IPv4 relay (includes 2002::/16)
  38. IPAddr.new("192.168.0.0/16"), # Private network
  39. IPAddr.new("198.18.0.0/15"), # Network benchmark tests
  40. IPAddr.new("198.51.100.0/24"), # TEST-NET-2, documentation and examples
  41. IPAddr.new("203.0.113.0/24"), # TEST-NET-3, documentation and examples
  42. IPAddr.new("224.0.0.0/4"), # IP multicast (former Class D network)
  43. IPAddr.new("240.0.0.0/4"), # Reserved (former Class E network)
  44. IPAddr.new("255.255.255.255"), # Broadcast
  45. ].freeze
  46. 3 IPV6_BLACKLIST = ([
  47. 6 IPAddr.new("::1/128"), # Loopback
  48. IPAddr.new("64:ff9b::/96"), # IPv4/IPv6 translation (RFC 6052)
  49. IPAddr.new("100::/64"), # Discard prefix (RFC 6666)
  50. IPAddr.new("2001::/32"), # Teredo tunneling
  51. IPAddr.new("2001:10::/28"), # Deprecated (previously ORCHID)
  52. IPAddr.new("2001:20::/28"), # ORCHIDv2
  53. IPAddr.new("2001:db8::/32"), # Addresses used in documentation and example source code
  54. IPAddr.new("2002::/16"), # 6to4
  55. IPAddr.new("fc00::/7"), # Unique local address
  56. IPAddr.new("fe80::/10"), # Link-local address
  57. IPAddr.new("ff00::/8"), # Multicast
  58. ] + IPV4_BLACKLIST.flat_map do |ipaddr|
  59. 128 prefixlen = ipaddr.prefixlen
  60. 128 ipv4_compatible = ipaddr.ipv4_compat.mask(96 + prefixlen)
  61. 128 ipv4_mapped = ipaddr.ipv4_mapped.mask(80 + prefixlen)
  62. 128 [ipv4_compatible, ipv4_mapped]
  63. end).freeze
  64. 8 class << self
  65. 8 def extra_options(options)
  66. 70 options.merge(allowed_schemes: %w[https http])
  67. end
  68. 8 def unsafe_ip_address?(ipaddr)
  69. 126 range = ipaddr.to_range
  70. 126 return true if range.first != range.last
  71. 142 return IPV6_BLACKLIST.any? { |r| r.include?(ipaddr) } if ipaddr.ipv6?
  72. 1488 IPV4_BLACKLIST.any? { |r| r.include?(ipaddr) } # then it's IPv4
  73. end
  74. end
  75. # adds support for the following options:
  76. #
  77. # :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
  78. 8 module OptionsMethods
  79. 8 def option_allowed_schemes(value)
  80. 78 Array(value)
  81. end
  82. end
  83. 8 module InstanceMethods
  84. 8 def send_requests(*requests)
  85. 86 responses = requests.map do |request|
  86. 86 next if @options.allowed_schemes.include?(request.uri.scheme)
  87. 8 error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
  88. 8 error.set_backtrace(caller)
  89. 8 response = ErrorResponse.new(request, error)
  90. 8 request.emit(:response, response)
  91. 8 response
  92. end
  93. 172 allowed_requests = requests.select { |req| responses[requests.index(req)].nil? }
  94. 86 allowed_responses = super(*allowed_requests)
  95. 86 allowed_responses.each_with_index do |res, idx|
  96. 78 req = allowed_requests[idx]
  97. 68 responses[requests.index(req)] = res
  98. end
  99. 86 responses
  100. end
  101. end
  102. 8 module ConnectionMethods
  103. 8 def initialize(*)
  104. begin
  105. 78 super
  106. 8 rescue ServerSideRequestForgeryError => e
  107. # may raise when IPs are passed as options via :addresses
  108. 16 throw(:resolve_error, e)
  109. end
  110. end
  111. 8 def addresses=(addrs)
  112. 204 addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
  113. 78 addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
  114. 78 raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
  115. 16 super
  116. end
  117. end
  118. end
  119. 8 register_plugin :ssrf_filter, SsrfFilter
  120. end
  121. end

lib/httpx/plugins/stream.rb

97.78% lines covered

90 relevant lines. 88 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 14 module HTTPX
  3. 14 class StreamResponse
  4. 14 attr_reader :request
  5. 14 def initialize(request, session)
  6. 186 @request = request
  7. 186 @options = @request.options
  8. 186 @session = session
  9. 186 @response_enum = nil
  10. 186 @buffered_chunks = []
  11. end
  12. 14 def each(&block)
  13. 314 return enum_for(__method__) unless block
  14. 202 if (response_enum = @response_enum)
  15. 16 @response_enum = nil
  16. # streaming already started, let's finish it
  17. 42 while (chunk = @buffered_chunks.shift)
  18. 16 block.call(chunk)
  19. end
  20. # consume enum til the end
  21. 1 begin
  22. 55 while (chunk = response_enum.next)
  23. 29 block.call(chunk)
  24. end
  25. rescue StopIteration
  26. 16 return
  27. end
  28. end
  29. 186 @request.stream = self
  30. 17 begin
  31. 186 @on_chunk = block
  32. 186 response = @session.request(@request)
  33. 170 response.raise_for_status
  34. ensure
  35. 170 @on_chunk = nil
  36. end
  37. end
  38. 14 def each_line
  39. 108 return enum_for(__method__) unless block_given?
  40. 54 line = "".b
  41. 54 each do |chunk|
  42. 50 line << chunk
  43. 137 while (idx = line.index("\n"))
  44. 54 yield line.byteslice(0..idx - 1)
  45. 54 line = line.byteslice(idx + 1..-1)
  46. end
  47. end
  48. 22 yield line unless line.empty?
  49. end
  50. # This is a ghost method. It's to be used ONLY internally, when processing streams
  51. 14 def on_chunk(chunk)
  52. 356 raise NoMethodError unless @on_chunk
  53. 356 @on_chunk.call(chunk)
  54. end
  55. skipped # :nocov:
  56. skipped def inspect
  57. skipped "#<#{self.class}:#{object_id}>"
  58. skipped end
  59. skipped # :nocov:
  60. 14 def to_s
  61. 16 if @request.response
  62. @request.response.to_s
  63. else
  64. 16 @buffered_chunks.join
  65. end
  66. end
  67. 14 private
  68. 14 def response
  69. 356 @request.response || begin
  70. 32 response_enum = each
  71. 56 while (chunk = response_enum.next)
  72. 32 @buffered_chunks << chunk
  73. 32 break if @request.response
  74. end
  75. 32 @response_enum = response_enum
  76. 32 @request.response
  77. end
  78. end
  79. 14 def respond_to_missing?(meth, include_private)
  80. 24 if (response = @request.response)
  81. response.respond_to_missing?(meth, include_private)
  82. else
  83. 24 @options.response_class.method_defined?(meth) || (include_private && @options.response_class.private_method_defined?(meth))
  84. end || super
  85. end
  86. 14 def method_missing(meth, *args, **kwargs, &block)
  87. 178 return super unless response.respond_to?(meth)
  88. 178 response.__send__(meth, *args, **kwargs, &block)
  89. end
  90. end
  91. 14 module Plugins
  92. #
  93. # This plugin adds support for streaming a response (useful for i.e. "text/event-stream" payloads).
  94. #
  95. # https://gitlab.com/os85/httpx/wikis/Stream
  96. #
  97. 14 module Stream
  98. 14 def self.extra_options(options)
  99. 316 options.merge(timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 })
  100. end
  101. 14 module InstanceMethods
  102. 14 def request(*args, stream: false, **options)
  103. 506 return super(*args, **options) unless stream
  104. 202 requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
  105. 202 raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
  106. 186 request = requests.first
  107. 186 StreamResponse.new(request, self)
  108. end
  109. end
  110. 14 module RequestMethods
  111. 14 attr_accessor :stream
  112. end
  113. 14 module ResponseMethods
  114. 14 def stream
  115. 298 request = @request.root_request if @request.respond_to?(:root_request)
  116. 298 request ||= @request
  117. 298 request.stream
  118. end
  119. end
  120. 14 module ResponseBodyMethods
  121. 14 def initialize(*)
  122. 298 super
  123. 298 @stream = @response.stream
  124. end
  125. 14 def write(chunk)
  126. 495 return super unless @stream
  127. 409 return 0 if chunk.empty?
  128. 356 chunk = decode_chunk(chunk)
  129. 356 @stream.on_chunk(chunk.dup)
  130. 340 chunk.size
  131. end
  132. 14 private
  133. 14 def transition(*)
  134. 110 return if @stream
  135. 110 super
  136. end
  137. end
  138. end
  139. 14 register_plugin :stream, Stream
  140. end
  141. end

lib/httpx/plugins/stream_bidi.rb

99.3% lines covered

142 relevant lines. 141 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for bidirectional HTTP/2 streams.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/StreamBidi
  8. #
  9. # It is required that the request body allows chunk to be buffered, (i.e., responds to +#<<(chunk)+).
  10. 8 module StreamBidi
  11. # Extension of the Connection::HTTP2 class, which adds functionality to
  12. # deal with a request that can't be drained and must be interleaved with
  13. # the response streams.
  14. #
  15. # The streams keeps send DATA frames while there's data; when they're ain't,
  16. # the stream is kept open; it must be explicitly closed by the end user.
  17. #
  18. 8 class HTTP2Bidi < Connection::HTTP2
  19. 8 def initialize(*)
  20. 16 super
  21. 16 @lock = Thread::Mutex.new
  22. end
  23. 8 %i[close empty? exhausted? send <<].each do |lock_meth|
  24. 40 class_eval(<<-METH, __FILE__, __LINE__ + 1)
  25. 5 # lock.aware version of +#{lock_meth}+
  26. 5 def #{lock_meth}(*) # def close(*)
  27. return super if @lock.owned?
  28. # small race condition between
  29. # checking for ownership and
  30. # acquiring lock.
  31. # TODO: fix this at the parser.
  32. @lock.synchronize { super }
  33. end
  34. METH
  35. end
  36. 8 private
  37. 8 %i[join_headers join_trailers join_body].each do |lock_meth|
  38. 24 class_eval(<<-METH, __FILE__, __LINE__ + 1)
  39. 3 # lock.aware version of +#{lock_meth}+
  40. 3 def #{lock_meth}(*) # def join_headers(*)
  41. return super if @lock.owned?
  42. # small race condition between
  43. # checking for ownership and
  44. # acquiring lock.
  45. # TODO: fix this at the parser.
  46. @lock.synchronize { super }
  47. end
  48. METH
  49. end
  50. 8 def handle_stream(stream, request)
  51. 16 request.on(:body) do
  52. 96 next unless request.headers_sent
  53. 80 handle(request, stream)
  54. 80 emit(:flush_buffer)
  55. end
  56. 16 super
  57. end
  58. # when there ain't more chunks, it makes the buffer as full.
  59. 8 def send_chunk(request, stream, chunk, next_chunk)
  60. 96 super
  61. 96 return if next_chunk
  62. 96 request.transition(:waiting_for_chunk)
  63. 96 throw(:buffer_full)
  64. end
  65. # sets end-stream flag when the request is closed.
  66. 8 def end_stream?(request, next_chunk)
  67. 96 request.closed? && next_chunk.nil?
  68. end
  69. end
  70. # BidiBuffer is a Buffer which can be receive data from threads othr
  71. # than the thread of the corresponding Connection/Session.
  72. #
  73. # It synchronizes access to a secondary internal +@oob_buffer+, which periodically
  74. # is reconciled to the main internal +@buffer+.
  75. 8 class BidiBuffer < Buffer
  76. 8 def initialize(*)
  77. 16 super
  78. 16 @parent_thread = Thread.current
  79. 16 @oob_mutex = Thread::Mutex.new
  80. 16 @oob_buffer = "".b
  81. end
  82. # buffers the +chunk+ to be sent
  83. 8 def <<(chunk)
  84. 176 return super if Thread.current == @parent_thread
  85. 80 @oob_mutex.synchronize { @oob_buffer << chunk }
  86. end
  87. # reconciles the main and secondary buffer (which receives data from other threads).
  88. 8 def rebuffer
  89. 1026 raise Error, "can only rebuffer while waiting on a response" unless Thread.current == @parent_thread
  90. 1026 @oob_mutex.synchronize do
  91. 1026 @buffer << @oob_buffer
  92. 1026 @oob_buffer.clear
  93. end
  94. end
  95. end
  96. # Proxy to wake up the session main loop when one
  97. # of the connections has buffered data to write. It abides by the HTTPX::_Selectable API,
  98. # which allows it to be registered in the selector alongside actual HTTP-based
  99. # HTTPX::Connection objects.
  100. 8 class Signal
  101. 8 def initialize
  102. 16 @closed = false
  103. 16 @pipe_read, @pipe_write = ::IO.pipe
  104. end
  105. 8 def state
  106. 231 @closed ? :closed : :open
  107. end
  108. # noop
  109. 8 def log(**); end
  110. 8 def to_io
  111. 433 @pipe_read.to_io
  112. end
  113. 8 def wakeup
  114. 80 return if @closed
  115. 80 @pipe_write.write("\0")
  116. end
  117. 8 def call
  118. 71 return if @closed
  119. 71 @pipe_read.readpartial(1)
  120. end
  121. 8 def interests
  122. 231 return if @closed
  123. 231 :r
  124. end
  125. 8 def timeout; end
  126. 8 def terminate
  127. 16 @pipe_write.close
  128. 16 @pipe_read.close
  129. 16 @closed = true
  130. end
  131. # noop (the owner connection will take of it)
  132. 8 def handle_socket_timeout(interval); end
  133. end
  134. 8 class << self
  135. 8 def load_dependencies(klass)
  136. 16 klass.plugin(:stream)
  137. end
  138. 8 def extra_options(options)
  139. 16 options.merge(fallback_protocol: "h2")
  140. end
  141. end
  142. 8 module InstanceMethods
  143. 8 def initialize(*)
  144. 16 @signal = Signal.new
  145. 16 super
  146. end
  147. 8 def close(selector = Selector.new)
  148. 16 @signal.terminate
  149. 16 selector.deregister(@signal)
  150. 16 super(selector)
  151. end
  152. 8 def select_connection(connection, selector)
  153. 32 super
  154. 32 selector.register(@signal)
  155. 32 connection.signal = @signal
  156. end
  157. 8 def deselect_connection(connection, *)
  158. 16 super
  159. 16 connection.signal = nil
  160. end
  161. end
  162. # Adds synchronization to request operations which may buffer payloads from different
  163. # threads.
  164. 8 module RequestMethods
  165. 8 attr_accessor :headers_sent
  166. 8 def initialize(*)
  167. 16 super
  168. 16 @headers_sent = false
  169. 16 @closed = false
  170. 16 @mutex = Thread::Mutex.new
  171. end
  172. 8 def closed?
  173. 96 @closed
  174. end
  175. 8 def can_buffer?
  176. 214 super && @state != :waiting_for_chunk
  177. end
  178. # overrides state management transitions to introduce an intermediate
  179. # +:waiting_for_chunk+ state, which the request transitions to once payload
  180. # is buffered.
  181. 8 def transition(nextstate)
  182. 372 headers_sent = @headers_sent
  183. 322 case nextstate
  184. when :waiting_for_chunk
  185. 96 return unless @state == :body
  186. when :body
  187. 154 case @state
  188. when :headers
  189. 16 headers_sent = true
  190. when :waiting_for_chunk
  191. # HACK: to allow super to pass through
  192. 80 @state = :headers
  193. end
  194. end
  195. 372 super.tap do
  196. # delay setting this up until after the first transition to :body
  197. 372 @headers_sent = headers_sent
  198. end
  199. end
  200. 8 def <<(chunk)
  201. 80 @mutex.synchronize do
  202. 80 if @drainer
  203. 80 @body.clear if @body.respond_to?(:clear)
  204. 80 @drainer = nil
  205. end
  206. 80 @body << chunk
  207. 80 transition(:body)
  208. end
  209. end
  210. 8 def close
  211. 16 @mutex.synchronize do
  212. 16 return if @closed
  213. 16 @closed = true
  214. end
  215. # last chunk to send which ends the stream
  216. 16 self << ""
  217. end
  218. end
  219. 8 module RequestBodyMethods
  220. 8 def initialize(*, **)
  221. 16 super
  222. 16 @headers.delete("content-length")
  223. end
  224. 8 def empty?
  225. 113 false
  226. end
  227. end
  228. # overrides the declaration of +@write_buffer+, which is now a thread-safe buffer
  229. # responding to the same API.
  230. 8 module ConnectionMethods
  231. 8 attr_writer :signal
  232. 8 def initialize(*)
  233. 16 super
  234. 16 @write_buffer = BidiBuffer.new(@options.buffer_size)
  235. end
  236. # rebuffers the +@write_buffer+ before calculating interests.
  237. 8 def interests
  238. 1026 @write_buffer.rebuffer
  239. 1026 super
  240. end
  241. 8 private
  242. 8 def parser_type(protocol)
  243. 16 return HTTP2Bidi if protocol == "h2"
  244. super
  245. end
  246. 8 def set_parser_callbacks(parser)
  247. 16 super
  248. 16 parser.on(:flush_buffer) do
  249. 80 @signal.wakeup if @signal
  250. end
  251. end
  252. end
  253. end
  254. 8 register_plugin :stream_bidi, StreamBidi
  255. end
  256. end

lib/httpx/plugins/upgrade.rb

100.0% lines covered

34 relevant lines. 34 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin helps negotiating a new protocol from an HTTP/1.1 connection, via the
  6. # Upgrade header.
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Upgrade
  9. #
  10. 8 module Upgrade
  11. 8 class << self
  12. 8 def configure(klass)
  13. 32 klass.plugin(:"upgrade/h2")
  14. end
  15. 8 def extra_options(options)
  16. 32 options.merge(upgrade_handlers: {})
  17. end
  18. end
  19. 8 module OptionsMethods
  20. 8 def option_upgrade_handlers(value)
  21. 88 raise TypeError, ":upgrade_handlers must be a Hash" unless value.is_a?(Hash)
  22. 88 value
  23. end
  24. end
  25. 8 module InstanceMethods
  26. 8 def fetch_response(request, selector, options)
  27. 287 response = super
  28. 287 if response
  29. 95 return response unless response.is_a?(Response)
  30. 95 return response unless response.headers.key?("upgrade")
  31. 39 upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
  32. 39 return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
  33. 39 protocol_handler = options.upgrade_handlers[upgrade_protocol]
  34. 39 return response unless protocol_handler
  35. 39 log { "upgrading to #{upgrade_protocol}..." }
  36. 39 connection = find_connection(request.uri, selector, options)
  37. # do not upgrade already upgraded connections
  38. 39 return if connection.upgrade_protocol == upgrade_protocol
  39. 32 protocol_handler.call(connection, request, response)
  40. # keep in the loop if the server is switching, unless
  41. # the connection has been hijacked, in which case you want
  42. # to terminante immediately
  43. 32 return if response.status == 101 && !connection.hijacked
  44. end
  45. 208 response
  46. end
  47. end
  48. 8 module ConnectionMethods
  49. 8 attr_reader :upgrade_protocol, :hijacked
  50. 8 def hijack_io
  51. 8 @hijacked = true
  52. # connection is taken away from selector and not given back to the pool.
  53. 8 @current_session.deselect_connection(self, @current_selector, true)
  54. end
  55. end
  56. end
  57. 8 register_plugin(:upgrade, Upgrade)
  58. end
  59. end

lib/httpx/plugins/upgrade/h2.rb

91.67% lines covered

24 relevant lines. 22 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for upgrading an HTTP/1.1 connection to HTTP/2
  6. # via an Upgrade: h2 response declaration
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2
  9. #
  10. 8 module H2
  11. 8 class << self
  12. 8 def extra_options(options)
  13. 32 options.merge(upgrade_handlers: options.upgrade_handlers.merge("h2" => self))
  14. end
  15. 8 def call(connection, _request, _response)
  16. 8 connection.upgrade_to_h2
  17. end
  18. end
  19. 8 module ConnectionMethods
  20. 8 using URIExtensions
  21. 8 def upgrade_to_h2
  22. 8 prev_parser = @parser
  23. 8 if prev_parser
  24. 8 prev_parser.reset
  25. 7 @inflight -= prev_parser.requests.size
  26. end
  27. 8 @parser = Connection::HTTP2.new(@write_buffer, @options)
  28. 8 set_parser_callbacks(@parser)
  29. 8 @upgrade_protocol = "h2"
  30. # what's happening here:
  31. # a deviation from the state machine is done to perform the actions when a
  32. # connection is closed, without transitioning, so the connection is kept in the pool.
  33. # the state is reset to initial, so that the socket reconnect works out of the box,
  34. # while the parser is already here.
  35. 8 purge_after_closed
  36. 8 transition(:idle)
  37. 8 prev_parser.requests.each do |req|
  38. req.transition(:idle)
  39. send(req)
  40. end
  41. end
  42. end
  43. end
  44. 8 register_plugin(:"upgrade/h2", H2)
  45. end
  46. end

lib/httpx/plugins/webdav.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin implements convenience methods for performing WEBDAV requests.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/WebDav
  8. #
  9. 8 module WebDav
  10. 8 def self.configure(klass)
  11. 96 klass.plugin(:xml)
  12. end
  13. 8 module InstanceMethods
  14. 8 def copy(src, dest)
  15. 16 request("COPY", src, headers: { "destination" => @options.origin.merge(dest) })
  16. end
  17. 8 def move(src, dest)
  18. 16 request("MOVE", src, headers: { "destination" => @options.origin.merge(dest) })
  19. end
  20. 8 def lock(path, timeout: nil, &blk)
  21. 48 headers = {}
  22. 42 headers["timeout"] = if timeout && timeout.positive?
  23. 16 "Second-#{timeout}"
  24. else
  25. 32 "Infinite, Second-4100000000"
  26. end
  27. 48 xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" \
  28. "<D:lockinfo xmlns:D=\"DAV:\">" \
  29. "<D:lockscope><D:exclusive/></D:lockscope>" \
  30. "<D:locktype><D:write/></D:locktype>" \
  31. "<D:owner>null</D:owner>" \
  32. "</D:lockinfo>"
  33. 48 response = request("LOCK", path, headers: headers, xml: xml)
  34. 48 return response unless response.is_a?(Response)
  35. 48 return response unless blk && response.status == 200
  36. 16 lock_token = response.headers["lock-token"]
  37. 1 begin
  38. 16 blk.call(response)
  39. ensure
  40. 16 unlock(path, lock_token)
  41. end
  42. 16 response
  43. end
  44. 8 def unlock(path, lock_token)
  45. 32 request("UNLOCK", path, headers: { "lock-token" => lock_token })
  46. end
  47. 8 def mkcol(dir)
  48. 16 request("MKCOL", dir)
  49. end
  50. 8 def propfind(path, xml = nil)
  51. 64 body = case xml
  52. when :acl
  53. 16 '<?xml version="1.0" encoding="utf-8" ?><D:propfind xmlns:D="DAV:"><D:prop><D:owner/>' \
  54. "<D:supported-privilege-set/><D:current-user-privilege-set/><D:acl/></D:prop></D:propfind>"
  55. when nil
  56. 32 '<?xml version="1.0" encoding="utf-8"?><DAV:propfind xmlns:DAV="DAV:"><DAV:allprop/></DAV:propfind>'
  57. else
  58. 16 xml
  59. end
  60. 64 request("PROPFIND", path, headers: { "depth" => "1" }, xml: body)
  61. end
  62. 8 def proppatch(path, xml)
  63. 6 body = "<?xml version=\"1.0\"?>" \
  64. 12 "<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"
  65. 16 request("PROPPATCH", path, xml: body)
  66. end
  67. # %i[ orderpatch acl report search]
  68. end
  69. end
  70. 8 register_plugin(:webdav, WebDav)
  71. end
  72. end

lib/httpx/plugins/xml.rb

100.0% lines covered

34 relevant lines. 34 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin supports request XML encoding/response decoding using the nokogiri gem.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/XML
  8. #
  9. 8 module XML
  10. 8 MIME_TYPES = %r{\b(application|text)/(.+\+)?xml\b}.freeze
  11. 8 module Transcoder
  12. 8 module_function
  13. 8 class Encoder
  14. 8 def initialize(xml)
  15. 160 @raw = xml
  16. end
  17. 8 def content_type
  18. 160 charset = @raw.respond_to?(:encoding) && @raw.encoding ? @raw.encoding.to_s.downcase : "utf-8"
  19. 160 "application/xml; charset=#{charset}"
  20. end
  21. 8 def bytesize
  22. 512 @raw.to_s.bytesize
  23. end
  24. 8 def to_s
  25. 160 @raw.to_s
  26. end
  27. end
  28. 8 def encode(xml)
  29. 160 Encoder.new(xml)
  30. end
  31. 8 def decode(response)
  32. 24 content_type = response.content_type.mime_type
  33. 24 raise HTTPX::Error, "invalid form mime type (#{content_type})" unless MIME_TYPES.match?(content_type)
  34. 24 Nokogiri::XML.method(:parse)
  35. end
  36. end
  37. 8 class << self
  38. 8 def load_dependencies(*)
  39. 144 require "nokogiri"
  40. end
  41. end
  42. 8 module ResponseMethods
  43. # decodes the response payload into a Nokogiri::XML::Node object **if** the payload is valid
  44. # "application/xml" (requires the "nokogiri" gem).
  45. 8 def xml
  46. 16 decode(Transcoder)
  47. end
  48. end
  49. 8 module RequestBodyClassMethods
  50. # ..., xml: Nokogiri::XML::Node #=> xml encoder
  51. 8 def initialize_body(params)
  52. 592 if (xml = params.delete(:xml))
  53. # @type var xml: Nokogiri::XML::Node | String
  54. 140 return Transcoder.encode(xml)
  55. end
  56. 432 super
  57. end
  58. end
  59. end
  60. 8 register_plugin(:xml, XML)
  61. end
  62. end

lib/httpx/pmatch_extensions.rb

100.0% lines covered

17 relevant lines. 17 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module ResponsePatternMatchExtensions
  4. 27 def deconstruct
  5. 41 [@status, @headers, @body]
  6. end
  7. 27 def deconstruct_keys(_keys)
  8. 70 { status: @status, headers: @headers, body: @body }
  9. end
  10. end
  11. 27 module ErrorResponsePatternMatchExtensions
  12. 27 def deconstruct
  13. 11 [@error]
  14. end
  15. 27 def deconstruct_keys(_keys)
  16. 35 { error: @error }
  17. end
  18. end
  19. 27 module HeadersPatternMatchExtensions
  20. 27 def deconstruct
  21. 7 to_a
  22. end
  23. end
  24. 27 Headers.include HeadersPatternMatchExtensions
  25. 27 Response.include ResponsePatternMatchExtensions
  26. 27 ErrorResponse.include ErrorResponsePatternMatchExtensions
  27. end

lib/httpx/pool.rb

100.0% lines covered

89 relevant lines. 89 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "httpx/selector"
  3. 27 require "httpx/connection"
  4. 27 require "httpx/resolver"
  5. 27 module HTTPX
  6. 27 class Pool
  7. 27 using ArrayExtensions::FilterMap
  8. 27 using URIExtensions
  9. 27 POOL_TIMEOUT = 5
  10. # Sets up the connection pool with the given +options+, which can be the following:
  11. #
  12. # :max_connections:: the maximum number of connections held in the pool.
  13. # :max_connections_per_origin :: the maximum number of connections held in the pool pointing to a given origin.
  14. # :pool_timeout :: the number of seconds to wait for a connection to a given origin (before raising HTTPX::PoolTimeoutError)
  15. #
  16. 27 def initialize(options)
  17. 11175 @max_connections = options.fetch(:max_connections, Float::INFINITY)
  18. 11175 @max_connections_per_origin = options.fetch(:max_connections_per_origin, Float::INFINITY)
  19. 11175 @pool_timeout = options.fetch(:pool_timeout, POOL_TIMEOUT)
  20. 17322 @resolvers = Hash.new { |hs, resolver_type| hs[resolver_type] = [] }
  21. 11175 @resolver_mtx = Thread::Mutex.new
  22. 11175 @connections = []
  23. 11175 @connection_mtx = Thread::Mutex.new
  24. 11175 @connections_counter = 0
  25. 11175 @max_connections_cond = ConditionVariable.new
  26. 11175 @origin_counters = Hash.new(0)
  27. 16576 @origin_conds = Hash.new { |hs, orig| hs[orig] = ConditionVariable.new }
  28. end
  29. # connections returned by this function are not expected to return to the connection pool.
  30. 27 def pop_connection
  31. 11632 @connection_mtx.synchronize do
  32. 11632 drop_connection
  33. end
  34. end
  35. # opens a connection to the IP reachable through +uri+.
  36. # Many hostnames are reachable through the same IP, so we try to
  37. # maximize pipelining by opening as few connections as possible.
  38. #
  39. 27 def checkout_connection(uri, options)
  40. 8081 return checkout_new_connection(uri, options) if options.io
  41. 8015 @connection_mtx.synchronize do
  42. 8015 acquire_connection(uri, options) || begin
  43. 7382 if @connections_counter == @max_connections
  44. # this takes precedence over per-origin
  45. 16 @max_connections_cond.wait(@connection_mtx, @pool_timeout)
  46. 16 acquire_connection(uri, options) || begin
  47. 11 if @connections_counter == @max_connections
  48. # if no matching usable connection was found, the pool will make room and drop a closed connection. if none is found,
  49. # this means that all of them are persistent or being used, so raise a timeout error.
  50. 14 conn = @connections.find { |c| c.state == :closed }
  51. 1 raise PoolTimeoutError.new(@pool_timeout,
  52. 11 "Timed out after #{@pool_timeout} seconds while waiting for a connection") unless conn
  53. 3 drop_connection(conn)
  54. end
  55. end
  56. end
  57. 7374 if @origin_counters[uri.origin] == @max_connections_per_origin
  58. 16 @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
  59. 16 return acquire_connection(uri, options) ||
  60. raise(PoolTimeoutError.new(@pool_timeout,
  61. 1 "Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
  62. end
  63. 6542 @connections_counter += 1
  64. 7358 @origin_counters[uri.origin] += 1
  65. 7358 checkout_new_connection(uri, options)
  66. end
  67. end
  68. end
  69. 27 def checkin_connection(connection)
  70. 7946 return if connection.options.io
  71. 7880 @connection_mtx.synchronize do
  72. 7880 @connections << connection
  73. 7880 @max_connections_cond.signal
  74. 7880 @origin_conds[connection.origin.to_s].signal
  75. end
  76. end
  77. 27 def checkout_mergeable_connection(connection)
  78. 7345 return if connection.options.io
  79. 7345 @connection_mtx.synchronize do
  80. 7345 idx = @connections.find_index do |ch|
  81. 239 ch != connection && ch.mergeable?(connection)
  82. end
  83. 7345 @connections.delete_at(idx) if idx
  84. end
  85. end
  86. 27 def reset_resolvers
  87. 14424 @resolver_mtx.synchronize { @resolvers.clear }
  88. end
  89. 27 def checkout_resolver(options)
  90. 7174 resolver_type = options.resolver_class
  91. 7174 resolver_type = Resolver.resolver_for(resolver_type)
  92. 7174 @resolver_mtx.synchronize do
  93. 7174 resolvers = @resolvers[resolver_type]
  94. 7174 idx = resolvers.find_index do |res|
  95. 33 res.options == options
  96. end
  97. 7174 resolvers.delete_at(idx) if idx
  98. end || checkout_new_resolver(resolver_type, options)
  99. end
  100. 27 def checkin_resolver(resolver)
  101. 372 @resolver_mtx.synchronize do
  102. 372 resolvers = @resolvers[resolver.class]
  103. 372 resolver = resolver.multi
  104. 372 resolvers << resolver unless resolvers.include?(resolver)
  105. end
  106. end
  107. skipped # :nocov:
  108. skipped def inspect
  109. skipped "#<#{self.class}:#{object_id} " \
  110. skipped "@max_connections_per_origin=#{@max_connections_per_origin} " \
  111. skipped "@pool_timeout=#{@pool_timeout} " \
  112. skipped "@connections=#{@connections.size}>"
  113. skipped end
  114. skipped # :nocov:
  115. 27 private
  116. 27 def acquire_connection(uri, options)
  117. 8047 idx = @connections.find_index do |connection|
  118. 863 connection.match?(uri, options)
  119. end
  120. 8047 return unless idx
  121. 646 @connections.delete_at(idx)
  122. end
  123. 27 def checkout_new_connection(uri, options)
  124. 7424 options.connection_class.new(uri, options)
  125. end
  126. 27 def checkout_new_resolver(resolver_type, options)
  127. 7146 if resolver_type.multi?
  128. 7121 Resolver::Multi.new(resolver_type, options)
  129. else
  130. 25 resolver_type.new(options)
  131. end
  132. end
  133. # drops and returns the +connection+ from the connection pool; if +connection+ is <tt>nil</tt> (default),
  134. # the first available connection from the pool will be dropped.
  135. 27 def drop_connection(connection = nil)
  136. 11635 if connection
  137. 3 @connections.delete(connection)
  138. else
  139. 11632 connection = @connections.shift
  140. 11632 return unless connection
  141. end
  142. 3904 @connections_counter -= 1
  143. 4423 @origin_conds.delete(connection.origin) if (@origin_counters[connection.origin.to_s] -= 1).zero?
  144. 4423 connection
  145. end
  146. end
  147. end

lib/httpx/punycode.rb

100.0% lines covered

9 relevant lines. 9 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Punycode
  4. 27 module_function
  5. begin
  6. 27 require "idnx"
  7. 26 def encode_hostname(hostname)
  8. 32 Idnx.to_punycode(hostname)
  9. end
  10. rescue LoadError
  11. 1 def encode_hostname(hostname)
  12. 1 warn "#{hostname} cannot be converted to punycode. Install the " \
  13. "\"idnx\" gem: https://github.com/HoneyryderChuck/idnx"
  14. 1 hostname
  15. end
  16. end
  17. end
  18. end

lib/httpx/request.rb

100.0% lines covered

132 relevant lines. 132 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "delegate"
  3. 27 require "forwardable"
  4. 27 module HTTPX
  5. # Defines how an HTTP request is handled internally, both in terms of making attributes accessible,
  6. # as well as maintaining the state machine which manages streaming the request onto the wire.
  7. 27 class Request
  8. 27 extend Forwardable
  9. 27 include Callbacks
  10. 27 using URIExtensions
  11. 27 ALLOWED_URI_SCHEMES = %w[https http].freeze
  12. # default value used for "user-agent" header, when not overridden.
  13. 27 USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
  14. # the upcased string HTTP verb for this request.
  15. 27 attr_reader :verb
  16. # the absolute URI object for this request.
  17. 27 attr_reader :uri
  18. # an HTTPX::Headers object containing the request HTTP headers.
  19. 27 attr_reader :headers
  20. # an HTTPX::Request::Body object containing the request body payload (or +nil+, whenn there is none).
  21. 27 attr_reader :body
  22. # a symbol describing which frame is currently being flushed.
  23. 27 attr_reader :state
  24. # an HTTPX::Options object containing request options.
  25. 27 attr_reader :options
  26. # the corresponding HTTPX::Response object, when there is one.
  27. 27 attr_reader :response
  28. # Exception raised during enumerable body writes.
  29. 27 attr_reader :drain_error
  30. # The IP address from the peer server.
  31. 27 attr_accessor :peer_address
  32. 27 attr_writer :persistent
  33. 27 attr_reader :active_timeouts
  34. # will be +true+ when request body has been completely flushed.
  35. 27 def_delegator :@body, :empty?
  36. # closes the body
  37. 27 def_delegator :@body, :close
  38. # initializes the instance with the given +verb+ (an upppercase String, ex. 'GEt'),
  39. # an absolute or relative +uri+ (either as String or URI::HTTP object), the
  40. # request +options+ (instance of HTTPX::Options) and an optional Hash of +params+.
  41. #
  42. # Besides any of the options documented in HTTPX::Options (which would override or merge with what
  43. # +options+ sets), it accepts also the following:
  44. #
  45. # :params :: hash or array of key-values which will be encoded and set in the query string of request uris.
  46. # :body :: to be encoded in the request body payload. can be a String, an IO object (i.e. a File), or an Enumerable.
  47. # :form :: hash of array of key-values which will be form-urlencoded- or multipart-encoded in requests body payload.
  48. # :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
  49. # :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
  50. #
  51. # :body, :form, :json and :xml are all mutually exclusive, i.e. only one of them gets picked up.
  52. 27 def initialize(verb, uri, options, params = EMPTY_HASH)
  53. 10272 @verb = verb.to_s.upcase
  54. 10272 @uri = Utils.to_uri(uri)
  55. 10271 @headers = options.headers.dup
  56. 10271 merge_headers(params.delete(:headers)) if params.key?(:headers)
  57. 10271 @headers["user-agent"] ||= USER_AGENT
  58. 10271 @headers["accept"] ||= "*/*"
  59. # forego compression in the Range request case
  60. 10271 if @headers.key?("range")
  61. 8 @headers.delete("accept-encoding")
  62. else
  63. 10263 @headers["accept-encoding"] ||= options.supported_compression_formats
  64. end
  65. 10271 @query_params = params.delete(:params) if params.key?(:params)
  66. 10271 @body = options.request_body_class.new(@headers, options, **params)
  67. 10263 @options = @body.options
  68. 10263 if @uri.relative? || @uri.host.nil?
  69. 600 origin = @options.origin
  70. 600 raise(Error, "invalid URI: #{@uri}") unless origin
  71. 576 base_path = @options.base_path
  72. 576 @uri = origin.merge("#{base_path}#{@uri}")
  73. end
  74. 10239 raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
  75. 10227 @state = :idle
  76. 10227 @response = nil
  77. 10227 @peer_address = nil
  78. 10227 @ping = false
  79. 10227 @persistent = @options.persistent
  80. 10227 @active_timeouts = []
  81. end
  82. # whether request has been buffered with a ping
  83. 27 def ping?
  84. 487 @ping
  85. end
  86. # marks the request as having been buffered with a ping
  87. 27 def ping!
  88. 20 @ping = true
  89. end
  90. # the read timeout defined for this request.
  91. 27 def read_timeout
  92. 19142 @options.timeout[:read_timeout]
  93. end
  94. # the write timeout defined for this request.
  95. 27 def write_timeout
  96. 19142 @options.timeout[:write_timeout]
  97. end
  98. # the request timeout defined for this request.
  99. 27 def request_timeout
  100. 18858 @options.timeout[:request_timeout]
  101. end
  102. 27 def persistent?
  103. 4892 @persistent
  104. end
  105. # if the request contains trailer headers
  106. 27 def trailers?
  107. 3130 defined?(@trailers)
  108. end
  109. # returns an instance of HTTPX::Headers containing the trailer headers
  110. 27 def trailers
  111. 88 @trailers ||= @options.headers_class.new
  112. end
  113. # returns +:r+ or +:w+, depending on whether the request is waiting for a response or flushing.
  114. 27 def interests
  115. 25190 return :r if @state == :done || @state == :expect
  116. 2925 :w
  117. end
  118. 27 def can_buffer?
  119. 26071 @state != :done
  120. end
  121. # merges +h+ into the instance of HTTPX::Headers of the request.
  122. 27 def merge_headers(h)
  123. 1107 @headers = @headers.merge(h)
  124. end
  125. # the URI scheme of the request +uri+.
  126. 27 def scheme
  127. 3695 @uri.scheme
  128. end
  129. # sets the +response+ on this request.
  130. 27 def response=(response)
  131. 9576 return unless response
  132. 9576 if response.is_a?(Response) && response.status < 200
  133. # deal with informational responses
  134. 160 if response.status == 100 && @headers.key?("expect")
  135. 136 @informational_status = response.status
  136. 136 return
  137. end
  138. # 103 Early Hints advertises resources in document to browsers.
  139. # not very relevant for an HTTP client, discard.
  140. 24 return if response.status >= 103
  141. end
  142. 9440 @response = response
  143. 9440 emit(:response_started, response)
  144. end
  145. # returnns the URI path of the request +uri+.
  146. 27 def path
  147. 8737 path = uri.path.dup
  148. 8737 path = +"" if path.nil?
  149. 8737 path << "/" if path.empty?
  150. 8737 path << "?#{query}" unless query.empty?
  151. 8737 path
  152. end
  153. # returs the URI authority of the request.
  154. #
  155. # session.build_request("GET", "https://google.com/query").authority #=> "google.com"
  156. # session.build_request("GET", "http://internal:3182/a").authority #=> "internal:3182"
  157. 27 def authority
  158. 8827 @uri.authority
  159. end
  160. # returs the URI origin of the request.
  161. #
  162. # session.build_request("GET", "https://google.com/query").authority #=> "https://google.com"
  163. # session.build_request("GET", "http://internal:3182/a").authority #=> "http://internal:3182"
  164. 27 def origin
  165. 3974 @uri.origin
  166. end
  167. # returs the URI query string of the request (when available).
  168. #
  169. # session.build_request("GET", "https://search.com").query #=> ""
  170. # session.build_request("GET", "https://search.com?q=a").query #=> "q=a"
  171. # session.build_request("GET", "https://search.com", params: { q: "a"}).query #=> "q=a"
  172. # session.build_request("GET", "https://search.com?q=a", params: { foo: "bar"}).query #=> "q=a&foo&bar"
  173. 27 def query
  174. 9700 return @query if defined?(@query)
  175. 8159 query = []
  176. 8159 if (q = @query_params) && !q.empty?
  177. 152 query << Transcoder::Form.encode(q)
  178. end
  179. 8159 query << @uri.query if @uri.query
  180. 8159 @query = query.join("&")
  181. end
  182. # consumes and returns the next available chunk of request body that can be sent
  183. 27 def drain_body
  184. 9297 return nil if @body.nil?
  185. 9297 @drainer ||= @body.each
  186. 9297 chunk = @drainer.next.dup
  187. 6115 emit(:body_chunk, chunk)
  188. 6115 chunk
  189. rescue StopIteration
  190. 3158 nil
  191. rescue StandardError => e
  192. 24 @drain_error = e
  193. 24 nil
  194. end
  195. skipped # :nocov:
  196. skipped def inspect
  197. skipped "#<#{self.class}:#{object_id} " \
  198. skipped "#{@verb} " \
  199. skipped "#{uri} " \
  200. skipped "@headers=#{@headers} " \
  201. skipped "@body=#{@body}>"
  202. skipped end
  203. skipped # :nocov:
  204. # moves on to the +nextstate+ of the request state machine (when all preconditions are met)
  205. 27 def transition(nextstate)
  206. 37184 case nextstate
  207. when :idle
  208. 742 @body.rewind
  209. 742 @ping = false
  210. 742 @response = nil
  211. 742 @drainer = nil
  212. 742 @active_timeouts.clear
  213. when :headers
  214. 11290 return unless @state == :idle
  215. when :body
  216. 11354 return unless @state == :headers ||
  217. @state == :expect
  218. 9195 if @headers.key?("expect")
  219. 522 if @informational_status && @informational_status == 100
  220. # check for 100 Continue response, and deallocate the var
  221. # if @informational_status == 100
  222. # @response = nil
  223. # end
  224. else
  225. 394 return if @state == :expect # do not re-set it
  226. 144 nextstate = :expect
  227. end
  228. end
  229. when :trailers
  230. 9092 return unless @state == :body
  231. when :done
  232. 9100 return if @state == :expect
  233. end
  234. 35915 @state = nextstate
  235. 35915 emit(@state, self)
  236. 14411 nil
  237. end
  238. # whether the request supports the 100-continue handshake and already processed the 100 response.
  239. 27 def expects?
  240. 8248 @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
  241. end
  242. 27 def set_timeout_callback(event, &callback)
  243. 96132 clb = once(event, &callback)
  244. # reset timeout callbacks when requests get rerouted to a different connection
  245. 96132 once(:idle) do
  246. 3550 callbacks(event).delete(clb)
  247. end
  248. end
  249. end
  250. end
  251. 27 require_relative "request/body"

lib/httpx/request/body.rb

100.0% lines covered

66 relevant lines. 66 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # Implementation of the HTTP Request body as a delegator which iterates (responds to +each+) payload chunks.
  4. 27 class Request::Body < SimpleDelegator
  5. 27 class << self
  6. 27 def new(_, options, body: nil, **params)
  7. 10279 if body.is_a?(self)
  8. # request derives its options from body
  9. 16 body.options = options.merge(params)
  10. 14 return body
  11. end
  12. 10263 super
  13. end
  14. end
  15. 27 attr_accessor :options
  16. # inits the instance with the request +headers+, +options+ and +params+, which contain the payload definition.
  17. # it wraps the given body with the appropriate encoder on initialization.
  18. #
  19. # ..., json: { foo: "bar" }) #=> json encoder
  20. # ..., form: { foo: "bar" }) #=> form urlencoded encoder
  21. # ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
  22. # ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
  23. # ..., form: { body: "bla") }) #=> raw data encoder
  24. 27 def initialize(h, options, **params)
  25. 10263 @headers = h
  26. 10263 @body = self.class.initialize_body(params)
  27. 10263 @options = options.merge(params)
  28. 10263 if @body
  29. 3196 if @options.compress_request_body && @headers.key?("content-encoding")
  30. 96 @headers.get("content-encoding").each do |encoding|
  31. 96 @body = self.class.initialize_deflater_body(@body, encoding)
  32. end
  33. end
  34. 3196 @headers["content-type"] ||= @body.content_type
  35. 3196 @headers["content-length"] = @body.bytesize unless unbounded_body?
  36. end
  37. 10255 super(@body)
  38. end
  39. # consumes and yields the request payload in chunks.
  40. 27 def each(&block)
  41. 6610 return enum_for(__method__) unless block
  42. 3309 return if @body.nil?
  43. 3237 body = stream(@body)
  44. 3237 if body.respond_to?(:read)
  45. 5434 while (chunk = body.read(16_384))
  46. 3363 block.call(chunk)
  47. end
  48. # TODO: use copy_stream once bug is resolved: https://bugs.ruby-lang.org/issues/21131
  49. # ::IO.copy_stream(body, ProcIO.new(block))
  50. 1847 elsif body.respond_to?(:each)
  51. 510 body.each(&block)
  52. else
  53. 1338 block[body.to_s]
  54. end
  55. end
  56. 27 def close
  57. 459 @body.close if @body.respond_to?(:close)
  58. end
  59. # if the +@body+ is rewindable, it rewinnds it.
  60. 27 def rewind
  61. 806 return if empty?
  62. 172 @body.rewind if @body.respond_to?(:rewind)
  63. end
  64. # return +true+ if the +body+ has been fully drained (or does nnot exist).
  65. 27 def empty?
  66. 19510 return true if @body.nil?
  67. 8725 return false if chunked?
  68. 8629 @body.bytesize.zero?
  69. end
  70. # returns the +@body+ payload size in bytes.
  71. 27 def bytesize
  72. 3472 return 0 if @body.nil?
  73. 128 @body.bytesize
  74. end
  75. # sets the body to yield using chunked trannsfer encoding format.
  76. 27 def stream(body)
  77. 3237 return body unless chunked?
  78. 96 Transcoder::Chunker.encode(body.enum_for(:each))
  79. end
  80. # returns whether the body yields infinitely.
  81. 27 def unbounded_body?
  82. 3720 return @unbounded_body if defined?(@unbounded_body)
  83. 3268 @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
  84. end
  85. # returns whether the chunked transfer encoding header is set.
  86. 27 def chunked?
  87. 20276 @headers["transfer-encoding"] == "chunked"
  88. end
  89. # sets the chunked transfer encoding header.
  90. 27 def chunk!
  91. 32 @headers.add("transfer-encoding", "chunked")
  92. end
  93. skipped # :nocov:
  94. skipped def inspect
  95. skipped "#<#{self.class}:#{object_id} " \
  96. skipped "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
  97. skipped end
  98. skipped # :nocov:
  99. 27 class << self
  100. 27 def initialize_body(params)
  101. 10103 if (body = params.delete(:body))
  102. # @type var body: bodyIO
  103. 1448 Transcoder::Body.encode(body)
  104. 8655 elsif (form = params.delete(:form))
  105. # @type var form: Transcoder::urlencoded_input
  106. 1505 Transcoder::Form.encode(form)
  107. 7150 elsif (json = params.delete(:json))
  108. # @type var body: _ToJson
  109. 83 Transcoder::JSON.encode(json)
  110. end
  111. end
  112. # returns the +body+ wrapped with the correct deflater accordinng to the given +encodisng+.
  113. 27 def initialize_deflater_body(body, encoding)
  114. 87 case encoding
  115. when "gzip"
  116. 48 Transcoder::GZIP.encode(body)
  117. when "deflate"
  118. 24 Transcoder::Deflate.encode(body)
  119. when "identity"
  120. 16 body
  121. else
  122. 8 body
  123. end
  124. end
  125. end
  126. end
  127. end

lib/httpx/resolver.rb

100.0% lines covered

82 relevant lines. 82 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "ipaddr"
  4. 27 module HTTPX
  5. 27 module Resolver
  6. 27 RESOLVE_TIMEOUT = [2, 3].freeze
  7. 27 require "httpx/resolver/resolver"
  8. 27 require "httpx/resolver/system"
  9. 27 require "httpx/resolver/native"
  10. 27 require "httpx/resolver/https"
  11. 27 require "httpx/resolver/multi"
  12. 27 @lookup_mutex = Thread::Mutex.new
  13. 201 @lookups = Hash.new { |h, k| h[k] = [] }
  14. 27 @identifier_mutex = Thread::Mutex.new
  15. 27 @identifier = 1
  16. 27 @system_resolver = Resolv::Hosts.new
  17. 27 module_function
  18. 27 def resolver_for(resolver_type)
  19. 6403 case resolver_type
  20. 7037 when :native then Native
  21. 33 when :system then System
  22. 74 when :https then HTTPS
  23. else
  24. 76 return resolver_type if resolver_type.is_a?(Class) && resolver_type < Resolver
  25. 8 raise Error, "unsupported resolver type (#{resolver_type})"
  26. end
  27. end
  28. 27 def nolookup_resolve(hostname)
  29. 7019 ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
  30. end
  31. 27 def ip_resolve(hostname)
  32. 7019 [IPAddr.new(hostname)]
  33. rescue ArgumentError
  34. end
  35. 27 def system_resolve(hostname)
  36. 603 ips = @system_resolver.getaddresses(hostname)
  37. 603 return if ips.empty?
  38. 795 ips.map { |ip| IPAddr.new(ip) }
  39. rescue IOError
  40. end
  41. 27 def cached_lookup(hostname)
  42. 6482 now = Utils.now
  43. 6482 lookup_synchronize do |lookups|
  44. 6482 lookup(hostname, lookups, now)
  45. end
  46. end
  47. 27 def cached_lookup_set(hostname, family, entries)
  48. 240 now = Utils.now
  49. 240 entries.each do |entry|
  50. 322 entry["TTL"] += now
  51. end
  52. 240 lookup_synchronize do |lookups|
  53. 212 case family
  54. when Socket::AF_INET6
  55. 40 lookups[hostname].concat(entries)
  56. when Socket::AF_INET
  57. 200 lookups[hostname].unshift(*entries)
  58. end
  59. 240 entries.each do |entry|
  60. 322 next unless entry["name"] != hostname
  61. 30 case family
  62. when Socket::AF_INET6
  63. 8 lookups[entry["name"]] << entry
  64. when Socket::AF_INET
  65. 24 lookups[entry["name"]].unshift(entry)
  66. end
  67. end
  68. end
  69. end
  70. # do not use directly!
  71. 27 def lookup(hostname, lookups, ttl)
  72. 6490 return unless lookups.key?(hostname)
  73. 5883 entries = lookups[hostname] = lookups[hostname].select do |address|
  74. 16613 address["TTL"] > ttl
  75. end
  76. 5883 ips = entries.flat_map do |address|
  77. 16595 if (als = address["alias"])
  78. 8 lookup(als, lookups, ttl)
  79. else
  80. 16587 IPAddr.new(address["data"])
  81. end
  82. end.compact
  83. 5883 ips unless ips.empty?
  84. end
  85. 27 def generate_id
  86. 1732 id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
  87. end
  88. 27 def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
  89. 800 Resolv::DNS::Message.new(message_id).tap do |query|
  90. 866 query.rd = 1
  91. 866 query.add_question(hostname, type)
  92. 131 end.encode
  93. end
  94. 27 def decode_dns_answer(payload)
  95. 65 begin
  96. 782 message = Resolv::DNS::Message.decode(payload)
  97. rescue Resolv::DNS::DecodeError => e
  98. 6 return :decode_error, e
  99. end
  100. # no domain was found
  101. 776 return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
  102. 268 return :message_truncated if message.tc == 1
  103. 256 return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
  104. 250 addresses = []
  105. 250 message.each_answer do |question, _, value|
  106. 1023 case value
  107. when Resolv::DNS::Resource::IN::CNAME
  108. 18 addresses << {
  109. "name" => question.to_s,
  110. "TTL" => value.ttl,
  111. "alias" => value.name.to_s,
  112. }
  113. when Resolv::DNS::Resource::IN::A,
  114. Resolv::DNS::Resource::IN::AAAA
  115. 1034 addresses << {
  116. 29 "name" => question.to_s,
  117. "TTL" => value.ttl,
  118. "data" => value.address.to_s,
  119. }
  120. end
  121. end
  122. 250 [:ok, addresses]
  123. end
  124. 27 def lookup_synchronize
  125. 13444 @lookup_mutex.synchronize { yield(@lookups) }
  126. end
  127. 27 def id_synchronize(&block)
  128. 866 @identifier_mutex.synchronize(&block)
  129. end
  130. end
  131. end

lib/httpx/resolver/https.rb

86.11% lines covered

144 relevant lines. 124 lines covered and 20 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "uri"
  4. 27 require "forwardable"
  5. 27 require "httpx/base64"
  6. 27 module HTTPX
  7. # Implementation of a DoH name resolver (https://www.youtube.com/watch?v=unMXvnY2FNM).
  8. # It wraps an HTTPX::Connection object which integrates with the main session in the
  9. # same manner as other performed HTTP requests.
  10. #
  11. 27 class Resolver::HTTPS < Resolver::Resolver
  12. 27 extend Forwardable
  13. 27 using URIExtensions
  14. 27 module DNSExtensions
  15. 27 refine Resolv::DNS do
  16. 27 def generate_candidates(name)
  17. 42 @config.generate_candidates(name)
  18. end
  19. end
  20. end
  21. 27 using DNSExtensions
  22. 27 NAMESERVER = "https://1.1.1.1/dns-query"
  23. 2 DEFAULTS = {
  24. 25 uri: NAMESERVER,
  25. use_get: false,
  26. }.freeze
  27. 27 def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?, :handle_socket_timeout
  28. 27 def initialize(_, options)
  29. 90 super
  30. 90 @resolver_options = DEFAULTS.merge(@options.resolver_options)
  31. 90 @queries = {}
  32. 90 @requests = {}
  33. 90 @uri = URI(@resolver_options[:uri])
  34. 90 @uri_addresses = nil
  35. 90 @resolver = Resolv::DNS.new
  36. 90 @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
  37. 90 @resolver.lazy_initialize
  38. end
  39. 27 def <<(connection)
  40. 90 return if @uri.origin == connection.peer.to_s
  41. 48 @uri_addresses ||= HTTPX::Resolver.nolookup_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
  42. 48 if @uri_addresses.empty?
  43. 6 ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
  44. 6 ex.set_backtrace(caller)
  45. 6 connection.force_reset
  46. 6 throw(:resolve_error, ex)
  47. end
  48. 42 resolve(connection)
  49. end
  50. 27 def closed?
  51. true
  52. end
  53. 27 def empty?
  54. 84 true
  55. end
  56. 27 def resolver_connection
  57. # TODO: leaks connection object into the pool
  58. 66 @resolver_connection ||= @current_session.find_connection(@uri, @current_selector,
  59. @options.merge(ssl: { alpn_protocols: %w[h2] })).tap do |conn|
  60. 42 emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
  61. end
  62. end
  63. 27 private
  64. 27 def resolve(connection = nil, hostname = nil)
  65. 66 @connections.shift until @connections.empty? || @connections.first.state != :closed
  66. 66 connection ||= @connections.first
  67. 66 return unless connection
  68. 66 hostname ||= @queries.key(connection)
  69. 66 if hostname.nil?
  70. 42 hostname = connection.peer.host
  71. log do
  72. "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
  73. 42 end if connection.peer.non_ascii_hostname
  74. 42 hostname = @resolver.generate_candidates(hostname).each do |name|
  75. 126 @queries[name.to_s] = connection
  76. end.first.to_s
  77. else
  78. 24 @queries[hostname] = connection
  79. end
  80. 66 log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
  81. begin
  82. 66 request = build_request(hostname)
  83. 66 request.on(:response, &method(:on_response).curry(2)[request])
  84. 66 request.on(:promise, &method(:on_promise))
  85. 66 @requests[request] = hostname
  86. 66 resolver_connection.send(request)
  87. 66 @connections << connection
  88. rescue ResolveError, Resolv::DNS::EncodeError => e
  89. reset_hostname(hostname)
  90. emit_resolve_error(connection, connection.peer.host, e)
  91. end
  92. end
  93. 27 def on_response(request, response)
  94. 66 response.raise_for_status
  95. rescue StandardError => e
  96. 6 hostname = @requests.delete(request)
  97. 6 connection = reset_hostname(hostname)
  98. 6 emit_resolve_error(connection, connection.peer.host, e)
  99. else
  100. # @type var response: HTTPX::Response
  101. 60 parse(request, response)
  102. ensure
  103. 66 @requests.delete(request)
  104. end
  105. 27 def on_promise(_, stream)
  106. log(level: 2) { "#{stream.id}: refusing stream!" }
  107. stream.refuse
  108. end
  109. 27 def parse(request, response)
  110. 60 code, result = decode_response_body(response)
  111. 60 case code
  112. when :ok
  113. 18 parse_addresses(result, request)
  114. when :no_domain_found
  115. # Indicates no such domain was found.
  116. 36 host = @requests.delete(request)
  117. 36 connection = reset_hostname(host, reset_candidates: false)
  118. 36 unless @queries.value?(connection)
  119. 12 emit_resolve_error(connection)
  120. 12 return
  121. end
  122. 24 resolve
  123. when :dns_error
  124. host = @requests.delete(request)
  125. connection = reset_hostname(host)
  126. emit_resolve_error(connection)
  127. when :decode_error
  128. 6 host = @requests.delete(request)
  129. 6 connection = reset_hostname(host)
  130. 6 emit_resolve_error(connection, connection.peer.host, result)
  131. end
  132. end
  133. 27 def parse_addresses(answers, request)
  134. 18 if answers.empty?
  135. # no address found, eliminate candidates
  136. host = @requests.delete(request)
  137. connection = reset_hostname(host)
  138. emit_resolve_error(connection)
  139. return
  140. else
  141. 42 answers = answers.group_by { |answer| answer["name"] }
  142. 18 answers.each do |hostname, addresses|
  143. 24 addresses = addresses.flat_map do |address|
  144. 24 if address.key?("alias")
  145. 6 alias_address = answers[address["alias"]]
  146. 6 if alias_address.nil?
  147. reset_hostname(address["name"])
  148. if early_resolve(connection, hostname: address["alias"])
  149. @connections.delete(connection)
  150. else
  151. resolve(connection, address["alias"])
  152. return # rubocop:disable Lint/NonLocalExitFromIterator
  153. end
  154. else
  155. 6 alias_address
  156. end
  157. else
  158. 18 address
  159. end
  160. end.compact
  161. 24 next if addresses.empty?
  162. 24 hostname.delete_suffix!(".") if hostname.end_with?(".")
  163. 24 connection = reset_hostname(hostname, reset_candidates: false)
  164. 24 next unless connection # probably a retried query for which there's an answer
  165. 18 @connections.delete(connection)
  166. # eliminate other candidates
  167. 54 @queries.delete_if { |_, conn| connection == conn }
  168. 18 Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
  169. 54 catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
  170. end
  171. end
  172. 18 return if @connections.empty?
  173. resolve
  174. end
  175. 27 def build_request(hostname)
  176. 60 uri = @uri.dup
  177. 60 rklass = @options.request_class
  178. 60 payload = Resolver.encode_dns_query(hostname, type: @record_type)
  179. 60 if @resolver_options[:use_get]
  180. 6 params = URI.decode_www_form(uri.query.to_s)
  181. 6 params << ["type", FAMILY_TYPES[@record_type]]
  182. 6 params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
  183. 6 uri.query = URI.encode_www_form(params)
  184. 6 request = rklass.new("GET", uri, @options)
  185. else
  186. 54 request = rklass.new("POST", uri, @options, body: [payload])
  187. 54 request.headers["content-type"] = "application/dns-message"
  188. end
  189. 60 request.headers["accept"] = "application/dns-message"
  190. 60 request
  191. end
  192. 27 def decode_response_body(response)
  193. 54 case response.headers["content-type"]
  194. when "application/dns-udpwireformat",
  195. "application/dns-message"
  196. 54 Resolver.decode_dns_answer(response.to_s)
  197. else
  198. raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
  199. end
  200. end
  201. 27 def reset_hostname(hostname, reset_candidates: true)
  202. 72 connection = @queries.delete(hostname)
  203. 72 return connection unless connection && reset_candidates
  204. # eliminate other candidates
  205. 36 candidates = @queries.select { |_, conn| connection == conn }.keys
  206. 36 @queries.delete_if { |h, _| candidates.include?(h) }
  207. 12 connection
  208. end
  209. end
  210. end

lib/httpx/resolver/multi.rb

88.24% lines covered

51 relevant lines. 45 lines covered and 6 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 require "resolv"
  4. 27 module HTTPX
  5. 27 class Resolver::Multi
  6. 27 include Callbacks
  7. 27 using ArrayExtensions::FilterMap
  8. 27 attr_reader :resolvers, :options
  9. 27 def initialize(resolver_type, options)
  10. 7121 @current_selector = nil
  11. 7121 @current_session = nil
  12. 7121 @options = options
  13. 7121 @resolver_options = @options.resolver_options
  14. 7121 @resolvers = options.ip_families.map do |ip_family|
  15. 7121 resolver = resolver_type.new(ip_family, options)
  16. 7121 resolver.multi = self
  17. 7121 resolver
  18. end
  19. 7121 @errors = Hash.new { |hs, k| hs[k] = [] }
  20. end
  21. 27 def current_selector=(s)
  22. 7149 @current_selector = s
  23. 14298 @resolvers.each { |r| r.__send__(__method__, s) }
  24. end
  25. 27 def current_session=(s)
  26. 7149 @current_session = s
  27. 14298 @resolvers.each { |r| r.__send__(__method__, s) }
  28. end
  29. 27 def closed?
  30. @resolvers.all?(&:closed?)
  31. end
  32. 27 def empty?
  33. @resolvers.all?(&:empty?)
  34. end
  35. 27 def inflight?
  36. @resolvers.any(&:inflight?)
  37. end
  38. 27 def timeout
  39. @resolvers.filter_map(&:timeout).min
  40. end
  41. 27 def close
  42. @resolvers.each(&:close)
  43. end
  44. 27 def connections
  45. @resolvers.filter_map { |r| r.resolver_connection if r.respond_to?(:resolver_connection) }
  46. end
  47. 27 def early_resolve(connection)
  48. 7151 hostname = connection.peer.host
  49. 7151 addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
  50. 7151 return false unless addresses
  51. 6681 resolved = false
  52. 6946 addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
  53. # try to match the resolver by family. However, there are cases where that's not possible, as when
  54. # the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
  55. 13860 resolver = @resolvers.find { |r| r.family == family } || @resolvers.first
  56. 6930 next unless resolver # this should ever happen
  57. # it does not matter which resolver it is, as early-resolve code is shared.
  58. 6930 resolver.emit_addresses(connection, family, addrs, true)
  59. 6891 resolved = true
  60. end
  61. 6642 resolved
  62. end
  63. 27 def lazy_resolve(connection)
  64. 470 @resolvers.each do |resolver|
  65. 470 resolver << @current_session.try_clone_connection(connection, @current_selector, resolver.family)
  66. 458 next if resolver.empty?
  67. 374 @current_session.select_resolver(resolver, @current_selector)
  68. end
  69. end
  70. end
  71. end

lib/httpx/resolver/native.rb

89.0% lines covered

309 relevant lines. 275 lines covered and 34 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 require "resolv"
  4. 27 module HTTPX
  5. # Implements a pure ruby name resolver, which abides by the Selectable API.
  6. # It delegates DNS payload encoding/decoding to the +resolv+ stlid gem.
  7. #
  8. 27 class Resolver::Native < Resolver::Resolver
  9. 27 extend Forwardable
  10. 27 using URIExtensions
  11. 17 DEFAULTS = {
  12. 10 nameserver: nil,
  13. **Resolv::DNS::Config.default_config_hash,
  14. packet_size: 512,
  15. timeouts: Resolver::RESOLVE_TIMEOUT,
  16. }.freeze
  17. 27 DNS_PORT = 53
  18. 27 def_delegator :@connections, :empty?
  19. 27 attr_reader :state
  20. 27 def initialize(family, options)
  21. 7031 super
  22. 7031 @ns_index = 0
  23. 7031 @resolver_options = DEFAULTS.merge(@options.resolver_options)
  24. 7031 @socket_type = @resolver_options.fetch(:socket_type, :udp)
  25. 7031 @nameserver = if (nameserver = @resolver_options[:nameserver])
  26. 7025 nameserver = nameserver[family] if nameserver.is_a?(Hash)
  27. 7025 Array(nameserver)
  28. end
  29. 7031 @ndots = @resolver_options.fetch(:ndots, 1)
  30. 21093 @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
  31. 7031 @_timeouts = Array(@resolver_options[:timeouts])
  32. 8471 @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
  33. 7031 @name = nil
  34. 7031 @queries = {}
  35. 7031 @read_buffer = "".b
  36. 7031 @write_buffer = Buffer.new(@resolver_options[:packet_size])
  37. 7031 @state = :idle
  38. end
  39. 27 def close
  40. 372 transition(:closed)
  41. end
  42. 27 def terminate
  43. 16 emit(:close, self)
  44. end
  45. 27 def closed?
  46. 778 @state == :closed
  47. end
  48. 27 def to_io
  49. 1179 @io.to_io
  50. end
  51. 27 def call
  52. 978 case @state
  53. when :open
  54. 1046 consume
  55. end
  56. end
  57. 27 def interests
  58. 10646 case @state
  59. when :idle
  60. 9958 transition(:open)
  61. when :closed
  62. 12 transition(:idle)
  63. 12 transition(:open)
  64. end
  65. 10720 calculate_interests
  66. end
  67. 27 def <<(connection)
  68. 380 if @nameserver.nil?
  69. 6 ex = ResolveError.new("No available nameserver")
  70. 6 ex.set_backtrace(caller)
  71. 6 connection.force_reset
  72. 6 throw(:resolve_error, ex)
  73. else
  74. 374 @connections << connection
  75. 374 resolve
  76. end
  77. end
  78. 27 def timeout
  79. 10720 return if @connections.empty?
  80. 10720 @start_timeout = Utils.now
  81. 10720 hosts = @queries.keys
  82. 10720 @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
  83. end
  84. 27 def handle_socket_timeout(interval); end
  85. 27 private
  86. 27 def calculate_interests
  87. 15202 return :w unless @write_buffer.empty?
  88. 13466 return :r unless @queries.empty?
  89. 259 nil
  90. end
  91. 27 def consume
  92. 1058 loop do
  93. 1858 dread if calculate_interests == :r
  94. 1824 break unless calculate_interests == :w
  95. # do_retry
  96. 818 dwrite
  97. 800 break unless calculate_interests == :r
  98. end
  99. rescue Errno::EHOSTUNREACH => e
  100. 18 @ns_index += 1
  101. 18 nameserver = @nameserver
  102. 18 if nameserver && @ns_index < nameserver.size
  103. 12 log { "resolver #{FAMILY_TYPES[@record_type]}: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
  104. 12 transition(:idle)
  105. 12 @timeouts.clear
  106. 12 retry
  107. else
  108. 6 handle_error(e)
  109. 6 emit(:close, self)
  110. end
  111. rescue NativeResolveError => e
  112. 18 handle_error(e)
  113. 18 close_or_resolve
  114. 18 retry unless closed?
  115. end
  116. 27 def schedule_retry
  117. 800 h = @name
  118. 800 return unless h
  119. 800 connection = @queries[h]
  120. 800 timeouts = @timeouts[h]
  121. 800 timeout = timeouts.shift
  122. 800 @timer = @current_selector.after(timeout) do
  123. 72 next unless @connections.include?(connection)
  124. 72 do_retry(h, connection, timeout)
  125. end
  126. end
  127. 27 def do_retry(h, connection, interval)
  128. 72 timeouts = @timeouts[h]
  129. 72 if !timeouts.empty?
  130. 42 log { "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..." }
  131. # must downgrade to tcp AND retry on same host as last
  132. 42 downgrade_socket
  133. 42 resolve(connection, h)
  134. 30 elsif @ns_index + 1 < @nameserver.size
  135. # try on the next nameserver
  136. 6 @ns_index += 1
  137. 6 log do
  138. "resolver #{FAMILY_TYPES[@record_type]}: failed resolving #{h} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)"
  139. end
  140. 6 transition(:idle)
  141. 6 @timeouts.clear
  142. 6 resolve(connection, h)
  143. else
  144. 24 @timeouts.delete(h)
  145. 24 reset_hostname(h, reset_candidates: false)
  146. 24 unless @queries.empty?
  147. 18 resolve(connection)
  148. 18 return
  149. end
  150. 6 @connections.delete(connection)
  151. 6 host = connection.peer.host
  152. # This loop_time passed to the exception is bogus. Ideally we would pass the total
  153. # resolve timeout, including from the previous retries.
  154. 6 ex = ResolveTimeoutError.new(interval, "Timed out while resolving #{host}")
  155. 6 ex.set_backtrace(ex ? ex.backtrace : caller)
  156. 6 emit_resolve_error(connection, host, ex)
  157. 6 close_or_resolve
  158. end
  159. end
  160. 27 def dread(wsize = @resolver_options[:packet_size])
  161. 1396 loop do
  162. 1408 wsize = @large_packet.capacity if @large_packet
  163. 1408 siz = @io.read(wsize, @read_buffer)
  164. 1408 unless siz
  165. ex = EOFError.new("descriptor closed")
  166. ex.set_backtrace(caller)
  167. raise ex
  168. end
  169. 1408 return unless siz.positive?
  170. 740 if @socket_type == :tcp
  171. # packet may be incomplete, need to keep draining from the socket
  172. 30 if @large_packet
  173. # large packet buffer already exists, continue pumping
  174. 12 @large_packet << @read_buffer
  175. 12 next unless @large_packet.full?
  176. 12 parse(@large_packet.to_s)
  177. 12 @large_packet = nil
  178. # downgrade to udp again
  179. 12 downgrade_socket
  180. 12 return
  181. else
  182. 18 size = @read_buffer[0, 2].unpack1("n")
  183. 18 buffer = @read_buffer.byteslice(2..-1)
  184. 18 if size > @read_buffer.bytesize
  185. # only do buffer logic if it's worth it, and the whole packet isn't here already
  186. 12 @large_packet = Buffer.new(size)
  187. 12 @large_packet << buffer
  188. 12 next
  189. else
  190. 6 parse(buffer)
  191. end
  192. end
  193. else # udp
  194. 710 parse(@read_buffer)
  195. end
  196. 682 return if @state == :closed || !@write_buffer.empty?
  197. end
  198. end
  199. 27 def dwrite
  200. 800 loop do
  201. 1600 return if @write_buffer.empty?
  202. 800 siz = @io.write(@write_buffer)
  203. 800 unless siz
  204. ex = EOFError.new("descriptor closed")
  205. ex.set_backtrace(caller)
  206. raise ex
  207. end
  208. 800 return unless siz.positive?
  209. 800 schedule_retry if @write_buffer.empty?
  210. 800 return if @state == :closed
  211. end
  212. end
  213. 27 def parse(buffer)
  214. 728 @timer.cancel
  215. 728 code, result = Resolver.decode_dns_answer(buffer)
  216. 663 case code
  217. when :ok
  218. 232 parse_addresses(result)
  219. when :no_domain_found
  220. # Indicates no such domain was found.
  221. 472 hostname, connection = @queries.first
  222. 472 reset_hostname(hostname, reset_candidates: false)
  223. 826 other_candidate, _ = @queries.find { |_, conn| conn == connection }
  224. 472 if other_candidate
  225. 354 resolve(connection, other_candidate)
  226. else
  227. 118 @connections.delete(connection)
  228. 118 ex = NativeResolveError.new(connection, connection.peer.host, "name or service not known")
  229. 118 ex.set_backtrace(ex ? ex.backtrace : caller)
  230. 118 emit_resolve_error(connection, connection.peer.host, ex)
  231. 102 close_or_resolve
  232. end
  233. when :message_truncated
  234. # TODO: what to do if it's already tcp??
  235. 12 return if @socket_type == :tcp
  236. 12 @socket_type = :tcp
  237. 12 hostname, _ = @queries.first
  238. 12 reset_hostname(hostname)
  239. 12 transition(:closed)
  240. when :dns_error
  241. 6 hostname, connection = @queries.first
  242. 6 reset_hostname(hostname)
  243. 6 @connections.delete(connection)
  244. 6 ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
  245. 6 raise ex
  246. when :decode_error
  247. 6 hostname, connection = @queries.first
  248. 6 reset_hostname(hostname)
  249. 6 @connections.delete(connection)
  250. 6 ex = NativeResolveError.new(connection, connection.peer.host, result.message)
  251. 6 ex.set_backtrace(result.backtrace)
  252. 6 raise ex
  253. end
  254. end
  255. 27 def parse_addresses(addresses)
  256. 232 if addresses.empty?
  257. # no address found, eliminate candidates
  258. 6 hostname, connection = @queries.first
  259. 6 reset_hostname(hostname)
  260. 6 @connections.delete(connection)
  261. 6 raise NativeResolveError.new(connection, connection.peer.host)
  262. else
  263. 226 address = addresses.first
  264. 226 name = address["name"]
  265. 226 connection = @queries.delete(name)
  266. 226 unless connection
  267. orig_name = name
  268. # absolute name
  269. name_labels = Resolv::DNS::Name.create(name).to_a
  270. name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
  271. # probably a retried query for which there's an answer
  272. unless name
  273. @timeouts.delete(orig_name)
  274. return
  275. end
  276. address["name"] = name
  277. connection = @queries.delete(name)
  278. end
  279. 1260 alias_addresses, addresses = addresses.partition { |addr| addr.key?("alias") }
  280. 226 if addresses.empty? && !alias_addresses.empty? # CNAME
  281. hostname_alias = alias_addresses.first["alias"]
  282. # clean up intermediate queries
  283. @timeouts.delete(name) unless connection.peer.host == name
  284. if early_resolve(connection, hostname: hostname_alias)
  285. @connections.delete(connection)
  286. else
  287. if @socket_type == :tcp
  288. # must downgrade to udp if tcp
  289. @socket_type = @resolver_options.fetch(:socket_type, :udp)
  290. transition(:idle)
  291. transition(:open)
  292. end
  293. log { "resolver #{FAMILY_TYPES[@record_type]}: ALIAS #{hostname_alias} for #{name}" }
  294. resolve(connection, hostname_alias)
  295. return
  296. end
  297. else
  298. 226 reset_hostname(name, connection: connection)
  299. 226 @timeouts.delete(connection.peer.host)
  300. 226 @connections.delete(connection)
  301. 226 Resolver.cached_lookup_set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
  302. 1468 catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
  303. end
  304. end
  305. 226 close_or_resolve
  306. end
  307. 27 def resolve(connection = nil, hostname = nil)
  308. 808 @connections.shift until @connections.empty? || @connections.first.state != :closed
  309. 1198 connection ||= @connections.find { |c| !@queries.value?(c) }
  310. 808 raise Error, "no URI to resolve" unless connection
  311. 808 return unless @write_buffer.empty?
  312. 806 hostname ||= @queries.key(connection)
  313. 806 if hostname.nil?
  314. 386 hostname = connection.peer.host
  315. 386 if connection.peer.non_ascii_hostname
  316. log { "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" }
  317. end
  318. 386 hostname = generate_candidates(hostname).each do |name|
  319. 1416 @queries[name] = connection
  320. end.first
  321. else
  322. 387 @queries[hostname] = connection
  323. end
  324. 806 @name = hostname
  325. 806 log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
  326. 65 begin
  327. 806 @write_buffer << encode_dns_query(hostname)
  328. rescue Resolv::DNS::EncodeError => e
  329. reset_hostname(hostname, connection: connection)
  330. @connections.delete(connection)
  331. emit_resolve_error(connection, hostname, e)
  332. close_or_resolve
  333. end
  334. end
  335. 27 def encode_dns_query(hostname)
  336. 806 message_id = Resolver.generate_id
  337. 806 msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
  338. 806 msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
  339. 806 msg
  340. end
  341. 27 def generate_candidates(name)
  342. 386 return [name] if name.end_with?(".")
  343. 386 candidates = []
  344. 386 name_parts = name.scan(/[^.]+/)
  345. 386 candidates = [name] if @ndots <= name_parts.size - 1
  346. 1158 candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
  347. 386 fname = "#{name}."
  348. 386 candidates << fname unless candidates.include?(fname)
  349. 386 candidates
  350. end
  351. 27 def build_socket
  352. 402 ip, port = @nameserver[@ns_index]
  353. 402 port ||= DNS_PORT
  354. 370 case @socket_type
  355. when :udp
  356. 384 log { "resolver #{FAMILY_TYPES[@record_type]}: server: udp://#{ip}:#{port}..." }
  357. 384 UDP.new(ip, port, @options)
  358. when :tcp
  359. 18 log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." }
  360. 18 origin = URI("tcp://#{ip}:#{port}")
  361. 18 TCP.new(origin, [ip], @options)
  362. end
  363. end
  364. 27 def downgrade_socket
  365. 54 return unless @socket_type == :tcp
  366. 12 @socket_type = @resolver_options.fetch(:socket_type, :udp)
  367. 12 transition(:idle)
  368. 12 transition(:open)
  369. end
  370. 27 def transition(nextstate)
  371. 10344 case nextstate
  372. when :idle
  373. 42 if @io
  374. 36 @io.close
  375. 36 @io = nil
  376. end
  377. when :open
  378. 9982 return unless @state == :idle
  379. 9982 @io ||= build_socket
  380. 9982 @io.connect
  381. 9982 return unless @io.connected?
  382. 402 resolve if @queries.empty? && !@connections.empty?
  383. when :closed
  384. 384 return unless @state == :open
  385. 378 @io.close if @io
  386. 378 @start_timeout = nil
  387. 378 @write_buffer.clear
  388. 378 @read_buffer.clear
  389. end
  390. 822 @state = nextstate
  391. rescue Errno::ECONNREFUSED,
  392. Errno::EADDRNOTAVAIL,
  393. Errno::EHOSTUNREACH,
  394. SocketError,
  395. IOError,
  396. ConnectTimeoutError => e
  397. # these errors may happen during TCP handshake
  398. # treat them as resolve errors.
  399. handle_error(e)
  400. emit(:close, self)
  401. end
  402. 27 def handle_error(error)
  403. 24 if error.respond_to?(:connection) &&
  404. error.respond_to?(:host)
  405. 18 reset_hostname(error.host, connection: error.connection)
  406. 18 @connections.delete(error.connection)
  407. 18 emit_resolve_error(error.connection, error.host, error)
  408. else
  409. 6 @queries.each do |host, connection|
  410. 6 reset_hostname(host, connection: connection)
  411. 6 @connections.delete(connection)
  412. 6 emit_resolve_error(connection, host, error)
  413. end
  414. 12 while (connection = @connections.shift)
  415. emit_resolve_error(connection, connection.peer.host, error)
  416. end
  417. end
  418. end
  419. 27 def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
  420. 776 @timeouts.delete(hostname)
  421. 776 return unless connection && reset_candidates
  422. # eliminate other candidates
  423. 1072 candidates = @queries.select { |_, conn| connection == conn }.keys
  424. 1072 @queries.delete_if { |h, _| candidates.include?(h) }
  425. # reset timeouts
  426. 1048 @timeouts.delete_if { |h, _| candidates.include?(h) }
  427. end
  428. 27 def close_or_resolve
  429. # drop already closed connections
  430. 352 @connections.shift until @connections.empty? || @connections.first.state != :closed
  431. 352 if (@connections - @queries.values).empty?
  432. 350 emit(:close, self)
  433. else
  434. 2 resolve
  435. end
  436. end
  437. end
  438. end

lib/httpx/resolver/resolver.rb

84.52% lines covered

84 relevant lines. 71 lines covered and 13 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "ipaddr"
  4. 27 module HTTPX
  5. # Base class for all internal internet name resolvers. It handles basic blocks
  6. # from the Selectable API.
  7. #
  8. 27 class Resolver::Resolver
  9. 27 include Callbacks
  10. 27 include Loggable
  11. 27 using ArrayExtensions::Intersect
  12. 2 RECORD_TYPES = {
  13. 25 Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
  14. Socket::AF_INET => Resolv::DNS::Resource::IN::A,
  15. }.freeze
  16. 2 FAMILY_TYPES = {
  17. 25 Resolv::DNS::Resource::IN::AAAA => "AAAA",
  18. Resolv::DNS::Resource::IN::A => "A",
  19. }.freeze
  20. 27 class << self
  21. 27 def multi?
  22. 7121 true
  23. end
  24. end
  25. 27 attr_reader :family, :options
  26. 27 attr_writer :current_selector, :current_session
  27. 27 attr_accessor :multi
  28. 27 def initialize(family, options)
  29. 7146 @family = family
  30. 7146 @record_type = RECORD_TYPES[family]
  31. 7146 @options = options
  32. 7146 @connections = []
  33. 7146 set_resolver_callbacks
  34. end
  35. 27 def each_connection(&block)
  36. 244 enum_for(__method__) unless block
  37. 244 return unless @connections
  38. 244 @connections.each(&block)
  39. end
  40. 27 def close; end
  41. 27 alias_method :terminate, :close
  42. 27 def closed?
  43. true
  44. end
  45. 27 def empty?
  46. true
  47. end
  48. 27 def inflight?
  49. 16 false
  50. end
  51. 27 def emit_addresses(connection, family, addresses, early_resolve = false)
  52. 7228 addresses.map! do |address|
  53. 18702 address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
  54. end
  55. # double emission check, but allow early resolution to work
  56. 7228 return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
  57. 7228 log do
  58. 72 "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
  59. 6 "answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
  60. end
  61. 7228 if !early_resolve && # do not apply resolution delay for non-dns name resolution
  62. @current_selector && # just in case...
  63. family == Socket::AF_INET && # resolution delay only applies to IPv4
  64. !connection.io && # connection already has addresses and initiated/ended handshake
  65. connection.options.ip_families.size > 1 && # no need to delay if not supporting dual stack IP
  66. addresses.first.to_s != connection.peer.host.to_s # connection URL host is already the IP (early resolve included perhaps?)
  67. log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
  68. @current_selector.after(0.05) do
  69. # double emission check
  70. unless connection.addresses && addresses.intersect?(connection.addresses)
  71. emit_resolved_connection(connection, addresses, early_resolve)
  72. end
  73. end
  74. else
  75. 7228 emit_resolved_connection(connection, addresses, early_resolve)
  76. end
  77. end
  78. 27 private
  79. 27 def emit_resolved_connection(connection, addresses, early_resolve)
  80. begin
  81. 7228 connection.addresses = addresses
  82. 7182 return if connection.state == :closed
  83. 7182 emit(:resolve, connection)
  84. 24 rescue StandardError => e
  85. 46 if early_resolve
  86. 39 connection.force_reset
  87. 39 throw(:resolve_error, e)
  88. else
  89. 7 emit(:error, connection, e)
  90. end
  91. end
  92. end
  93. 27 def early_resolve(connection, hostname: connection.peer.host)
  94. addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
  95. return false unless addresses
  96. addresses = addresses.select { |addr| addr.family == @family }
  97. return false if addresses.empty?
  98. emit_addresses(connection, @family, addresses, true)
  99. true
  100. end
  101. 27 def emit_resolve_error(connection, hostname = connection.peer.host, ex = nil)
  102. 185 emit_connection_error(connection, resolve_error(hostname, ex))
  103. end
  104. 27 def resolve_error(hostname, ex = nil)
  105. 185 return ex if ex.is_a?(ResolveError) || ex.is_a?(ResolveTimeoutError)
  106. 42 message = ex ? ex.message : "Can't resolve #{hostname}"
  107. 42 error = ResolveError.new(message)
  108. 42 error.set_backtrace(ex ? ex.backtrace : caller)
  109. 42 error
  110. end
  111. 27 def set_resolver_callbacks
  112. 7146 on(:resolve, &method(:resolve_connection))
  113. 7146 on(:error, &method(:emit_connection_error))
  114. 7146 on(:close, &method(:close_resolver))
  115. end
  116. 27 def resolve_connection(connection)
  117. 7182 @current_session.__send__(:on_resolver_connection, connection, @current_selector)
  118. end
  119. 27 def emit_connection_error(connection, error)
  120. 179 return connection.handle_connect_error(error) if connection.connecting?
  121. connection.emit(:error, error)
  122. end
  123. 27 def close_resolver(resolver)
  124. 372 @current_session.__send__(:on_resolver_close, resolver, @current_selector)
  125. end
  126. end
  127. end

lib/httpx/resolver/system.rb

78.99% lines covered

138 relevant lines. 109 lines covered and 29 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 module HTTPX
  4. # Implementation of a synchronous name resolver which relies on the system resolver,
  5. # which is lib'c getaddrinfo function (abstracted in ruby via Addrinfo.getaddrinfo).
  6. #
  7. # Its main advantage is relying on the reference implementation for name resolution
  8. # across most/all OSs which deploy ruby (it's what TCPSocket also uses), its main
  9. # disadvantage is the inability to set timeouts / check socket for readiness events,
  10. # hence why it relies on using the Timeout module, which poses a lot of problems for
  11. # the selector loop, specially when network is unstable.
  12. #
  13. 27 class Resolver::System < Resolver::Resolver
  14. 27 using URIExtensions
  15. 27 RESOLV_ERRORS = [Resolv::ResolvError,
  16. Resolv::DNS::Requester::RequestError,
  17. Resolv::DNS::EncodeError,
  18. Resolv::DNS::DecodeError].freeze
  19. 27 DONE = 1
  20. 27 ERROR = 2
  21. 27 class << self
  22. 27 def multi?
  23. 25 false
  24. end
  25. end
  26. 27 attr_reader :state
  27. 27 def initialize(options)
  28. 25 super(0, options)
  29. 25 @resolver_options = @options.resolver_options
  30. 25 resolv_options = @resolver_options.dup
  31. 25 timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
  32. 25 @_timeouts = Array(timeouts)
  33. 50 @timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
  34. 25 resolv_options.delete(:cache)
  35. 25 @queries = []
  36. 25 @ips = []
  37. 25 @pipe_mutex = Thread::Mutex.new
  38. 25 @state = :idle
  39. end
  40. 27 def resolvers
  41. return enum_for(__method__) unless block_given?
  42. yield self
  43. end
  44. 27 def multi
  45. self
  46. end
  47. 27 def empty?
  48. true
  49. end
  50. 27 def close
  51. transition(:closed)
  52. end
  53. 27 def closed?
  54. @state == :closed
  55. end
  56. 27 def to_io
  57. @pipe_read.to_io
  58. end
  59. 27 def call
  60. case @state
  61. when :open
  62. consume
  63. end
  64. nil
  65. end
  66. 27 def interests
  67. return if @queries.empty?
  68. :r
  69. end
  70. 27 def timeout
  71. return unless @queries.empty?
  72. _, connection = @queries.first
  73. return unless connection
  74. @timeouts[connection.peer.host].first
  75. end
  76. 27 def <<(connection)
  77. 25 @connections << connection
  78. 25 resolve
  79. end
  80. 27 def early_resolve(connection, **)
  81. 25 self << connection
  82. 12 true
  83. end
  84. 27 def handle_socket_timeout(interval)
  85. error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
  86. error.set_backtrace(caller)
  87. @queries.each do |host, connection|
  88. @connections.delete(connection)
  89. emit_resolve_error(connection, host, error)
  90. end
  91. while (connection = @connections.shift)
  92. emit_resolve_error(connection, connection.peer.host, error)
  93. end
  94. end
  95. 27 private
  96. 27 def transition(nextstate)
  97. 25 case nextstate
  98. when :idle
  99. @timeouts.clear
  100. when :open
  101. 25 return unless @state == :idle
  102. 25 @pipe_read, @pipe_write = ::IO.pipe
  103. when :closed
  104. return unless @state == :open
  105. @pipe_write.close
  106. @pipe_read.close
  107. end
  108. 25 @state = nextstate
  109. end
  110. 27 def consume
  111. 25 return if @connections.empty?
  112. 25 if @pipe_read.wait_readable
  113. 25 event = @pipe_read.getbyte
  114. 25 case event
  115. when DONE
  116. 24 *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
  117. 12 if pair
  118. 12 @queries.delete(pair)
  119. 12 family, connection = pair
  120. 12 @connections.delete(connection)
  121. 24 catch(:coalesced) { emit_addresses(connection, family, addrs) }
  122. end
  123. when ERROR
  124. 26 *pair, error = @pipe_mutex.synchronize { @ips.pop }
  125. 13 if pair && error
  126. 13 @queries.delete(pair)
  127. 13 @connections.delete(connection)
  128. 13 _, connection = pair
  129. 13 emit_resolve_error(connection, connection.peer.host, error)
  130. end
  131. end
  132. end
  133. 12 return emit(:close, self) if @connections.empty?
  134. resolve
  135. end
  136. 27 def resolve(connection = nil, hostname = nil)
  137. 25 @connections.shift until @connections.empty? || @connections.first.state != :closed
  138. 25 connection ||= @connections.first
  139. 25 raise Error, "no URI to resolve" unless connection
  140. 25 return unless @queries.empty?
  141. 25 hostname ||= connection.peer.host
  142. 25 scheme = connection.origin.scheme
  143. log do
  144. "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
  145. 25 end if connection.peer.non_ascii_hostname
  146. 25 transition(:open)
  147. 25 connection.options.ip_families.each do |family|
  148. 25 @queries << [family, connection]
  149. end
  150. 25 async_resolve(connection, hostname, scheme)
  151. 25 consume
  152. end
  153. 27 def async_resolve(connection, hostname, scheme)
  154. 25 families = connection.options.ip_families
  155. 25 log { "resolver: query for #{hostname}" }
  156. 25 timeouts = @timeouts[connection.peer.host]
  157. 25 resolve_timeout = timeouts.first
  158. 25 Thread.start do
  159. 25 Thread.current.report_on_exception = false
  160. begin
  161. 25 addrs = if resolve_timeout
  162. 25 Timeout.timeout(resolve_timeout) do
  163. 25 __addrinfo_resolve(hostname, scheme)
  164. end
  165. else
  166. __addrinfo_resolve(hostname, scheme)
  167. end
  168. 12 addrs = addrs.sort_by(&:afamily).group_by(&:afamily)
  169. 12 families.each do |family|
  170. 12 addresses = addrs[family]
  171. 12 next unless addresses
  172. 12 addresses.map!(&:ip_address)
  173. 12 addresses.uniq!
  174. 12 @pipe_mutex.synchronize do
  175. 12 @ips.unshift([family, connection, addresses])
  176. 12 @pipe_write.putc(DONE) unless @pipe_write.closed?
  177. end
  178. end
  179. rescue StandardError => e
  180. 13 if e.is_a?(Timeout::Error)
  181. 1 timeouts.shift
  182. 1 retry unless timeouts.empty?
  183. 1 e = ResolveTimeoutError.new(resolve_timeout, e.message)
  184. 1 e.set_backtrace(e.backtrace)
  185. end
  186. 13 @pipe_mutex.synchronize do
  187. 13 families.each do |family|
  188. 13 @ips.unshift([family, connection, e])
  189. 13 @pipe_write.putc(ERROR) unless @pipe_write.closed?
  190. end
  191. end
  192. end
  193. end
  194. end
  195. 27 def __addrinfo_resolve(host, scheme)
  196. 25 Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
  197. end
  198. 27 def emit_connection_error(_, error)
  199. 13 throw(:resolve_error, error)
  200. end
  201. 27 def close_resolver(resolver); end
  202. end
  203. end

lib/httpx/response.rb

100.0% lines covered

115 relevant lines. 115 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "objspace"
  3. 27 require "stringio"
  4. 27 require "tempfile"
  5. 27 require "fileutils"
  6. 27 require "forwardable"
  7. 27 module HTTPX
  8. # Defines a HTTP response is handled internally, with a few properties exposed as attributes.
  9. #
  10. # It delegates the following methods to the corresponding HTTPX::Request:
  11. #
  12. # * HTTPX::Request#uri
  13. # * HTTPX::Request#peer_address
  14. #
  15. # It implements (indirectly, via the +body+) the IO write protocol to internally buffer payloads.
  16. #
  17. # It implements the IO reader protocol in order for users to buffer/stream it, acts as an enumerable
  18. # (of payload chunks).
  19. #
  20. 27 class Response
  21. 27 extend Forwardable
  22. 27 include Callbacks
  23. # the HTTP response status code
  24. 27 attr_reader :status
  25. # an HTTPX::Headers object containing the response HTTP headers.
  26. 27 attr_reader :headers
  27. # a HTTPX::Response::Body object wrapping the response body. The following methods are delegated to it:
  28. #
  29. # * HTTPX::Response::Body#to_s
  30. # * HTTPX::Response::Body#to_str
  31. # * HTTPX::Response::Body#read
  32. # * HTTPX::Response::Body#copy_to
  33. # * HTTPX::Response::Body#close
  34. 27 attr_reader :body
  35. # The HTTP protocol version used to fetch the response.
  36. 27 attr_reader :version
  37. # returns the response body buffered in a string.
  38. 27 def_delegator :@body, :to_s
  39. 27 def_delegator :@body, :to_str
  40. # implements the IO reader +#read+ interface.
  41. 27 def_delegator :@body, :read
  42. # copies the response body to a different location.
  43. 27 def_delegator :@body, :copy_to
  44. # the corresponding request uri.
  45. 27 def_delegator :@request, :uri
  46. # the IP address of the peer server.
  47. 27 def_delegator :@request, :peer_address
  48. # inits the instance with the corresponding +request+ to this response, an the
  49. # response HTTP +status+, +version+ and HTTPX::Headers instance of +headers+.
  50. 27 def initialize(request, status, version, headers)
  51. 9335 @request = request
  52. 9335 @options = request.options
  53. 9335 @version = version
  54. 9335 @status = Integer(status)
  55. 9335 @headers = @options.headers_class.new(headers)
  56. 9335 @body = @options.response_body_class.new(self, @options)
  57. 9335 @finished = complete?
  58. 9335 @content_type = nil
  59. end
  60. # dupped initialization
  61. 27 def initialize_dup(orig)
  62. 64 super
  63. # if a response gets dupped, the body handle must also get dupped to prevent
  64. # two responses from using the same file handle to read.
  65. 64 @body = orig.body.dup
  66. end
  67. # closes the respective +@request+ and +@body+.
  68. 27 def close
  69. 459 @request.close
  70. 459 @body.close
  71. end
  72. # merges headers defined in +h+ into the response headers.
  73. 27 def merge_headers(h)
  74. 193 @headers = @headers.merge(h)
  75. end
  76. # writes +data+ chunk into the response body.
  77. 27 def <<(data)
  78. 12354 @body.write(data)
  79. end
  80. # returns the HTTPX::ContentType for the response, as per what's declared in the content-type header.
  81. #
  82. # response.content_type #=> #<HTTPX::ContentType:xxx @header_value="text/plain">
  83. # response.content_type.mime_type #=> "text/plain"
  84. 27 def content_type
  85. 9704 @content_type ||= ContentType.new(@headers["content-type"])
  86. end
  87. # returns whether the response has been fully fetched.
  88. 27 def finished?
  89. 14540 @finished
  90. end
  91. # marks the response as finished, freezes the headers.
  92. 27 def finish!
  93. 8266 @finished = true
  94. 8266 @headers.freeze
  95. end
  96. # returns whether the response contains body payload.
  97. 27 def bodyless?
  98. 9335 @request.verb == "HEAD" ||
  99. @status < 200 || # informational response
  100. @status == 204 ||
  101. @status == 205 ||
  102. @status == 304 || begin
  103. 8889 content_length = @headers["content-length"]
  104. 8889 return false if content_length.nil?
  105. 7523 content_length == "0"
  106. end
  107. end
  108. 27 def complete?
  109. 9335 bodyless? || (@request.verb == "CONNECT" && @status == 200)
  110. end
  111. skipped # :nocov:
  112. skipped def inspect
  113. skipped "#<#{self.class}:#{object_id} " \
  114. skipped "HTTP/#{version} " \
  115. skipped "@status=#{@status} " \
  116. skipped "@headers=#{@headers} " \
  117. skipped "@body=#{@body.bytesize}>"
  118. skipped end
  119. skipped # :nocov:
  120. # returns an instance of HTTPX::HTTPError if the response has a 4xx or 5xx
  121. # status code, or nothing.
  122. #
  123. # ok_response.error #=> nil
  124. # not_found_response.error #=> HTTPX::HTTPError instance, status 404
  125. 27 def error
  126. 578 return if @status < 400
  127. 54 HTTPError.new(self)
  128. end
  129. # it raises the exception returned by +error+, or itself otherwise.
  130. #
  131. # ok_response.raise_for_status #=> ok_response
  132. # not_found_response.raise_for_status #=> raises HTTPX::HTTPError exception
  133. 27 def raise_for_status
  134. 538 return self unless (err = error)
  135. 38 raise err
  136. end
  137. # decodes the response payload into a ruby object **if** the payload is valid json.
  138. #
  139. # response.json #≈> { "foo" => "bar" } for "{\"foo\":\"bar\"}" payload
  140. # response.json(symbolize_names: true) #≈> { foo: "bar" } for "{\"foo\":\"bar\"}" payload
  141. 27 def json(*args)
  142. 129 decode(Transcoder::JSON, *args)
  143. end
  144. # decodes the response payload into a ruby object **if** the payload is valid
  145. # "application/x-www-urlencoded" or "multipart/form-data".
  146. 27 def form
  147. 64 decode(Transcoder::Form)
  148. end
  149. 27 def xml
  150. # TODO: remove at next major version.
  151. 8 warn "DEPRECATION WARNING: calling `.#{__method__}` on plain HTTPX responses is deprecated. " \
  152. 1 "Use HTTPX.plugin(:xml) sessions and call `.#{__method__}` in its responses instead."
  153. 8 require "httpx/plugins/xml"
  154. 8 decode(Plugins::XML::Transcoder)
  155. end
  156. 27 private
  157. # decodes the response payload using the given +transcoder+, which implements the decoding logic.
  158. #
  159. # +transcoder+ must implement the internal transcoder API, i.e. respond to <tt>decode(HTTPX::Response response)</tt>,
  160. # which returns a decoder which responds to <tt>call(HTTPX::Response response, **kwargs)</tt>
  161. 27 def decode(transcoder, *args)
  162. # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
  163. 217 decoder = transcoder.decode(self)
  164. 193 raise Error, "no decoder available for \"#{transcoder}\"" unless decoder
  165. 193 @body.rewind
  166. 193 decoder.call(self, *args)
  167. end
  168. end
  169. # Helper class which decodes the HTTP "content-type" header.
  170. 27 class ContentType
  171. 27 MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
  172. 27 CHARSET_RE = /;\s*charset=([^;]+)/i.freeze
  173. 27 def initialize(header_value)
  174. 9666 @header_value = header_value
  175. end
  176. # returns the mime type declared in the header.
  177. #
  178. # ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
  179. 27 def mime_type
  180. 217 return @mime_type if defined?(@mime_type)
  181. 179 m = @header_value.to_s[MIME_TYPE_RE, 1]
  182. 179 m && @mime_type = m.strip.downcase
  183. end
  184. # returns the charset declared in the header.
  185. #
  186. # ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
  187. # ContentType.new("text/plain").charset #=> nil
  188. 27 def charset
  189. 9487 return @charset if defined?(@charset)
  190. 9487 m = @header_value.to_s[CHARSET_RE, 1]
  191. 9487 m && @charset = m.strip.delete('"')
  192. end
  193. end
  194. # Wraps an error which has happened while processing an HTTP Request. It has partial
  195. # public API parity with HTTPX::Response, so users should rely on it to infer whether
  196. # the returned response is one or the other.
  197. #
  198. # response = HTTPX.get("https://some-domain/path") #=> response is HTTPX::Response or HTTPX::ErrorResponse
  199. # response.raise_for_status #=> raises if it wraps an error
  200. 27 class ErrorResponse
  201. 27 include Loggable
  202. 27 extend Forwardable
  203. # the corresponding HTTPX::Request instance.
  204. 27 attr_reader :request
  205. # the HTTPX::Response instance, when there is one (i.e. error happens fetching the response).
  206. 27 attr_reader :response
  207. # the wrapped exception.
  208. 27 attr_reader :error
  209. # the request uri
  210. 27 def_delegator :@request, :uri
  211. # the IP address of the peer server.
  212. 27 def_delegator :@request, :peer_address
  213. 27 def initialize(request, error)
  214. 1202 @request = request
  215. 1202 @response = request.response if request.response.is_a?(Response)
  216. 1202 @error = error
  217. 1202 @options = request.options
  218. 1202 log_exception(@error)
  219. end
  220. # returns the exception full message.
  221. 27 def to_s
  222. 8 @error.full_message(highlight: false)
  223. end
  224. # closes the error resources.
  225. 27 def close
  226. 40 @response.close if @response
  227. end
  228. # always true for error responses.
  229. 27 def finished?
  230. 1068 true
  231. end
  232. 27 def finish!; end
  233. # raises the wrapped exception.
  234. 27 def raise_for_status
  235. 82 raise @error
  236. end
  237. # buffers lost chunks to error response
  238. 27 def <<(data)
  239. 8 return unless @response
  240. 8 @response << data
  241. end
  242. end
  243. end
  244. 27 require_relative "response/body"
  245. 27 require_relative "response/buffer"
  246. 27 require_relative "pmatch_extensions" if RUBY_VERSION >= "2.7.0"

lib/httpx/response/body.rb

100.0% lines covered

109 relevant lines. 109 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # Implementation of the HTTP Response body as a buffer which implements the IO writer protocol
  4. # (for buffering the response payload), the IO reader protocol (for consuming the response payload),
  5. # and can be iterated over (via #each, which yields the payload in chunks).
  6. 27 class Response::Body
  7. # the payload encoding (i.e. "utf-8", "ASCII-8BIT")
  8. 27 attr_reader :encoding
  9. # Array of encodings contained in the response "content-encoding" header.
  10. 27 attr_reader :encodings
  11. 27 attr_reader :buffer
  12. 27 protected :buffer
  13. # initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
  14. 27 def initialize(response, options)
  15. 9487 @response = response
  16. 9487 @headers = response.headers
  17. 9487 @options = options
  18. 9487 @window_size = options.window_size
  19. 9487 @encodings = []
  20. 9487 @length = 0
  21. 9487 @buffer = nil
  22. 9487 @reader = nil
  23. 9487 @state = :idle
  24. # initialize response encoding
  25. 9487 @encoding = if (enc = response.content_type.charset)
  26. 182 begin
  27. 1614 Encoding.find(enc)
  28. rescue ArgumentError
  29. 32 Encoding::BINARY
  30. end
  31. else
  32. 7873 Encoding::BINARY
  33. end
  34. 9487 initialize_inflaters
  35. end
  36. 27 def initialize_dup(other)
  37. 96 super
  38. 96 @buffer = other.instance_variable_get(:@buffer).dup
  39. end
  40. 27 def closed?
  41. 52 @state == :closed
  42. end
  43. # write the response payload +chunk+ into the buffer. Inflates the chunk when required
  44. # and supported.
  45. 27 def write(chunk)
  46. 12357 return if @state == :closed
  47. 12357 return 0 if chunk.empty?
  48. 11937 chunk = decode_chunk(chunk)
  49. 11937 size = chunk.bytesize
  50. 10588 @length += size
  51. 11937 transition(:open)
  52. 11937 @buffer.write(chunk)
  53. 11937 @response.emit(:chunk_received, chunk)
  54. 11921 size
  55. end
  56. # reads a chunk from the payload (implementation of the IO reader protocol).
  57. 27 def read(*args)
  58. 409 return unless @buffer
  59. 409 unless @reader
  60. 187 rewind
  61. 187 @reader = @buffer
  62. end
  63. 409 @reader.read(*args)
  64. end
  65. # size of the decoded response payload. May differ from "content-length" header if
  66. # response was encoded over-the-wire.
  67. 27 def bytesize
  68. 222 @length
  69. end
  70. # yields the payload in chunks.
  71. 27 def each
  72. 48 return enum_for(__method__) unless block_given?
  73. 3 begin
  74. 32 if @buffer
  75. 32 rewind
  76. 84 while (chunk = @buffer.read(@window_size))
  77. 32 yield(chunk.force_encoding(@encoding))
  78. end
  79. end
  80. ensure
  81. 32 close
  82. end
  83. end
  84. # returns the declared filename in the "contennt-disposition" header, when present.
  85. 27 def filename
  86. 48 return unless @headers.key?("content-disposition")
  87. 40 Utils.get_filename(@headers["content-disposition"])
  88. end
  89. # returns the full response payload as a string.
  90. 27 def to_s
  91. 4635 return "".b unless @buffer
  92. 4306 @buffer.to_s
  93. end
  94. 27 alias_method :to_str, :to_s
  95. # whether the payload is empty.
  96. 27 def empty?
  97. 32 @length.zero?
  98. end
  99. # copies the payload to +dest+.
  100. #
  101. # body.copy_to("path/to/file")
  102. # body.copy_to(Pathname.new("path/to/file"))
  103. # body.copy_to(File.new("path/to/file"))
  104. 27 def copy_to(dest)
  105. 48 return unless @buffer
  106. 48 rewind
  107. 48 if dest.respond_to?(:path) && @buffer.respond_to?(:path)
  108. 8 FileUtils.mv(@buffer.path, dest.path)
  109. else
  110. 40 ::IO.copy_stream(@buffer, dest)
  111. end
  112. end
  113. # closes/cleans the buffer, resets everything
  114. 27 def close
  115. 719 if @buffer
  116. 555 @buffer.close
  117. 555 @buffer = nil
  118. end
  119. 719 @length = 0
  120. 719 transition(:closed)
  121. end
  122. 27 def ==(other)
  123. 294 super || case other
  124. when Response::Body
  125. 152 @buffer == other.buffer
  126. else
  127. 86 @buffer = other
  128. end
  129. end
  130. skipped # :nocov:
  131. skipped def inspect
  132. skipped "#<#{self.class}:#{object_id} " \
  133. skipped "@state=#{@state} " \
  134. skipped "@length=#{@length}>"
  135. skipped end
  136. skipped # :nocov:
  137. # rewinds the response payload buffer.
  138. 27 def rewind
  139. 900 return unless @buffer
  140. # in case there's some reading going on
  141. 900 @reader = nil
  142. 900 @buffer.rewind
  143. end
  144. 27 private
  145. # prepares inflaters for the advertised encodings in "content-encoding" header.
  146. 27 def initialize_inflaters
  147. 9487 @inflaters = nil
  148. 9487 return unless @headers.key?("content-encoding")
  149. 179 return unless @options.decompress_response_body
  150. 163 @inflaters = @headers.get("content-encoding").filter_map do |encoding|
  151. 163 next if encoding == "identity"
  152. 163 inflater = self.class.initialize_inflater_by_encoding(encoding, @response)
  153. # do not uncompress if there is no decoder available. In fact, we can't reliably
  154. # continue decompressing beyond that, so ignore.
  155. 163 break unless inflater
  156. 163 @encodings << encoding
  157. 163 inflater
  158. end
  159. end
  160. # passes the +chunk+ through all inflaters to decode it.
  161. 27 def decode_chunk(chunk)
  162. 46 @inflaters.reverse_each do |inflater|
  163. 411 chunk = inflater.call(chunk)
  164. 12238 end if @inflaters
  165. 12239 chunk
  166. end
  167. # tries transitioning the body STM to the +nextstate+.
  168. 27 def transition(nextstate)
  169. 11229 case nextstate
  170. when :open
  171. 11937 return unless @state == :idle
  172. 7517 @buffer = Response::Buffer.new(
  173. threshold_size: @options.body_threshold_size,
  174. bytesize: @length,
  175. encoding: @encoding
  176. )
  177. when :closed
  178. 719 return if @state == :closed
  179. end
  180. 8236 @state = nextstate
  181. end
  182. 27 class << self
  183. 27 def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
  184. 147 case encoding
  185. when "gzip"
  186. 147 Transcoder::GZIP.decode(response, **kwargs)
  187. when "deflate"
  188. 16 Transcoder::Deflate.decode(response, **kwargs)
  189. end
  190. end
  191. end
  192. end
  193. end

lib/httpx/response/buffer.rb

92.19% lines covered

64 relevant lines. 59 lines covered and 5 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "delegate"
  3. 27 require "stringio"
  4. 27 require "tempfile"
  5. 27 module HTTPX
  6. # wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
  7. 27 class Response::Buffer < SimpleDelegator
  8. 27 attr_reader :buffer
  9. 27 protected :buffer
  10. # initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
  11. # the initial +bytesize+, and the +encoding+.
  12. 27 def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
  13. 7713 @threshold_size = threshold_size
  14. 7713 @bytesize = bytesize
  15. 7713 @encoding = encoding
  16. 7713 @buffer = StringIO.new("".b)
  17. 7713 super(@buffer)
  18. end
  19. 27 def initialize_dup(other)
  20. 96 super
  21. # create new descriptor in READ-ONLY mode
  22. 12 @buffer =
  23. 83 case other.buffer
  24. when StringIO
  25. 96 StringIO.new(other.buffer.string, mode: File::RDONLY)
  26. else
  27. other.buffer.class.new(other.buffer.path, encoding: Encoding::BINARY, mode: File::RDONLY)
  28. end
  29. end
  30. # size in bytes of the buffered content.
  31. 27 def size
  32. 324 @bytesize
  33. end
  34. # writes the +chunk+ into the buffer.
  35. 27 def write(chunk)
  36. 10927 @bytesize += chunk.bytesize
  37. 12317 try_upgrade_buffer
  38. 12317 @buffer.write(chunk)
  39. end
  40. # returns the buffered content as a string.
  41. 27 def to_s
  42. 3882 case @buffer
  43. when StringIO
  44. 494 begin
  45. 4322 @buffer.string.force_encoding(@encoding)
  46. rescue ArgumentError
  47. @buffer.string
  48. end
  49. when Tempfile
  50. 72 rewind
  51. 72 content = @buffer.read
  52. 8 begin
  53. 72 content.force_encoding(@encoding)
  54. rescue ArgumentError # ex: unknown encoding name - utf
  55. content
  56. end
  57. end
  58. end
  59. # closes the buffer.
  60. 27 def close
  61. 667 @buffer.close
  62. 667 @buffer.unlink if @buffer.respond_to?(:unlink)
  63. end
  64. 27 def ==(other)
  65. 152 super || begin
  66. 152 return false unless other.is_a?(Response::Buffer)
  67. 152 if @buffer.nil?
  68. other.buffer.nil?
  69. 151 elsif @buffer.respond_to?(:read) &&
  70. other.respond_to?(:read)
  71. 152 buffer_pos = @buffer.pos
  72. 152 other_pos = other.buffer.pos
  73. 152 @buffer.rewind
  74. 152 other.buffer.rewind
  75. 18 begin
  76. 152 FileUtils.compare_stream(@buffer, other.buffer)
  77. ensure
  78. 152 @buffer.pos = buffer_pos
  79. 152 other.buffer.pos = other_pos
  80. end
  81. else
  82. to_s == other.to_s
  83. end
  84. end
  85. end
  86. 27 private
  87. # initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
  88. # has been reached.
  89. 27 def try_upgrade_buffer
  90. 12317 return unless @bytesize > @threshold_size
  91. 438 return if @buffer.is_a?(Tempfile)
  92. 163 aux = @buffer
  93. 163 @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  94. 163 if aux
  95. 163 aux.rewind
  96. 163 ::IO.copy_stream(aux, @buffer)
  97. 163 aux.close
  98. end
  99. 163 __setobj__(@buffer)
  100. end
  101. end
  102. end

lib/httpx/selector.rb

93.52% lines covered

108 relevant lines. 101 lines covered and 7 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "io/wait"
  3. 27 module HTTPX
  4. 27 class Selector
  5. 27 extend Forwardable
  6. 27 READABLE = %i[rw r].freeze
  7. 27 WRITABLE = %i[rw w].freeze
  8. 27 private_constant :READABLE
  9. 27 private_constant :WRITABLE
  10. 27 def_delegator :@timers, :after
  11. 27 def_delegator :@selectables, :empty?
  12. 27 def initialize
  13. 7705 @timers = Timers.new
  14. 7705 @selectables = []
  15. 7705 @is_timer_interval = false
  16. end
  17. 27 def each(&blk)
  18. @selectables.each(&blk)
  19. end
  20. 27 def next_tick
  21. 10727882 catch(:jump_tick) do
  22. 10727882 timeout = next_timeout
  23. 10727882 if timeout && timeout.negative?
  24. @timers.fire
  25. throw(:jump_tick)
  26. end
  27. 471212 begin
  28. 10727882 select(timeout) do |c|
  29. 23425 c.log(level: 2) { "[#{c.state}] selected#{" after #{timeout} secs" unless timeout.nil?}..." }
  30. 23312 c.call
  31. end
  32. 10727723 @timers.fire
  33. rescue TimeoutError => e
  34. @timers.fire(e)
  35. end
  36. end
  37. rescue StandardError => e
  38. 23 each_connection do |c|
  39. c.emit(:error, e)
  40. end
  41. rescue Exception # rubocop:disable Lint/RescueException
  42. 120 each_connection do |conn|
  43. 96 conn.force_reset
  44. 96 conn.disconnect
  45. end
  46. 120 raise
  47. end
  48. 27 def terminate
  49. # array may change during iteration
  50. 7212 selectables = @selectables.reject(&:inflight?)
  51. 7212 selectables.each(&:terminate)
  52. 7528 until selectables.empty?
  53. 2895 next_tick
  54. 2571 selectables &= @selectables
  55. end
  56. end
  57. 27 def find_resolver(options)
  58. 7176 res = @selectables.find do |c|
  59. 53 c.is_a?(Resolver::Resolver) && options == c.options
  60. end
  61. 7176 res.multi if res
  62. end
  63. 27 def each_connection(&block)
  64. 33915 return enum_for(__method__) unless block
  65. 17275 @selectables.each do |c|
  66. 2123 case c
  67. when Resolver::Resolver
  68. 244 c.each_connection(&block)
  69. when Connection
  70. 2011 yield c
  71. end
  72. end
  73. end
  74. 27 def find_connection(request_uri, options)
  75. 9268 each_connection.find do |connection|
  76. 1241 connection.match?(request_uri, options)
  77. end
  78. end
  79. 27 def find_mergeable_connection(connection)
  80. 7372 each_connection.find do |ch|
  81. 328 ch != connection && ch.mergeable?(connection)
  82. end
  83. end
  84. # deregisters +io+ from selectables.
  85. 27 def deregister(io)
  86. 8374 @selectables.delete(io)
  87. end
  88. # register +io+.
  89. 27 def register(io)
  90. 8862 return if @selectables.include?(io)
  91. 8417 @selectables << io
  92. end
  93. 27 private
  94. 27 def select(interval, &block)
  95. # do not cause an infinite loop here.
  96. #
  97. # this may happen if timeout calculation actually triggered an error which causes
  98. # the connections to be reaped (such as the total timeout error) before #select
  99. # gets called.
  100. 10727882 return if interval.nil? && @selectables.empty?
  101. 10725011 return select_one(interval, &block) if @selectables.size == 1
  102. 574 select_many(interval, &block)
  103. end
  104. 27 def select_many(interval, &block)
  105. 574 r, w = nil
  106. # first, we group IOs based on interest type. On call to #interests however,
  107. # things might already happen, and new IOs might be registered, so we might
  108. # have to start all over again. We do this until we group all selectables
  109. 574 @selectables.delete_if do |io|
  110. 838 interests = io.interests
  111. 838 io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
  112. 838 (r ||= []) << io if READABLE.include?(interests)
  113. 838 (w ||= []) << io if WRITABLE.include?(interests)
  114. 838 io.state == :closed
  115. end
  116. # TODO: what to do if there are no selectables?
  117. 574 readers, writers = IO.select(r, w, nil, interval)
  118. 574 if readers.nil? && writers.nil? && interval
  119. 162 [*r, *w].each { |io| io.handle_socket_timeout(interval) }
  120. 147 return
  121. end
  122. 412 if writers
  123. 33 readers.each do |io|
  124. 290 yield io
  125. # so that we don't yield 2 times
  126. 290 writers.delete(io)
  127. 411 end if readers
  128. 412 writers.each(&block)
  129. else
  130. readers.each(&block) if readers
  131. end
  132. end
  133. 27 def select_one(interval)
  134. 10724437 io = @selectables.first
  135. 10724437 return unless io
  136. 10724437 interests = io.interests
  137. 10724561 io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
  138. 10724436 result = case interests
  139. 13594 when :r then io.to_io.wait_readable(interval)
  140. 9830 when :w then io.to_io.wait_writable(interval)
  141. when :rw then io.to_io.wait(interval, :read_write)
  142. 10701012 when nil then return
  143. end
  144. 23424 unless result || interval.nil?
  145. 536 io.handle_socket_timeout(interval) unless @is_timer_interval
  146. 477 return
  147. end
  148. # raise TimeoutError.new(interval, "timed out while waiting on select")
  149. 22888 yield io
  150. # rescue IOError, SystemCallError
  151. # @selectables.reject!(&:closed?)
  152. # raise unless @selectables.empty?
  153. end
  154. 27 def next_timeout
  155. 10727882 @is_timer_interval = false
  156. 10727882 timer_interval = @timers.wait_interval
  157. 10727882 connection_interval = @selectables.filter_map(&:timeout).min
  158. 10727882 return connection_interval unless timer_interval
  159. 10701895 if connection_interval.nil? || timer_interval <= connection_interval
  160. 10701857 @is_timer_interval = true
  161. 10520214 return timer_interval
  162. end
  163. 38 connection_interval
  164. end
  165. end
  166. end

lib/httpx/session.rb

95.6% lines covered

273 relevant lines. 261 lines covered and 12 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # Class implementing the APIs being used publicly.
  4. #
  5. # HTTPX.get(..) #=> delegating to an internal HTTPX::Session object.
  6. # HTTPX.plugin(..).get(..) #=> creating an intermediate HTTPX::Session with plugin, then sending the GET request
  7. 27 class Session
  8. 27 include Loggable
  9. 27 include Chainable
  10. # initializes the session with a set of +options+, which will be shared by all
  11. # requests sent from it.
  12. #
  13. # When pass a block, it'll yield itself to it, then closes after the block is evaluated.
  14. 27 def initialize(options = EMPTY_HASH, &blk)
  15. 11143 @options = self.class.default_options.merge(options)
  16. 11143 @persistent = @options.persistent
  17. 11143 @pool = @options.pool_class.new(@options.pool_options)
  18. 11143 @wrapped = false
  19. 11143 @closing = false
  20. 11143 INSTANCES[self] = self if @persistent && @options.close_on_fork && INSTANCES
  21. 11143 wrap(&blk) if blk
  22. end
  23. # Yields itself the block, then closes it after the block is evaluated.
  24. #
  25. # session.wrap do |http|
  26. # http.get("https://wikipedia.com")
  27. # end # wikipedia connection closes here
  28. 27 def wrap
  29. 583 prev_wrapped = @wrapped
  30. 583 @wrapped = true
  31. 583 was_initialized = false
  32. 583 current_selector = get_current_selector do
  33. 583 selector = Selector.new
  34. 583 set_current_selector(selector)
  35. 583 was_initialized = true
  36. 583 selector
  37. end
  38. 58 begin
  39. 583 yield self
  40. ensure
  41. 583 unless prev_wrapped
  42. 583 if @persistent
  43. 1 deactivate(current_selector)
  44. else
  45. 582 close(current_selector)
  46. end
  47. end
  48. 583 @wrapped = prev_wrapped
  49. 583 set_current_selector(nil) if was_initialized
  50. end
  51. end
  52. # closes all the active connections from the session.
  53. #
  54. # when called directly without specifying +selector+, all available connections
  55. # will be picked up from the connection pool and closed. Connections in use
  56. # by other sessions, or same session in a different thread, will not be reaped.
  57. 27 def close(selector = Selector.new)
  58. # throw resolvers away from the pool
  59. 7212 @pool.reset_resolvers
  60. # preparing to throw away connections
  61. 16695 while (connection = @pool.pop_connection)
  62. 4420 next if connection.state == :closed
  63. 212 select_connection(connection, selector)
  64. end
  65. 813 begin
  66. 7212 @closing = true
  67. 7212 selector.terminate
  68. ensure
  69. 7212 @closing = false
  70. end
  71. end
  72. # performs one, or multple requests; it accepts:
  73. #
  74. # 1. one or multiple HTTPX::Request objects;
  75. # 2. an HTTP verb, then a sequence of URIs or URI/options tuples;
  76. # 3. one or multiple HTTP verb / uri / (optional) options tuples;
  77. #
  78. # when present, the set of +options+ kwargs is applied to all of the
  79. # sent requests.
  80. #
  81. # respectively returns a single HTTPX::Response response, or all of them in an Array, in the same order.
  82. #
  83. # resp1 = session.request(req1)
  84. # resp1, resp2 = session.request(req1, req2)
  85. # resp1 = session.request("GET", "https://server.org/a")
  86. # resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"])
  87. # resp1, resp2 = session.request(["GET", "https://server.org/a"], ["GET", "https://server.org/b"])
  88. # resp1 = session.request("POST", "https://server.org/a", form: { "foo" => "bar" })
  89. # resp1, resp2 = session.request(["POST", "https://server.org/a", form: { "foo" => "bar" }], ["GET", "https://server.org/b"])
  90. # resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"], headers: { "x-api-token" => "TOKEN" })
  91. #
  92. 27 def request(*args, **params)
  93. 7504 raise ArgumentError, "must perform at least one request" if args.empty?
  94. 7504 requests = args.first.is_a?(Request) ? args : build_requests(*args, params)
  95. 7467 responses = send_requests(*requests)
  96. 7317 return responses.first if responses.size == 1
  97. 199 responses
  98. end
  99. # returns a HTTP::Request instance built from the HTTP +verb+, the request +uri+, and
  100. # the optional set of request-specific +options+. This request **must** be sent through
  101. # the same session it was built from.
  102. #
  103. # req = session.build_request("GET", "https://server.com")
  104. # resp = session.request(req)
  105. 27 def build_request(verb, uri, params = EMPTY_HASH, options = @options)
  106. 9194 rklass = options.request_class
  107. 9194 request = rklass.new(verb, uri, options, params)
  108. 9157 request.persistent = @persistent
  109. 9157 set_request_callbacks(request)
  110. 9157 request
  111. end
  112. 27 def select_connection(connection, selector)
  113. 8830 pin_connection(connection, selector)
  114. 8830 selector.register(connection)
  115. end
  116. 27 def pin_connection(connection, selector)
  117. 8849 connection.current_session = self
  118. 8849 connection.current_selector = selector
  119. end
  120. 27 alias_method :select_resolver, :select_connection
  121. 27 def deselect_connection(connection, selector, cloned = false)
  122. 7986 selector.deregister(connection)
  123. # when connections coalesce
  124. 7986 return if connection.state == :idle
  125. 7962 return if cloned
  126. 7954 return if @closing && connection.state == :closed
  127. 7946 @pool.checkin_connection(connection)
  128. end
  129. 27 def deselect_resolver(resolver, selector)
  130. 372 selector.deregister(resolver)
  131. 372 return if @closing && resolver.closed?
  132. 372 @pool.checkin_resolver(resolver)
  133. end
  134. 27 def try_clone_connection(connection, selector, family)
  135. 470 connection.family ||= family
  136. 470 return connection if connection.family == family
  137. new_connection = connection.class.new(connection.origin, connection.options)
  138. new_connection.family = family
  139. connection.sibling = new_connection
  140. do_init_connection(new_connection, selector)
  141. new_connection
  142. end
  143. # returns the HTTPX::Connection through which the +request+ should be sent through.
  144. 27 def find_connection(request_uri, selector, options)
  145. 9268 if (connection = selector.find_connection(request_uri, options))
  146. 1187 connection.idling if connection.state == :closed
  147. 1187 connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
  148. 1120 return connection
  149. end
  150. 8081 connection = @pool.checkout_connection(request_uri, options)
  151. 8109 connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
  152. 7150 case connection.state
  153. when :idle
  154. 7342 do_init_connection(connection, selector)
  155. when :open
  156. 66 if options.io
  157. 66 select_connection(connection, selector)
  158. else
  159. pin_connection(connection, selector)
  160. end
  161. when :closing, :closed
  162. 622 connection.idling
  163. 622 select_connection(connection, selector)
  164. else
  165. 19 pin_connection(connection, selector)
  166. end
  167. 7985 connection
  168. end
  169. 27 private
  170. 27 def deactivate(selector)
  171. 492 selector.each_connection do |connection|
  172. 366 connection.deactivate
  173. 366 deselect_connection(connection, selector) if connection.state == :inactive
  174. end
  175. end
  176. # callback executed when an HTTP/2 promise frame has been received.
  177. 27 def on_promise(_, stream)
  178. 8 log(level: 2) { "#{stream.id}: refusing stream!" }
  179. 8 stream.refuse
  180. end
  181. # returns the corresponding HTTP::Response to the given +request+ if it has been received.
  182. 27 def fetch_response(request, _selector, _options)
  183. 10733231 response = request.response
  184. 10733231 response if response && response.finished?
  185. end
  186. # sends the +request+ to the corresponding HTTPX::Connection
  187. 27 def send_request(request, selector, options = request.options)
  188. 1919 error = begin
  189. 9187 catch(:resolve_error) do
  190. 9187 connection = find_connection(request.uri, selector, options)
  191. 9067 connection.send(request)
  192. end
  193. rescue StandardError => e
  194. 40 e
  195. end
  196. 9181 return unless error && error.is_a?(Exception)
  197. 120 raise error unless error.is_a?(Error)
  198. 120 response = ErrorResponse.new(request, error)
  199. 120 request.response = response
  200. 120 request.emit(:response, response)
  201. end
  202. # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
  203. 27 def build_requests(*args, params)
  204. 6902 requests = if args.size == 1
  205. 78 reqs = args.first
  206. 78 reqs.map do |verb, uri, ps = EMPTY_HASH|
  207. 156 request_params = params
  208. 156 request_params = request_params.merge(ps) unless ps.empty?
  209. 156 build_request(verb, uri, request_params)
  210. end
  211. else
  212. 6824 verb, uris = args
  213. 6824 if uris.respond_to?(:each)
  214. 6584 uris.enum_for(:each).map do |uri, ps = EMPTY_HASH|
  215. 7342 request_params = params
  216. 7342 request_params = request_params.merge(ps) unless ps.empty?
  217. 7342 build_request(verb, uri, request_params)
  218. end
  219. else
  220. 240 [build_request(verb, uris, params)]
  221. end
  222. end
  223. 6865 raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
  224. 6865 requests
  225. end
  226. 27 def set_request_callbacks(request)
  227. 9030 request.on(:promise, &method(:on_promise))
  228. end
  229. 27 def do_init_connection(connection, selector)
  230. 7342 resolve_connection(connection, selector) unless connection.family
  231. end
  232. # sends an array of HTTPX::Request +requests+, returns the respective array of HTTPX::Response objects.
  233. 27 def send_requests(*requests)
  234. 14407 selector = get_current_selector { Selector.new }
  235. 835 begin
  236. 7565 _send_requests(requests, selector)
  237. 7559 receive_requests(requests, selector)
  238. ensure
  239. 7549 unless @wrapped
  240. 6841 if @persistent
  241. 491 deactivate(selector)
  242. else
  243. 6350 close(selector)
  244. end
  245. end
  246. end
  247. end
  248. # sends an array of HTTPX::Request objects
  249. 27 def _send_requests(requests, selector)
  250. 7565 requests.each do |request|
  251. 8386 send_request(request, selector)
  252. end
  253. end
  254. # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
  255. 27 def receive_requests(requests, selector)
  256. # @type var responses: Array[response]
  257. 7559 responses = []
  258. # guarantee ordered responses
  259. 7559 loop do
  260. 8388 request = requests.first
  261. 8388 return responses unless request
  262. 10915193 catch(:coalesced) { selector.next_tick } until (response = fetch_response(request, selector, request.options))
  263. 8244 request.emit(:complete, response)
  264. 8244 responses << response
  265. 8244 requests.shift
  266. 8244 break if requests.empty?
  267. 829 next unless selector.empty?
  268. # in some cases, the pool of connections might have been drained because there was some
  269. # handshake error, and the error responses have already been emitted, but there was no
  270. # opportunity to traverse the requests, hence we're returning only a fraction of the errors
  271. # we were supposed to. This effectively fetches the existing responses and return them.
  272. while (request = requests.shift)
  273. response = fetch_response(request, selector, request.options)
  274. request.emit(:complete, response) if response
  275. responses << response
  276. end
  277. break
  278. end
  279. 7415 responses
  280. end
  281. 27 def resolve_connection(connection, selector)
  282. 7366 if connection.addresses || connection.open?
  283. #
  284. # there are two cases in which we want to activate initialization of
  285. # connection immediately:
  286. #
  287. # 1. when the connection already has addresses, i.e. it doesn't need to
  288. # resolve a name (not the same as name being an IP, yet)
  289. # 2. when the connection is initialized with an external already open IO.
  290. #
  291. 190 on_resolver_connection(connection, selector)
  292. 188 return
  293. end
  294. 7176 resolver = find_resolver_for(connection, selector)
  295. 7176 resolver.early_resolve(connection) || resolver.lazy_resolve(connection)
  296. end
  297. 27 def on_resolver_connection(connection, selector)
  298. 7372 from_pool = false
  299. 7372 found_connection = selector.find_mergeable_connection(connection) || begin
  300. 7345 from_pool = true
  301. 7345 @pool.checkout_mergeable_connection(connection)
  302. end
  303. 7372 return select_connection(connection, selector) unless found_connection
  304. 27 connection.log(level: 2) do
  305. "try coalescing from #{from_pool ? "pool##{@pool.object_id}" : "selector##{selector.object_id}"} " \
  306. "(conn##{found_connection.object_id}[#{found_connection.origin}])"
  307. end
  308. 27 coalesce_connections(found_connection, connection, selector, from_pool)
  309. end
  310. 27 def on_resolver_close(resolver, selector)
  311. 372 return if resolver.closed?
  312. 372 deselect_resolver(resolver, selector)
  313. 372 resolver.close unless resolver.closed?
  314. end
  315. 27 def find_resolver_for(connection, selector)
  316. 7176 resolver = selector.find_resolver(connection.options)
  317. 7176 unless resolver
  318. 7174 resolver = @pool.checkout_resolver(connection.options)
  319. 7174 resolver.current_session = self
  320. 7174 resolver.current_selector = selector
  321. end
  322. 7176 resolver
  323. end
  324. # coalesces +conn2+ into +conn1+. if +conn1+ was loaded from the connection pool
  325. # (it is known via +from_pool+), then it adds its to the +selector+.
  326. 27 def coalesce_connections(conn1, conn2, selector, from_pool)
  327. 27 unless conn1.coalescable?(conn2)
  328. 14 conn2.log(level: 2) { "not coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
  329. 14 select_connection(conn2, selector)
  330. 14 @pool.checkin_connection(conn1) if from_pool
  331. 14 return false
  332. end
  333. 13 conn2.log(level: 2) { "coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
  334. 13 conn2.coalesce!(conn1)
  335. 13 select_connection(conn1, selector) if from_pool
  336. 13 conn2.disconnect
  337. 13 true
  338. end
  339. 27 def get_current_selector
  340. 8148 selector_store[self] || (yield if block_given?)
  341. end
  342. 27 def set_current_selector(selector)
  343. 1641 if selector
  344. 956 selector_store[self] = selector
  345. else
  346. 583 selector_store.delete(self)
  347. end
  348. end
  349. 27 def selector_store
  350. 9789 th_current = Thread.current
  351. 9789 th_current.thread_variable_get(:httpx_persistent_selector_store) || begin
  352. 163 {}.compare_by_identity.tap do |store|
  353. 163 th_current.thread_variable_set(:httpx_persistent_selector_store, store)
  354. end
  355. end
  356. end
  357. 27 @default_options = Options.new
  358. 27 @default_options.freeze
  359. 27 @plugins = []
  360. 27 class << self
  361. 27 attr_reader :default_options
  362. 27 def inherited(klass)
  363. 5615 super
  364. 5615 klass.instance_variable_set(:@default_options, @default_options)
  365. 5615 klass.instance_variable_set(:@plugins, @plugins.dup)
  366. 5615 klass.instance_variable_set(:@callbacks, @callbacks.dup)
  367. end
  368. # returns a new HTTPX::Session instance, with the plugin pointed by +pl+ loaded.
  369. #
  370. # session_with_retries = session.plugin(:retries)
  371. # session_with_custom = session.plugin(CustomPlugin)
  372. #
  373. 27 def plugin(pl, options = nil, &block)
  374. 7641 label = pl
  375. # raise Error, "Cannot add a plugin to a frozen config" if frozen?
  376. 7641 pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
  377. 7641 if !@plugins.include?(pl)
  378. 7411 @plugins << pl
  379. 7411 pl.load_dependencies(self, &block) if pl.respond_to?(:load_dependencies)
  380. 7411 @default_options = @default_options.dup
  381. 7411 include(pl::InstanceMethods) if defined?(pl::InstanceMethods)
  382. 7411 extend(pl::ClassMethods) if defined?(pl::ClassMethods)
  383. 7411 opts = @default_options
  384. 7411 opts.extend_with_plugin_classes(pl)
  385. 7411 if defined?(pl::OptionsMethods)
  386. 2997 (pl::OptionsMethods.instance_methods - Object.instance_methods).each do |meth|
  387. 8911 opts.options_class.method_added(meth)
  388. end
  389. 2997 @default_options = opts.options_class.new(opts)
  390. end
  391. 7411 @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
  392. 7411 @default_options = @default_options.merge(options) if options
  393. 7411 if pl.respond_to?(:subplugins)
  394. 32 pl.subplugins.transform_keys(&Plugins.method(:load_plugin)).each do |main_pl, sub_pl|
  395. # in case the main plugin has already been loaded, then apply subplugin functionality
  396. # immediately
  397. 32 next unless @plugins.include?(main_pl)
  398. 8 plugin(sub_pl, options, &block)
  399. end
  400. end
  401. 7411 pl.configure(self, &block) if pl.respond_to?(:configure)
  402. 7411 if label.is_a?(Symbol)
  403. # in case an already-loaded plugin complements functionality of
  404. # the plugin currently being loaded, loaded it now
  405. 5656 @plugins.each do |registered_pl|
  406. 13783 next if registered_pl == pl
  407. 8127 next unless registered_pl.respond_to?(:subplugins)
  408. 16 sub_pl = registered_pl.subplugins[label]
  409. 16 next unless sub_pl
  410. 16 plugin(sub_pl, options, &block)
  411. end
  412. end
  413. 7411 @default_options.freeze
  414. 7411 set_temporary_name("#{superclass}/#{pl}") if respond_to?(:set_temporary_name) # ruby 3.4 only
  415. 229 elsif options
  416. # this can happen when two plugins are loaded, an one of them calls the other under the hood,
  417. # albeit changing some default.
  418. 16 @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
  419. 16 @default_options = @default_options.merge(options) if options
  420. 16 @default_options.freeze
  421. end
  422. 7641 self
  423. end
  424. end
  425. # setup of the support for close_on_fork sessions.
  426. # adapted from https://github.com/mperham/connection_pool/blob/main/lib/connection_pool.rb#L48
  427. 27 if Process.respond_to?(:fork)
  428. 25 INSTANCES = ObjectSpace::WeakMap.new
  429. 25 private_constant :INSTANCES
  430. 25 def self.after_fork
  431. 1 INSTANCES.each_value(&:close)
  432. 1 nil
  433. end
  434. 25 if ::Process.respond_to?(:_fork)
  435. 21 module ForkTracker
  436. 21 def _fork
  437. 1 pid = super
  438. 1 Session.after_fork if pid.zero?
  439. 1 pid
  440. end
  441. end
  442. 21 Process.singleton_class.prepend(ForkTracker)
  443. end
  444. else
  445. 2 INSTANCES = nil
  446. 2 private_constant :INSTANCES
  447. 2 def self.after_fork
  448. # noop
  449. end
  450. end
  451. end
  452. # session may be overridden by certain adapters.
  453. 27 S = Session
  454. end

lib/httpx/session_extensions.rb

100.0% lines covered

14 relevant lines. 14 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 unless ENV.keys.grep(/\Ahttps?_proxy\z/i).empty?
  4. 1 proxy_session = plugin(:proxy)
  5. 1 remove_const(:Session)
  6. 1 const_set(:Session, proxy_session.class)
  7. # redefine the default options static var, which needs to
  8. # refresh options_class
  9. 1 options = proxy_session.class.default_options.to_hash
  10. 1 original_verbosity = $VERBOSE
  11. 1 $VERBOSE = nil
  12. 1 const_set(:Options, proxy_session.class.default_options.options_class)
  13. 1 options[:options_class] = Class.new(options[:options_class])
  14. 1 options.freeze
  15. 1 Options.send(:const_set, :DEFAULT_OPTIONS, options)
  16. 1 Session.instance_variable_set(:@default_options, Options.new(options))
  17. 1 $VERBOSE = original_verbosity
  18. end
  19. skipped # :nocov:
  20. skipped if Session.default_options.debug_level > 2
  21. skipped proxy_session = plugin(:internal_telemetry)
  22. skipped remove_const(:Session)
  23. skipped const_set(:Session, proxy_session.class)
  24. skipped end
  25. skipped # :nocov:
  26. end

lib/httpx/timers.rb

93.94% lines covered

66 relevant lines. 62 lines covered and 4 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 class Timers
  4. 27 def initialize
  5. 7705 @intervals = []
  6. end
  7. 27 def after(interval_in_secs, cb = nil, &blk)
  8. 39326 callback = cb || blk
  9. 39326 raise Error, "timer must have a callback" unless callback
  10. # I'm assuming here that most requests will have the same
  11. # request timeout, as in most cases they share common set of
  12. # options. A user setting different request timeouts for 100s of
  13. # requests will already have a hard time dealing with that.
  14. 70746 unless (interval = @intervals.bsearch { |t| t.interval == interval_in_secs })
  15. 9454 interval = Interval.new(interval_in_secs)
  16. 9454 @intervals << interval
  17. 9454 @intervals.sort!
  18. end
  19. 39326 interval << callback
  20. 39326 @next_interval_at = nil
  21. 39326 Timer.new(interval, callback)
  22. end
  23. 27 def wait_interval
  24. 10727882 drop_elapsed!
  25. 10727882 return if @intervals.empty?
  26. 10701895 @next_interval_at = Utils.now
  27. 10701895 @intervals.first.interval
  28. end
  29. 27 def fire(error = nil)
  30. 10727723 raise error if error && error.timeout != @intervals.first
  31. 10727723 return if @intervals.empty? || !@next_interval_at
  32. 10701201 elapsed_time = Utils.elapsed_time(@next_interval_at)
  33. 10701201 drop_elapsed!(elapsed_time)
  34. 21395011 @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
  35. 10701201 @next_interval_at = nil if @intervals.empty?
  36. end
  37. 27 private
  38. 27 def drop_elapsed!(elapsed_time = 0)
  39. # check first, if not elapsed, then return
  40. 21429083 first_interval = @intervals.first
  41. 21429083 return unless first_interval && first_interval.elapsed?(elapsed_time)
  42. # TODO: would be nice to have a drop_while!
  43. 17037 @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
  44. end
  45. 27 class Timer
  46. 27 def initialize(interval, callback)
  47. 39326 @interval = interval
  48. 39326 @callback = callback
  49. end
  50. 27 def cancel
  51. 57802 @interval.delete(@callback)
  52. end
  53. end
  54. 27 class Interval
  55. 27 include Comparable
  56. 27 attr_reader :interval
  57. 27 def initialize(interval)
  58. 9454 @interval = interval
  59. 9454 @callbacks = []
  60. end
  61. 27 def <=>(other)
  62. 822 @interval <=> other.interval
  63. end
  64. 27 def ==(other)
  65. return @interval == other if other.is_a?(Numeric)
  66. @interval == other.to_f # rubocop:disable Lint/FloatComparison
  67. end
  68. 27 def to_f
  69. Float(@interval)
  70. end
  71. 27 def <<(callback)
  72. 39326 @callbacks << callback
  73. end
  74. 27 def delete(callback)
  75. 57802 @callbacks.delete(callback)
  76. end
  77. 27 def no_callbacks?
  78. @callbacks.empty?
  79. end
  80. 27 def elapsed?(elapsed = 0)
  81. 21403767 (@interval - elapsed) <= 0 || @callbacks.empty?
  82. end
  83. 27 def elapse(elapsed)
  84. # same as elapsing
  85. 10702637 return 0 if @callbacks.empty?
  86. 10513533 @interval -= elapsed
  87. 10694586 if @interval <= 0
  88. 652 cb = @callbacks.dup
  89. 652 cb.each(&:call)
  90. end
  91. 10694586 @interval
  92. end
  93. end
  94. 27 private_constant :Interval
  95. end
  96. end

lib/httpx/transcoder.rb

100.0% lines covered

52 relevant lines. 52 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Transcoder
  4. 27 module_function
  5. 27 def normalize_keys(key, value, cond = nil, &block)
  6. 3415 if cond && cond.call(value)
  7. 1065 block.call(key.to_s, value)
  8. 2349 elsif value.respond_to?(:to_ary)
  9. 454 if value.empty?
  10. 128 block.call("#{key}[]")
  11. else
  12. 326 value.to_ary.each do |element|
  13. 524 normalize_keys("#{key}[]", element, cond, &block)
  14. end
  15. end
  16. 1895 elsif value.respond_to?(:to_hash)
  17. 512 value.to_hash.each do |child_key, child_value|
  18. 512 normalize_keys("#{key}[#{child_key}]", child_value, cond, &block)
  19. end
  20. else
  21. 1384 block.call(key.to_s, value)
  22. end
  23. end
  24. # based on https://github.com/rack/rack/blob/d15dd728440710cfc35ed155d66a98dc2c07ae42/lib/rack/query_parser.rb#L82
  25. 27 def normalize_query(params, name, v, depth)
  26. 184 raise Error, "params depth surpasses what's supported" if depth <= 0
  27. 184 name =~ /\A[\[\]]*([^\[\]]+)\]*/
  28. 184 k = Regexp.last_match(1) || ""
  29. 184 after = Regexp.last_match ? Regexp.last_match.post_match : ""
  30. 184 if k.empty?
  31. 16 return Array(v) if !v.empty? && name == "[]"
  32. 7 return
  33. end
  34. 147 case after
  35. when ""
  36. 49 params[k] = v
  37. when "["
  38. 7 params[name] = v
  39. when "[]"
  40. 16 params[k] ||= []
  41. 16 raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
  42. 16 params[k] << v
  43. when /^\[\]\[([^\[\]]+)\]$/, /^\[\](.+)$/
  44. 32 child_key = Regexp.last_match(1)
  45. 32 params[k] ||= []
  46. 32 raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
  47. 32 if params[k].last.is_a?(Hash) && !params_hash_has_key?(params[k].last, child_key)
  48. 8 normalize_query(params[k].last, child_key, v, depth - 1)
  49. else
  50. 24 params[k] << normalize_query({}, child_key, v, depth - 1)
  51. end
  52. else
  53. 56 params[k] ||= {}
  54. 56 raise Error, "expected Hash (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Hash)
  55. 49 params[k] = normalize_query(params[k], after, v, depth - 1)
  56. end
  57. 168 params
  58. end
  59. 27 def params_hash_has_key?(hash, key)
  60. 16 return false if key.include?("[]")
  61. 16 key.split(/[\[\]]+/).inject(hash) do |h, part|
  62. 16 next h if part == ""
  63. 16 return false unless h.is_a?(Hash) && h.key?(part)
  64. 8 h[part]
  65. end
  66. 8 true
  67. end
  68. end
  69. end
  70. 27 require "httpx/transcoder/body"
  71. 27 require "httpx/transcoder/form"
  72. 27 require "httpx/transcoder/json"
  73. 27 require "httpx/transcoder/chunker"
  74. 27 require "httpx/transcoder/deflate"
  75. 27 require "httpx/transcoder/gzip"

lib/httpx/transcoder/body.rb

100.0% lines covered

26 relevant lines. 26 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "delegate"
  3. 27 module HTTPX::Transcoder
  4. 27 module Body
  5. 27 class Error < HTTPX::Error; end
  6. 27 module_function
  7. 27 class Encoder < SimpleDelegator
  8. 27 def initialize(body)
  9. 1448 body = body.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && body.is_a?(Pathname)
  10. 1448 @body = body
  11. 1448 super(body)
  12. end
  13. 27 def bytesize
  14. 5704 if @body.respond_to?(:bytesize)
  15. 2550 @body.bytesize
  16. 3153 elsif @body.respond_to?(:to_ary)
  17. 998 @body.sum(&:bytesize)
  18. 2155 elsif @body.respond_to?(:size)
  19. 1500 @body.size || Float::INFINITY
  20. 655 elsif @body.respond_to?(:length)
  21. 360 @body.length || Float::INFINITY
  22. 295 elsif @body.respond_to?(:each)
  23. 288 Float::INFINITY
  24. else
  25. 8 raise Error, "cannot determine size of body: #{@body.inspect}"
  26. end
  27. end
  28. 27 def content_type
  29. 1370 "application/octet-stream"
  30. end
  31. end
  32. 27 def encode(body)
  33. 1448 Encoder.new(body)
  34. end
  35. end
  36. end

lib/httpx/transcoder/chunker.rb

100.0% lines covered

66 relevant lines. 66 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 module HTTPX::Transcoder
  4. 27 module Chunker
  5. 27 class Error < HTTPX::Error; end
  6. 27 CRLF = "\r\n".b
  7. 27 class Encoder
  8. 27 extend Forwardable
  9. 27 def initialize(body)
  10. 96 @raw = body
  11. end
  12. 27 def each
  13. 96 return enum_for(__method__) unless block_given?
  14. 96 @raw.each do |chunk|
  15. 448 yield "#{chunk.bytesize.to_s(16)}#{CRLF}#{chunk}#{CRLF}"
  16. end
  17. 96 yield "0#{CRLF}"
  18. end
  19. 27 def respond_to_missing?(meth, *args)
  20. 108 @raw.respond_to?(meth, *args) || super
  21. end
  22. end
  23. 27 class Decoder
  24. 27 extend Forwardable
  25. 27 def_delegator :@buffer, :empty?
  26. 27 def_delegator :@buffer, :<<
  27. 27 def_delegator :@buffer, :clear
  28. 27 def initialize(buffer, trailers = false)
  29. 115 @buffer = buffer
  30. 115 @chunk_buffer = "".b
  31. 115 @finished = false
  32. 115 @state = :length
  33. 115 @trailers = trailers
  34. end
  35. 27 def to_s
  36. 107 @buffer
  37. end
  38. 27 def each
  39. 200 loop do
  40. 1055 case @state
  41. when :length
  42. 344 index = @buffer.index(CRLF)
  43. 344 return unless index && index.positive?
  44. # Read hex-length
  45. 344 hexlen = @buffer.byteslice(0, index)
  46. 344 @buffer = @buffer.byteslice(index..-1) || "".b
  47. 344 hexlen[/\h/] || raise(Error, "wrong chunk size line: #{hexlen}")
  48. 344 @chunk_length = hexlen.hex
  49. # check if is last chunk
  50. 344 @finished = @chunk_length.zero?
  51. 344 nextstate(:crlf)
  52. when :crlf
  53. 573 crlf_size = @finished && !@trailers ? 4 : 2
  54. # consume CRLF
  55. 573 return if @buffer.bytesize < crlf_size
  56. 573 raise Error, "wrong chunked encoding format" unless @buffer.start_with?(CRLF * (crlf_size / 2))
  57. 573 @buffer = @buffer.byteslice(crlf_size..-1)
  58. 573 if @chunk_length.nil?
  59. 229 nextstate(:length)
  60. else
  61. 344 return if @finished
  62. 237 nextstate(:data)
  63. end
  64. when :data
  65. 285 chunk = @buffer.byteslice(0, @chunk_length)
  66. 285 @buffer = @buffer.byteslice(@chunk_length..-1) || "".b
  67. 285 @chunk_buffer << chunk
  68. 250 @chunk_length -= chunk.bytesize
  69. 285 if @chunk_length.zero?
  70. 237 yield @chunk_buffer unless @chunk_buffer.empty?
  71. 229 @chunk_buffer.clear
  72. 229 @chunk_length = nil
  73. 229 nextstate(:crlf)
  74. end
  75. end
  76. 1087 break if @buffer.empty?
  77. end
  78. end
  79. 27 def finished?
  80. 192 @finished
  81. end
  82. 27 private
  83. 27 def nextstate(state)
  84. 1039 @state = state
  85. end
  86. end
  87. 27 module_function
  88. 27 def encode(chunks)
  89. 96 Encoder.new(chunks)
  90. end
  91. end
  92. end

lib/httpx/transcoder/deflate.rb

100.0% lines covered

20 relevant lines. 20 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "zlib"
  3. 27 require_relative "utils/deflater"
  4. 27 module HTTPX
  5. 27 module Transcoder
  6. 27 module Deflate
  7. 27 class Deflater < Transcoder::Deflater
  8. 27 def deflate(chunk)
  9. 72 @deflater ||= Zlib::Deflate.new
  10. 72 if chunk.nil?
  11. 48 unless @deflater.closed?
  12. 24 last = @deflater.finish
  13. 24 @deflater.close
  14. 24 last.empty? ? nil : last
  15. end
  16. else
  17. 24 @deflater.deflate(chunk)
  18. end
  19. end
  20. end
  21. 27 module_function
  22. 27 def encode(body)
  23. 24 Deflater.new(body)
  24. end
  25. 27 def decode(response, bytesize: nil)
  26. 16 bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
  27. 16 GZIP::Inflater.new(bytesize)
  28. end
  29. end
  30. end
  31. end

lib/httpx/transcoder/form.rb

100.0% lines covered

42 relevant lines. 42 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 require "uri"
  4. 27 require_relative "multipart"
  5. 27 module HTTPX
  6. 27 module Transcoder
  7. 27 module Form
  8. 27 module_function
  9. 27 PARAM_DEPTH_LIMIT = 32
  10. 27 class Encoder
  11. 27 extend Forwardable
  12. 27 def_delegator :@raw, :to_s
  13. 27 def_delegator :@raw, :to_str
  14. 27 def_delegator :@raw, :bytesize
  15. 27 def_delegator :@raw, :==
  16. 27 def initialize(form)
  17. 706 @raw = form.each_with_object("".b) do |(key, val), buf|
  18. 1186 HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
  19. 1384 buf << "&" unless buf.empty?
  20. 1384 buf << URI.encode_www_form_component(k)
  21. 1384 buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
  22. end
  23. end
  24. end
  25. 27 def content_type
  26. 554 "application/x-www-form-urlencoded"
  27. end
  28. end
  29. 27 module Decoder
  30. 27 module_function
  31. 27 def call(response, *)
  32. 40 URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
  33. 96 HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
  34. end
  35. end
  36. end
  37. 27 def encode(form)
  38. 1657 if multipart?(form)
  39. 951 Multipart::Encoder.new(form)
  40. else
  41. 706 Encoder.new(form)
  42. end
  43. end
  44. 27 def decode(response)
  45. 64 content_type = response.content_type.mime_type
  46. 56 case content_type
  47. when "application/x-www-form-urlencoded"
  48. 40 Decoder
  49. when "multipart/form-data"
  50. 16 Multipart::Decoder.new(response)
  51. else
  52. 8 raise Error, "invalid form mime type (#{content_type})"
  53. end
  54. end
  55. 27 def multipart?(data)
  56. 1657 data.any? do |_, v|
  57. 2201 Multipart::MULTIPART_VALUE_COND.call(v) ||
  58. 1698 (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
  59. 2082 (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
  60. end
  61. end
  62. end
  63. end
  64. end

lib/httpx/transcoder/gzip.rb

100.0% lines covered

40 relevant lines. 40 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "zlib"
  3. 27 module HTTPX
  4. 27 module Transcoder
  5. 27 module GZIP
  6. 27 class Deflater < Transcoder::Deflater
  7. 27 def initialize(body)
  8. 48 @compressed_chunk = "".b
  9. 48 super
  10. end
  11. 27 def deflate(chunk)
  12. 96 @deflater ||= Zlib::GzipWriter.new(self)
  13. 96 if chunk.nil?
  14. 48 unless @deflater.closed?
  15. 48 @deflater.flush
  16. 48 @deflater.close
  17. 48 compressed_chunk
  18. end
  19. else
  20. 48 @deflater.write(chunk)
  21. 48 compressed_chunk
  22. end
  23. end
  24. 27 private
  25. 27 def write(chunk)
  26. 144 @compressed_chunk << chunk
  27. end
  28. 27 def compressed_chunk
  29. 96 @compressed_chunk.dup
  30. ensure
  31. 96 @compressed_chunk.clear
  32. end
  33. end
  34. 27 class Inflater
  35. 27 def initialize(bytesize)
  36. 163 @inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
  37. 163 @bytesize = bytesize
  38. end
  39. 27 def call(chunk)
  40. 411 buffer = @inflater.inflate(chunk)
  41. 365 @bytesize -= chunk.bytesize
  42. 411 if @bytesize <= 0
  43. 107 buffer << @inflater.finish
  44. 107 @inflater.close
  45. end
  46. 411 buffer
  47. end
  48. end
  49. 27 module_function
  50. 27 def encode(body)
  51. 48 Deflater.new(body)
  52. end
  53. 27 def decode(response, bytesize: nil)
  54. 147 bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
  55. 147 Inflater.new(bytesize)
  56. end
  57. end
  58. end
  59. end

lib/httpx/transcoder/json.rb

100.0% lines covered

33 relevant lines. 33 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 module HTTPX::Transcoder
  4. 27 module JSON
  5. 27 module_function
  6. 27 JSON_REGEX = %r{
  7. \b
  8. application/
  9. # optional vendor specific type
  10. (?:
  11. # token as per https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6
  12. [!#$%&'*+\-.^_`|~0-9a-z]+
  13. # literal plus sign
  14. \+
  15. )?
  16. json
  17. \b
  18. }ix.freeze
  19. 27 class Encoder
  20. 27 extend Forwardable
  21. 27 def_delegator :@raw, :to_s
  22. 27 def_delegator :@raw, :bytesize
  23. 27 def_delegator :@raw, :==
  24. 27 def initialize(json)
  25. 83 @raw = JSON.json_dump(json)
  26. 83 @charset = @raw.encoding.name.downcase
  27. end
  28. 27 def content_type
  29. 83 "application/json; charset=#{@charset}"
  30. end
  31. end
  32. 27 def encode(json)
  33. 83 Encoder.new(json)
  34. end
  35. 27 def decode(response)
  36. 129 content_type = response.content_type.mime_type
  37. 129 raise HTTPX::Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type)
  38. 113 method(:json_load)
  39. end
  40. # rubocop:disable Style/SingleLineMethods
  41. 27 if defined?(MultiJson)
  42. 4 def json_load(*args); MultiJson.load(*args); end
  43. 2 def json_dump(*args); MultiJson.dump(*args); end
  44. 25 elsif defined?(Oj)
  45. 5 def json_load(response, *args); Oj.load(response.to_s, *args); end
  46. 3 def json_dump(obj, options = {}); Oj.dump(obj, { mode: :compat }.merge(options)); end
  47. 23 elsif defined?(Yajl)
  48. 4 def json_load(response, *args); Yajl::Parser.new(*args).parse(response.to_s); end
  49. 2 def json_dump(*args); Yajl::Encoder.encode(*args); end
  50. else
  51. 23 require "json"
  52. 114 def json_load(*args); ::JSON.parse(*args); end
  53. 93 def json_dump(*args); ::JSON.dump(*args); end
  54. end
  55. # rubocop:enable Style/SingleLineMethods
  56. end
  57. end

lib/httpx/transcoder/multipart.rb

100.0% lines covered

10 relevant lines. 10 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require_relative "multipart/encoder"
  3. 27 require_relative "multipart/decoder"
  4. 27 require_relative "multipart/part"
  5. 27 require_relative "multipart/mime_type_detector"
  6. 27 module HTTPX::Transcoder
  7. 27 module Multipart
  8. 27 MULTIPART_VALUE_COND = lambda do |value|
  9. 4878 value.respond_to?(:read) ||
  10. 3506 (value.respond_to?(:to_hash) &&
  11. value.key?(:body) &&
  12. 644 (value.key?(:filename) || value.key?(:content_type)))
  13. end
  14. end
  15. end

lib/httpx/transcoder/multipart/decoder.rb

94.05% lines covered

84 relevant lines. 79 lines covered and 5 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "tempfile"
  3. 27 require "delegate"
  4. 27 module HTTPX
  5. 27 module Transcoder
  6. 27 module Multipart
  7. 27 class FilePart < SimpleDelegator
  8. 27 attr_reader :original_filename, :content_type
  9. 27 def initialize(filename, content_type)
  10. 32 @original_filename = filename
  11. 32 @content_type = content_type
  12. 32 @current = nil
  13. 32 @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  14. 32 super(@file)
  15. end
  16. end
  17. 27 class Decoder
  18. 27 include HTTPX::Utils
  19. 27 CRLF = "\r\n"
  20. 27 BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
  21. 27 MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
  22. 27 MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
  23. 27 MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
  24. 27 WINDOW_SIZE = 2 << 14
  25. 27 def initialize(response)
  26. 2 @boundary = begin
  27. 16 m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
  28. 16 raise Error, "no boundary declared in content-type header" unless m
  29. 16 m.strip
  30. end
  31. 16 @buffer = "".b
  32. 16 @parts = {}
  33. 16 @intermediate_boundary = "--#{@boundary}"
  34. 16 @state = :idle
  35. end
  36. 27 def call(response, *)
  37. 16 response.body.each do |chunk|
  38. 16 @buffer << chunk
  39. 16 parse
  40. end
  41. 16 raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
  42. 16 @parts
  43. end
  44. 27 private
  45. 27 def parse
  46. 14 case @state
  47. when :idle
  48. 16 raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
  49. 16 @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
  50. 16 @state = :part_header
  51. when :part_header
  52. 48 idx = @buffer.index("#{CRLF}#{CRLF}")
  53. # raise Error, "couldn't parse part headers" unless idx
  54. 48 return unless idx
  55. # @type var head: String
  56. 48 head = @buffer.byteslice(0..idx + 4 - 1)
  57. 48 @buffer = @buffer.byteslice(head.bytesize..-1)
  58. 48 content_type = head[MULTIPART_CONTENT_TYPE, 1] || "text/plain"
  59. 84 if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
  60. 48 name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
  61. 48 name.gsub!(/\\(.)/, "\\1")
  62. 12 name
  63. else
  64. name = head[MULTIPART_CONTENT_ID, 1]
  65. end
  66. 48 filename = HTTPX::Utils.get_filename(head)
  67. 48 name = filename || +"#{content_type}[]" if name.nil? || name.empty?
  68. 48 @current = name
  69. 42 @parts[name] = if filename
  70. 32 FilePart.new(filename, content_type)
  71. else
  72. 16 "".b
  73. end
  74. 48 @state = :part_body
  75. when :part_body
  76. 48 part = @parts[@current]
  77. 48 body_separator = if part.is_a?(FilePart)
  78. 28 "#{CRLF}#{CRLF}"
  79. else
  80. 16 CRLF
  81. end
  82. 48 idx = @buffer.index(body_separator)
  83. 48 if idx
  84. 48 payload = @buffer.byteslice(0..idx - 1)
  85. 48 @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
  86. 48 part << payload
  87. 48 part.rewind if part.respond_to?(:rewind)
  88. 48 @state = :parse_boundary
  89. else
  90. part << @buffer
  91. @buffer.clear
  92. end
  93. when :parse_boundary
  94. 48 raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
  95. 48 @buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
  96. 48 if @buffer == "--"
  97. 16 @buffer.clear
  98. 16 @state = :done
  99. 16 return
  100. 31 elsif @buffer.start_with?(CRLF)
  101. 32 @buffer = @buffer.byteslice(2..-1)
  102. 32 @state = :part_header
  103. else
  104. return
  105. end
  106. when :done
  107. raise Error, "parsing should have been over by now"
  108. 20 end until @buffer.empty?
  109. end
  110. end
  111. end
  112. end
  113. end

lib/httpx/transcoder/multipart/encoder.rb

100.0% lines covered

70 relevant lines. 70 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Transcoder::Multipart
  4. 27 class Encoder
  5. 27 attr_reader :bytesize
  6. 27 def initialize(form)
  7. 951 @boundary = ("-" * 21) << SecureRandom.hex(21)
  8. 951 @part_index = 0
  9. 951 @buffer = "".b
  10. 951 @form = form
  11. 951 @bytesize = 0
  12. 951 @parts = to_parts(form)
  13. end
  14. 27 def content_type
  15. 951 "multipart/form-data; boundary=#{@boundary}"
  16. end
  17. 27 def to_s
  18. 18 read || ""
  19. ensure
  20. 18 rewind
  21. end
  22. 27 def read(length = nil, outbuf = nil)
  23. 3518 data = String(outbuf).clear.force_encoding(Encoding::BINARY) if outbuf
  24. 3518 data ||= "".b
  25. 3518 read_chunks(data, length)
  26. 3518 data unless length && data.empty?
  27. end
  28. 27 def rewind
  29. 50 form = @form.each_with_object([]) do |(key, val), aux|
  30. 50 if val.respond_to?(:path) && val.respond_to?(:reopen) && val.respond_to?(:closed?) && val.closed?
  31. # @type var val: File
  32. 50 val = val.reopen(val.path, File::RDONLY)
  33. end
  34. 50 val.rewind if val.respond_to?(:rewind)
  35. 50 aux << [key, val]
  36. end
  37. 50 @form = form
  38. 50 @bytesize = 0
  39. 50 @parts = to_parts(form)
  40. 50 @part_index = 0
  41. end
  42. 27 private
  43. 27 def to_parts(form)
  44. 1001 params = form.each_with_object([]) do |(key, val), aux|
  45. 1193 Transcoder.normalize_keys(key, val, MULTIPART_VALUE_COND) do |k, v|
  46. 1193 next if v.nil?
  47. 1193 value, content_type, filename = Part.call(v)
  48. 1193 header = header_part(k, content_type, filename)
  49. 1049 @bytesize += header.size
  50. 1193 aux << header
  51. 1049 @bytesize += value.size
  52. 1193 aux << value
  53. 1193 delimiter = StringIO.new("\r\n")
  54. 1049 @bytesize += delimiter.size
  55. 1193 aux << delimiter
  56. end
  57. end
  58. 1001 final_delimiter = StringIO.new("--#{@boundary}--\r\n")
  59. 881 @bytesize += final_delimiter.size
  60. 1001 params << final_delimiter
  61. 1001 params
  62. end
  63. 27 def header_part(key, content_type, filename)
  64. 1193 header = "--#{@boundary}\r\n".b
  65. 1193 header << "Content-Disposition: form-data; name=#{key.inspect}".b
  66. 1193 header << "; filename=#{filename.inspect}" if filename
  67. 1193 header << "\r\nContent-Type: #{content_type}\r\n\r\n"
  68. 1193 StringIO.new(header)
  69. end
  70. 27 def read_chunks(buffer, length = nil)
  71. 4618 while @part_index < @parts.size
  72. 10584 chunk = read_from_part(length)
  73. 10584 next unless chunk
  74. 6052 buffer << chunk.force_encoding(Encoding::BINARY)
  75. 6052 next unless length
  76. 5246 length -= chunk.bytesize
  77. 5986 break if length.zero?
  78. end
  79. end
  80. # if there's a current part to read from, tries to read a chunk.
  81. 27 def read_from_part(max_length = nil)
  82. 10584 part = @parts[@part_index]
  83. 10584 chunk = part.read(max_length, @buffer)
  84. 10584 return chunk if chunk && !chunk.empty?
  85. 4532 part.close if part.respond_to?(:close)
  86. 3980 @part_index += 1
  87. 1692 nil
  88. end
  89. end
  90. end
  91. end

lib/httpx/transcoder/multipart/mime_type_detector.rb

92.11% lines covered

38 relevant lines. 35 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Transcoder::Multipart
  4. 27 module MimeTypeDetector
  5. 27 module_function
  6. 27 DEFAULT_MIMETYPE = "application/octet-stream"
  7. # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
  8. 27 if defined?(FileMagic)
  9. 1 MAGIC_NUMBER = 256 * 1024
  10. 1 def call(file, _)
  11. 1 return nil if file.eof? # FileMagic returns "application/x-empty" for empty files
  12. 1 mime = FileMagic.open(FileMagic::MAGIC_MIME_TYPE) do |filemagic|
  13. 1 filemagic.buffer(file.read(MAGIC_NUMBER))
  14. end
  15. 1 file.rewind
  16. 1 mime
  17. end
  18. 25 elsif defined?(Marcel)
  19. 1 def call(file, filename)
  20. 1 return nil if file.eof? # marcel returns "application/octet-stream" for empty files
  21. 1 Marcel::MimeType.for(file, name: filename)
  22. end
  23. 24 elsif defined?(MimeMagic)
  24. 1 def call(file, _)
  25. 1 mime = MimeMagic.by_magic(file)
  26. 1 mime.type if mime
  27. end
  28. 23 elsif system("which file", out: File::NULL)
  29. 24 require "open3"
  30. 24 def call(file, _)
  31. 677 return if file.eof? # file command returns "application/x-empty" for empty files
  32. 633 Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
  33. 75 begin
  34. 633 ::IO.copy_stream(file, stdin.binmode)
  35. rescue Errno::EPIPE
  36. end
  37. 633 file.rewind
  38. 633 stdin.close
  39. 633 status = thread.value
  40. # call to file command failed
  41. 633 if status.nil? || !status.success?
  42. $stderr.print(stderr.read)
  43. else
  44. 633 output = stdout.read.strip
  45. 633 if output.include?("cannot open")
  46. $stderr.print(output)
  47. else
  48. 633 output
  49. end
  50. end
  51. end
  52. end
  53. else
  54. def call(_, _); end
  55. end
  56. end
  57. end
  58. end

lib/httpx/transcoder/multipart/part.rb

100.0% lines covered

18 relevant lines. 18 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Transcoder::Multipart
  4. 27 module Part
  5. 27 module_function
  6. 27 def call(value)
  7. # take out specialized objects of the way
  8. 1193 if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
  9. 112 return value, value.content_type, value.filename
  10. end
  11. 1065 content_type = filename = nil
  12. 1065 if value.is_a?(Hash)
  13. 322 content_type = value[:content_type]
  14. 322 filename = value[:filename]
  15. 322 value = value[:body]
  16. end
  17. 1065 value = value.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
  18. 1065 if value.respond_to?(:path) && value.respond_to?(:read)
  19. # either a File, a Tempfile, or something else which has to quack like a file
  20. 681 filename ||= File.basename(value.path)
  21. 681 content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
  22. 681 [value, content_type, filename]
  23. else
  24. 384 [StringIO.new(value.to_s), content_type || "text/plain", filename]
  25. end
  26. end
  27. end
  28. end
  29. end

lib/httpx/transcoder/utils/body_reader.rb

92.0% lines covered

25 relevant lines. 23 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "stringio"
  3. 27 module HTTPX
  4. 27 module Transcoder
  5. 27 class BodyReader
  6. 27 def initialize(body)
  7. 210 @body = if body.respond_to?(:read)
  8. 20 body.rewind if body.respond_to?(:rewind)
  9. 20 body
  10. 189 elsif body.respond_to?(:each)
  11. 36 body.enum_for(:each)
  12. else
  13. 154 StringIO.new(body.to_s)
  14. end
  15. end
  16. 27 def bytesize
  17. 450 return @body.bytesize if @body.respond_to?(:bytesize)
  18. 414 Float::INFINITY
  19. end
  20. 27 def read(length = nil, outbuf = nil)
  21. 468 return @body.read(length, outbuf) if @body.respond_to?(:read)
  22. begin
  23. 96 chunk = @body.next
  24. 48 if outbuf
  25. outbuf.clear.force_encoding(Encoding::BINARY)
  26. outbuf << chunk
  27. else
  28. 48 outbuf = chunk
  29. end
  30. 48 outbuf unless length && outbuf.empty?
  31. 32 rescue StopIteration
  32. end
  33. end
  34. 27 def close
  35. 48 @body.close if @body.respond_to?(:close)
  36. end
  37. end
  38. end
  39. end

lib/httpx/transcoder/utils/deflater.rb

100.0% lines covered

37 relevant lines. 37 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require_relative "body_reader"
  3. 27 module HTTPX
  4. 27 module Transcoder
  5. 27 class Deflater
  6. 27 attr_reader :content_type
  7. 27 def initialize(body)
  8. 84 @content_type = body.content_type
  9. 84 @body = BodyReader.new(body)
  10. 84 @closed = false
  11. end
  12. 27 def bytesize
  13. 324 buffer_deflate!
  14. 324 @buffer.size
  15. end
  16. 27 def read(length = nil, outbuf = nil)
  17. 416 return @buffer.read(length, outbuf) if @buffer
  18. 240 return if @closed
  19. 192 chunk = @body.read(length)
  20. 192 compressed_chunk = deflate(chunk)
  21. 192 return unless compressed_chunk
  22. 156 if outbuf
  23. 144 outbuf.clear.force_encoding(Encoding::BINARY)
  24. 144 outbuf << compressed_chunk
  25. else
  26. 12 compressed_chunk
  27. end
  28. end
  29. 27 def close
  30. 48 return if @closed
  31. 48 @buffer.close if @buffer
  32. 48 @body.close
  33. 48 @closed = true
  34. end
  35. 27 def rewind
  36. 28 return unless @buffer
  37. 16 @buffer.rewind
  38. end
  39. 27 private
  40. # rubocop:disable Naming/MemoizedInstanceVariableName
  41. 27 def buffer_deflate!
  42. 324 return @buffer if defined?(@buffer)
  43. 84 buffer = Response::Buffer.new(
  44. threshold_size: Options::MAX_BODY_THRESHOLD_SIZE
  45. )
  46. 84 ::IO.copy_stream(self, buffer)
  47. 84 buffer.rewind if buffer.respond_to?(:rewind)
  48. 84 @buffer = buffer
  49. end
  50. # rubocop:enable Naming/MemoizedInstanceVariableName
  51. end
  52. end
  53. end

lib/httpx/utils.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Utils
  4. 27 using URIExtensions
  5. 27 TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
  6. 27 VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
  7. 27 FILENAME_REGEX = /\s*filename=(#{VALUE})/.freeze
  8. 27 FILENAME_EXTENSION_REGEX = /\s*filename\*=(#{VALUE})/.freeze
  9. 27 module_function
  10. 27 def now
  11. 10735207 Process.clock_gettime(Process::CLOCK_MONOTONIC)
  12. end
  13. 27 def elapsed_time(monotonic_timestamp)
  14. 10701680 Process.clock_gettime(Process::CLOCK_MONOTONIC) - monotonic_timestamp
  15. end
  16. # The value of this field can be either an HTTP-date or a number of
  17. # seconds to delay after the response is received.
  18. 27 def parse_retry_after(retry_after)
  19. # first: bet on it being an integer
  20. 63 Integer(retry_after)
  21. rescue ArgumentError
  22. # Then it's a datetime
  23. 16 time = Time.httpdate(retry_after)
  24. 16 time - Time.now
  25. end
  26. 27 def get_filename(header, _prefix_regex = nil)
  27. 88 filename = nil
  28. 77 case header
  29. when FILENAME_REGEX
  30. 56 filename = Regexp.last_match(1)
  31. 56 filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
  32. when FILENAME_EXTENSION_REGEX
  33. 16 filename = Regexp.last_match(1)
  34. 16 encoding, _, filename = filename.split("'", 3)
  35. end
  36. 88 return unless filename
  37. 136 filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
  38. 72 filename.scrub!
  39. 72 filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
  40. 72 filename.force_encoding ::Encoding.find(encoding) if encoding
  41. 72 filename
  42. end
  43. 27 URIParser = URI::RFC2396_Parser.new
  44. 27 def to_uri(uri)
  45. 18009 return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
  46. 33 uri = URI(URIParser.escape(uri))
  47. 33 non_ascii_hostname = URIParser.unescape(uri.host)
  48. 33 non_ascii_hostname.force_encoding(Encoding::UTF_8)
  49. 33 idna_hostname = Punycode.encode_hostname(non_ascii_hostname)
  50. 33 uri.host = idna_hostname
  51. 32 uri.non_ascii_hostname = non_ascii_hostname
  52. 32 uri
  53. end
  54. end
  55. end