loading
Generated 2024-03-27T09:50:40+00:00

All Files ( 96.32% covered at 30077.09 hits/line )

101 files in total.
6819 relevant lines, 6568 lines covered and 251 lines missed. ( 96.32% )
File % covered Lines Relevant Lines Lines covered Lines missed Avg. Hits / Line
lib/httpx.rb 100.00 % 67 39 39 0 514.46
lib/httpx/adapters/datadog.rb 86.96 % 348 161 140 21 29.88
lib/httpx/adapters/faraday.rb 98.14 % 294 161 158 3 56.19
lib/httpx/adapters/sentry.rb 100.00 % 121 62 62 0 69.29
lib/httpx/adapters/webmock.rb 100.00 % 158 84 84 0 109.67
lib/httpx/altsvc.rb 96.39 % 163 83 80 3 173.54
lib/httpx/buffer.rb 100.00 % 50 21 21 0 76497.67
lib/httpx/callbacks.rb 100.00 % 40 22 22 0 96668.09
lib/httpx/chainable.rb 95.35 % 104 43 41 2 769.40
lib/httpx/connection.rb 96.22 % 733 370 356 14 91138.68
lib/httpx/connection/http1.rb 90.91 % 396 220 200 20 6834.24
lib/httpx/connection/http2.rb 95.93 % 405 246 236 10 44805.44
lib/httpx/domain_name.rb 95.45 % 145 44 42 2 188.73
lib/httpx/errors.rb 97.56 % 107 41 40 1 66.34
lib/httpx/extensions.rb 67.86 % 59 28 19 9 371.54
lib/httpx/headers.rb 100.00 % 175 71 71 0 13644.65
lib/httpx/io.rb 100.00 % 11 5 5 0 23.00
lib/httpx/io/ssl.rb 89.41 % 162 85 76 9 1507.51
lib/httpx/io/tcp.rb 91.07 % 206 112 102 10 5367.03
lib/httpx/io/udp.rb 85.71 % 62 35 30 5 303.86
lib/httpx/io/unix.rb 96.97 % 68 33 32 1 18.21
lib/httpx/loggable.rb 100.00 % 34 14 14 0 19963.86
lib/httpx/options.rb 98.70 % 353 154 152 2 12526.86
lib/httpx/parser/http1.rb 100.00 % 182 109 109 0 5602.21
lib/httpx/plugins/auth.rb 100.00 % 25 9 9 0 18.00
lib/httpx/plugins/auth/basic.rb 100.00 % 20 10 10 0 64.00
lib/httpx/plugins/auth/digest.rb 100.00 % 101 59 59 0 99.93
lib/httpx/plugins/auth/ntlm.rb 100.00 % 35 19 19 0 4.00
lib/httpx/plugins/auth/socks5.rb 100.00 % 22 11 11 0 18.18
lib/httpx/plugins/aws_sdk_authentication.rb 100.00 % 106 43 43 0 9.91
lib/httpx/plugins/aws_sigv4.rb 100.00 % 217 114 114 0 73.95
lib/httpx/plugins/basic_auth.rb 100.00 % 29 12 12 0 26.50
lib/httpx/plugins/brotli.rb 100.00 % 50 25 25 0 9.00
lib/httpx/plugins/callbacks.rb 100.00 % 91 43 43 0 107.98
lib/httpx/plugins/circuit_breaker.rb 97.06 % 138 68 66 2 51.32
lib/httpx/plugins/circuit_breaker/circuit.rb 100.00 % 100 47 47 0 42.17
lib/httpx/plugins/circuit_breaker/circuit_store.rb 100.00 % 53 23 23 0 73.78
lib/httpx/plugins/cookies.rb 100.00 % 104 50 50 0 90.84
lib/httpx/plugins/cookies/cookie.rb 100.00 % 174 77 77 0 238.71
lib/httpx/plugins/cookies/jar.rb 100.00 % 97 47 47 0 199.66
lib/httpx/plugins/cookies/set_cookie_parser.rb 100.00 % 140 70 70 0 121.04
lib/httpx/plugins/digest_auth.rb 100.00 % 62 29 29 0 74.28
lib/httpx/plugins/expect.rb 100.00 % 112 56 56 0 61.68
lib/httpx/plugins/follow_redirects.rb 100.00 % 173 82 82 0 21559.04
lib/httpx/plugins/grpc.rb 100.00 % 279 133 133 0 98.16
lib/httpx/plugins/grpc/call.rb 90.91 % 63 33 30 3 33.48
lib/httpx/plugins/grpc/grpc_encoding.rb 97.83 % 88 46 45 1 61.85
lib/httpx/plugins/grpc/message.rb 95.83 % 55 24 23 1 32.92
lib/httpx/plugins/h2c.rb 94.44 % 108 54 51 3 10.70
lib/httpx/plugins/ntlm_auth.rb 100.00 % 60 30 30 0 4.83
lib/httpx/plugins/oauth.rb 89.53 % 175 86 77 9 34.44
lib/httpx/plugins/persistent.rb 100.00 % 36 11 11 0 121.09
lib/httpx/plugins/proxy.rb 97.28 % 297 147 143 4 6139.01
lib/httpx/plugins/proxy/http.rb 100.00 % 179 99 99 0 4518.75
lib/httpx/plugins/proxy/socks4.rb 98.72 % 135 78 77 1 11214.21
lib/httpx/plugins/proxy/socks5.rb 100.00 % 194 112 112 0 11777.33
lib/httpx/plugins/proxy/ssh.rb 92.31 % 92 52 48 4 6.83
lib/httpx/plugins/push_promise.rb 100.00 % 81 41 41 0 7.85
lib/httpx/plugins/rate_limiter.rb 100.00 % 53 16 16 0 31.88
lib/httpx/plugins/response_cache.rb 100.00 % 178 80 80 0 58.53
lib/httpx/plugins/response_cache/store.rb 100.00 % 93 47 47 0 86.94
lib/httpx/plugins/retries.rb 96.77 % 198 93 90 3 104626.44
lib/httpx/plugins/ssrf_filter.rb 100.00 % 142 61 61 0 109.05
lib/httpx/plugins/stream.rb 100.00 % 148 70 70 0 91.74
lib/httpx/plugins/upgrade.rb 100.00 % 83 37 37 0 33.65
lib/httpx/plugins/upgrade/h2.rb 91.67 % 54 24 22 2 6.21
lib/httpx/plugins/webdav.rb 100.00 % 80 35 35 0 16.63
lib/httpx/pmatch_extensions.rb 100.00 % 33 17 17 0 23.29
lib/httpx/pool.rb 82.14 % 303 168 138 30 133383.57
lib/httpx/punycode.rb 100.00 % 22 9 9 0 15.67
lib/httpx/request.rb 100.00 % 250 109 109 0 3778.39
lib/httpx/request/body.rb 100.00 % 158 71 71 0 2197.65
lib/httpx/resolver.rb 100.00 % 154 76 76 0 1141.38
lib/httpx/resolver/https.rb 87.92 % 251 149 131 18 28.22
lib/httpx/resolver/multi.rb 100.00 % 75 40 40 0 186098.45
lib/httpx/resolver/native.rb 94.19 % 441 258 243 15 56479.84
lib/httpx/resolver/resolver.rb 90.00 % 120 60 54 6 803.03
lib/httpx/resolver/system.rb 92.68 % 214 123 114 9 19.63
lib/httpx/response.rb 100.00 % 287 105 105 0 1320.87
lib/httpx/response/body.rb 100.00 % 242 109 109 0 1909.72
lib/httpx/response/buffer.rb 100.00 % 96 49 49 0 1298.06
lib/httpx/selector.rb 86.36 % 138 66 57 9 394568.73
lib/httpx/session.rb 95.81 % 350 167 160 7 71724.80
lib/httpx/session_extensions.rb 100.00 % 29 14 14 0 5.79
lib/httpx/timers.rb 100.00 % 110 57 57 0 774974.61
lib/httpx/transcoder.rb 100.00 % 92 53 53 0 204.15
lib/httpx/transcoder/body.rb 100.00 % 57 32 32 0 628.06
lib/httpx/transcoder/chunker.rb 100.00 % 115 66 66 0 152.82
lib/httpx/transcoder/deflate.rb 100.00 % 37 20 20 0 20.50
lib/httpx/transcoder/form.rb 100.00 % 78 41 41 0 328.78
lib/httpx/transcoder/gzip.rb 100.00 % 74 43 43 0 69.60
lib/httpx/transcoder/json.rb 100.00 % 57 32 32 0 28.16
lib/httpx/transcoder/multipart.rb 100.00 % 17 10 10 0 680.10
lib/httpx/transcoder/multipart/decoder.rb 93.98 % 139 83 78 5 22.58
lib/httpx/transcoder/multipart/encoder.rb 100.00 % 110 65 65 0 1493.92
lib/httpx/transcoder/multipart/mime_type_detector.rb 91.89 % 78 37 34 3 130.38
lib/httpx/transcoder/multipart/part.rb 100.00 % 35 18 18 0 357.39
lib/httpx/transcoder/utils/body_reader.rb 96.00 % 46 25 24 1 77.08
lib/httpx/transcoder/utils/deflater.rb 97.22 % 72 36 35 1 69.14
lib/httpx/transcoder/xml.rb 92.31 % 52 26 24 2 45.12
lib/httpx/utils.rb 100.00 % 75 39 39 0 189417.51

lib/httpx.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "httpx/version"
  3. 23 require "httpx/extensions"
  4. 23 require "httpx/errors"
  5. 23 require "httpx/utils"
  6. 23 require "httpx/punycode"
  7. 23 require "httpx/domain_name"
  8. 23 require "httpx/altsvc"
  9. 23 require "httpx/callbacks"
  10. 23 require "httpx/loggable"
  11. 23 require "httpx/transcoder"
  12. 23 require "httpx/timers"
  13. 23 require "httpx/pool"
  14. 23 require "httpx/headers"
  15. 23 require "httpx/request"
  16. 23 require "httpx/response"
  17. 23 require "httpx/options"
  18. 23 require "httpx/chainable"
  19. # Top-Level Namespace
  20. #
  21. 23 module HTTPX
  22. 23 EMPTY = [].freeze
  23. # All plugins should be stored under this module/namespace. Can register and load
  24. # plugins.
  25. #
  26. 23 module Plugins
  27. 23 @plugins = {}
  28. 23 @plugins_mutex = Thread::Mutex.new
  29. # Loads a plugin based on a name. If the plugin hasn't been loaded, tries to load
  30. # it from the load path under "httpx/plugins/" directory.
  31. #
  32. 23 def self.load_plugin(name)
  33. 3652 h = @plugins
  34. 3652 m = @plugins_mutex
  35. 7304 unless (plugin = m.synchronize { h[name] })
  36. 80 require "httpx/plugins/#{name}"
  37. 160 raise "Plugin #{name} hasn't been registered" unless (plugin = m.synchronize { h[name] })
  38. end
  39. 3652 plugin
  40. end
  41. # Registers a plugin (+mod+) in the central store indexed by +name+.
  42. #
  43. 23 def self.register_plugin(name, mod)
  44. 225 h = @plugins
  45. 225 m = @plugins_mutex
  46. 424 m.synchronize { h[name] = mod }
  47. end
  48. end
  49. 23 extend Chainable
  50. end
  51. 23 require "httpx/session"
  52. 23 require "httpx/session_extensions"
  53. # load integrations when possible
  54. 23 require "httpx/adapters/datadog" if defined?(DDTrace) || defined?(Datadog)
  55. 23 require "httpx/adapters/sentry" if defined?(Sentry)
  56. 23 require "httpx/adapters/webmock" if defined?(WebMock)

lib/httpx/adapters/datadog.rb

86.96% lines covered

161 relevant lines. 140 lines covered and 21 lines missed.
    
  1. # frozen_string_literal: true
  2. 5 require "datadog/tracing/contrib/integration"
  3. 5 require "datadog/tracing/contrib/configuration/settings"
  4. 5 require "datadog/tracing/contrib/patcher"
  5. 5 module Datadog::Tracing
  6. 5 module Contrib
  7. 5 module HTTPX
  8. 5 DATADOG_VERSION = defined?(::DDTrace) ? ::DDTrace::VERSION : ::Datadog::VERSION
  9. 5 METADATA_MODULE = Datadog::Tracing::Metadata
  10. 5 TYPE_OUTBOUND = Datadog::Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
  11. 5 TAG_PEER_SERVICE = Datadog::Tracing::Metadata::Ext::TAG_PEER_SERVICE
  12. 5 TAG_URL = Datadog::Tracing::Metadata::Ext::HTTP::TAG_URL
  13. 5 TAG_METHOD = Datadog::Tracing::Metadata::Ext::HTTP::TAG_METHOD
  14. 5 TAG_TARGET_HOST = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_HOST
  15. 5 TAG_TARGET_PORT = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_PORT
  16. 5 TAG_STATUS_CODE = Datadog::Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE
  17. # HTTPX Datadog Plugin
  18. #
  19. # Enables tracing for httpx requests.
  20. #
  21. # A span will be created for each request transaction; the span is created lazily only when
  22. # receiving a response, and it is fed the start time stored inside the tracer object.
  23. #
  24. 5 module Plugin
  25. 5 class RequestTracer
  26. 5 include Contrib::HttpAnnotationHelper
  27. 5 SPAN_REQUEST = "httpx.request"
  28. # initializes the tracer object on the +request+.
  29. 5 def initialize(request)
  30. 106 @request = request
  31. 106 @start_time = nil
  32. # request objects are reused, when already buffered requests get rerouted to a different
  33. # connection due to connection issues, or when they already got a response, but need to
  34. # be retried. In such situations, the original span needs to be extended for the former,
  35. # while a new is required for the latter.
  36. 123 request.on(:idle) { reset }
  37. # the span is initialized when the request is buffered in the parser, which is the closest
  38. # one gets to actually sending the request.
  39. 193 request.on(:headers) { call }
  40. end
  41. # sets up the span start time, while preparing the on response callback.
  42. 5 def call(*args)
  43. 93 return if @start_time
  44. 88 start(*args)
  45. 88 @request.once(:response, &method(:finish))
  46. end
  47. 5 private
  48. # just sets the span init time. It can be passed a +start_time+ in cases where
  49. # this is collected outside the request transaction.
  50. 5 def start(start_time = now)
  51. 93 @start_time = start_time
  52. end
  53. # resets the start time for already finished request transactions.
  54. 5 def reset
  55. 17 return unless @start_time
  56. 5 start
  57. end
  58. # creates the span from the collected +@start_time+ to what the +response+ state
  59. # contains. It also resets internal state to allow this object to be reused.
  60. 5 def finish(response)
  61. 88 return unless @start_time
  62. 88 span = initialize_span
  63. 88 return unless span
  64. 88 if response.is_a?(::HTTPX::ErrorResponse)
  65. 6 span.set_error(response.error)
  66. else
  67. 82 span.set_tag(TAG_STATUS_CODE, response.status.to_s)
  68. 82 span.set_error(::HTTPX::HTTPError.new(response)) if response.status >= 400 && response.status <= 599
  69. end
  70. 88 span.finish
  71. ensure
  72. 88 @start_time = nil
  73. end
  74. # return a span initialized with the +@request+ state.
  75. 5 def initialize_span
  76. 88 verb = @request.verb
  77. 88 uri = @request.uri
  78. 88 span = create_span(@request)
  79. 88 span.resource = verb
  80. # Add additional request specific tags to the span.
  81. 88 span.set_tag(TAG_URL, @request.path)
  82. 88 span.set_tag(TAG_METHOD, verb)
  83. 88 span.set_tag(TAG_TARGET_HOST, uri.host)
  84. 88 span.set_tag(TAG_TARGET_PORT, uri.port.to_s)
  85. # Tag as an external peer service
  86. 88 span.set_tag(TAG_PEER_SERVICE, span.service)
  87. 88 if configuration[:distributed_tracing]
  88. 83 propagate_trace_http(
  89. Datadog::Tracing.active_trace.to_digest,
  90. @request.headers
  91. )
  92. end
  93. # Set analytics sample rate
  94. 88 if Contrib::Analytics.enabled?(configuration[:analytics_enabled])
  95. 10 Contrib::Analytics.set_sample_rate(span, configuration[:analytics_sample_rate])
  96. end
  97. 88 span
  98. rescue StandardError => e
  99. Datadog.logger.error("error preparing span for http request: #{e}")
  100. Datadog.logger.error(e.backtrace)
  101. end
  102. 5 def now
  103. 87 ::Datadog::Core::Utils::Time.now.utc
  104. end
  105. 5 def configuration
  106. 274 @configuration ||= Datadog.configuration.tracing[:httpx, @request.uri.host]
  107. end
  108. 5 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("2.0.0.beta1")
  109. 2 def propagate_trace_http(digest, headers)
  110. 30 Datadog::Tracing::Contrib::HTTP.inject(digest, headers)
  111. end
  112. 2 def create_span(request)
  113. 32 Datadog::Tracing.trace(
  114. SPAN_REQUEST,
  115. service: service_name(request.uri.host, configuration, Datadog.configuration_for(self)),
  116. type: TYPE_OUTBOUND,
  117. start_time: @start_time
  118. )
  119. end
  120. else
  121. 3 def propagate_trace_http(digest, headers)
  122. 53 Datadog::Tracing::Propagation::HTTP.inject!(digest, headers)
  123. end
  124. 3 def create_span(request)
  125. 56 Datadog::Tracing.trace(
  126. SPAN_REQUEST,
  127. service: service_name(request.uri.host, configuration, Datadog.configuration_for(self)),
  128. span_type: TYPE_OUTBOUND,
  129. start_time: @start_time
  130. )
  131. end
  132. end
  133. end
  134. 5 module RequestMethods
  135. # intercepts request initialization to inject the tracing logic.
  136. 5 def initialize(*)
  137. 100 super
  138. 100 return unless Datadog::Tracing.enabled?
  139. 100 RequestTracer.new(self)
  140. end
  141. end
  142. 5 module ConnectionMethods
  143. 5 attr_reader :init_time
  144. 5 def initialize(*)
  145. 91 super
  146. 91 @init_time = ::Datadog::Core::Utils::Time.now.utc
  147. end
  148. # handles the case when the +error+ happened during name resolution, which meanns
  149. # that the tracing logic hasn't been injected yet; in such cases, the approximate
  150. # initial resolving time is collected from the connection, and used as span start time,
  151. # and the tracing object in inserted before the on response callback is called.
  152. 5 def handle_error(error)
  153. 6 return super unless Datadog::Tracing.enabled?
  154. 6 return super unless error.respond_to?(:connection)
  155. 6 @pending.each do |request|
  156. 6 RequestTracer.new(request).call(error.connection.init_time)
  157. end
  158. 6 super
  159. end
  160. end
  161. end
  162. 5 module Configuration
  163. # Default settings for httpx
  164. #
  165. 5 class Settings < Datadog::Tracing::Contrib::Configuration::Settings
  166. 5 DEFAULT_ERROR_HANDLER = lambda do |response|
  167. Datadog::Ext::HTTP::ERROR_RANGE.cover?(response.status)
  168. end
  169. 5 option :service_name, default: "httpx"
  170. 5 option :distributed_tracing, default: true
  171. 5 option :split_by_domain, default: false
  172. 5 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  173. 5 option :enabled do |o|
  174. 5 o.type :bool
  175. 5 o.env "DD_TRACE_HTTPX_ENABLED"
  176. 5 o.default true
  177. end
  178. 5 option :analytics_enabled do |o|
  179. 5 o.type :bool
  180. 5 o.env "DD_TRACE_HTTPX_ANALYTICS_ENABLED"
  181. 5 o.default false
  182. end
  183. 5 option :analytics_sample_rate do |o|
  184. 5 o.type :float
  185. 5 o.env "DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE"
  186. 5 o.default 1.0
  187. end
  188. else
  189. option :enabled do |o|
  190. o.default { env_to_bool("DD_TRACE_HTTPX_ENABLED", true) }
  191. o.lazy
  192. end
  193. option :analytics_enabled do |o|
  194. o.default { env_to_bool(%w[DD_TRACE_HTTPX_ANALYTICS_ENABLED DD_HTTPX_ANALYTICS_ENABLED], false) }
  195. o.lazy
  196. end
  197. option :analytics_sample_rate do |o|
  198. o.default { env_to_float(%w[DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE DD_HTTPX_ANALYTICS_SAMPLE_RATE], 1.0) }
  199. o.lazy
  200. end
  201. end
  202. 5 if defined?(Datadog::Tracing::Contrib::SpanAttributeSchema)
  203. 5 option :service_name do |o|
  204. 5 o.default do
  205. 56 Datadog::Tracing::Contrib::SpanAttributeSchema.fetch_service_name(
  206. "DD_TRACE_HTTPX_SERVICE_NAME",
  207. "httpx"
  208. )
  209. end
  210. 5 o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  211. end
  212. else
  213. option :service_name do |o|
  214. o.default do
  215. ENV.fetch("DD_TRACE_HTTPX_SERVICE_NAME", "httpx")
  216. end
  217. o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  218. end
  219. end
  220. 5 option :distributed_tracing, default: true
  221. 5 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.15.0")
  222. 5 option :error_handler do |o|
  223. 5 o.type :proc
  224. 5 o.default_proc(&DEFAULT_ERROR_HANDLER)
  225. end
  226. elsif Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  227. option :error_handler do |o|
  228. o.type :proc
  229. o.experimental_default_proc(&DEFAULT_ERROR_HANDLER)
  230. end
  231. else
  232. option :error_handler, default: DEFAULT_ERROR_HANDLER
  233. end
  234. end
  235. end
  236. # Patcher enables patching of 'httpx' with datadog components.
  237. #
  238. 5 module Patcher
  239. 5 include Datadog::Tracing::Contrib::Patcher
  240. 5 module_function
  241. 5 def target_version
  242. 10 Integration.version
  243. end
  244. # loads a session instannce with the datadog plugin, and replaces the
  245. # base HTTPX::Session with the patched session class.
  246. 5 def patch
  247. 5 datadog_session = ::HTTPX.plugin(Plugin)
  248. 5 ::HTTPX.send(:remove_const, :Session)
  249. 5 ::HTTPX.send(:const_set, :Session, datadog_session.class)
  250. end
  251. end
  252. # Datadog Integration for HTTPX.
  253. #
  254. 5 class Integration
  255. 5 include Contrib::Integration
  256. 5 MINIMUM_VERSION = Gem::Version.new("0.10.2")
  257. 5 register_as :httpx
  258. 5 def self.version
  259. 205 Gem.loaded_specs["httpx"] && Gem.loaded_specs["httpx"].version
  260. end
  261. 5 def self.loaded?
  262. 65 defined?(::HTTPX::Request)
  263. end
  264. 5 def self.compatible?
  265. 65 super && version >= MINIMUM_VERSION
  266. end
  267. 5 def new_configuration
  268. 66 Configuration::Settings.new
  269. end
  270. 5 def patcher
  271. 130 Patcher
  272. end
  273. end
  274. end
  275. end
  276. end

lib/httpx/adapters/faraday.rb

98.14% lines covered

161 relevant lines. 158 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 7 require "delegate"
  3. 7 require "httpx"
  4. 7 require "faraday"
  5. 7 module Faraday
  6. 7 class Adapter
  7. 7 class HTTPX < Faraday::Adapter
  8. 7 module RequestMixin
  9. 7 def build_connection(env)
  10. 163 return @connection if defined?(@connection)
  11. 163 @connection = ::HTTPX.plugin(:persistent).plugin(ReasonPlugin)
  12. 163 @connection = @connection.with(@connection_options) unless @connection_options.empty?
  13. 163 connection_opts = options_from_env(env)
  14. 163 if (bind = env.request.bind)
  15. 6 @bind = TCPSocket.new(bind[:host], bind[:port])
  16. 5 connection_opts[:io] = @bind
  17. end
  18. 163 @connection = @connection.with(connection_opts)
  19. 163 if (proxy = env.request.proxy)
  20. 6 proxy_options = { uri: proxy.uri }
  21. 6 proxy_options[:username] = proxy.user if proxy.user
  22. 6 proxy_options[:password] = proxy.password if proxy.password
  23. 6 @connection = @connection.plugin(:proxy).with(proxy: proxy_options)
  24. end
  25. 163 @connection = @connection.plugin(OnDataPlugin) if env.request.stream_response?
  26. 163 @connection
  27. end
  28. 7 def close
  29. 168 @connection.close if @connection
  30. 168 @bind.close if @bind
  31. end
  32. 7 private
  33. 7 def connect(env, &blk)
  34. 163 connection(env, &blk)
  35. rescue ::HTTPX::TLSError => e
  36. 6 raise Faraday::SSLError, e
  37. rescue Errno::ECONNABORTED,
  38. Errno::ECONNREFUSED,
  39. Errno::ECONNRESET,
  40. Errno::EHOSTUNREACH,
  41. Errno::EINVAL,
  42. Errno::ENETUNREACH,
  43. Errno::EPIPE,
  44. ::HTTPX::ConnectionError => e
  45. 6 raise Faraday::ConnectionFailed, e
  46. end
  47. 7 def build_request(env)
  48. 169 meth = env[:method]
  49. 28 request_options = {
  50. 141 headers: env.request_headers,
  51. body: env.body,
  52. }
  53. 169 [meth.to_s.upcase, env.url, request_options]
  54. end
  55. 7 def options_from_env(env)
  56. 163 timeout_options = {}
  57. 163 req_opts = env.request
  58. 163 if (sec = request_timeout(:read, req_opts))
  59. 5 timeout_options[:read_timeout] = sec
  60. end
  61. 163 if (sec = request_timeout(:write, req_opts))
  62. 5 timeout_options[:write_timeout] = sec
  63. end
  64. 163 if (sec = request_timeout(:open, req_opts))
  65. 5 timeout_options[:connect_timeout] = sec
  66. end
  67. 27 {
  68. 136 ssl: ssl_options_from_env(env),
  69. timeout: timeout_options,
  70. }
  71. end
  72. 7 if defined?(::OpenSSL)
  73. 7 def ssl_options_from_env(env)
  74. 163 ssl_options = {}
  75. 163 unless env.ssl.verify.nil?
  76. 10 ssl_options[:verify_mode] = env.ssl.verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
  77. end
  78. 163 ssl_options[:ca_file] = env.ssl.ca_file if env.ssl.ca_file
  79. 163 ssl_options[:ca_path] = env.ssl.ca_path if env.ssl.ca_path
  80. 163 ssl_options[:cert_store] = env.ssl.cert_store if env.ssl.cert_store
  81. 163 ssl_options[:cert] = env.ssl.client_cert if env.ssl.client_cert
  82. 163 ssl_options[:key] = env.ssl.client_key if env.ssl.client_key
  83. 163 ssl_options[:ssl_version] = env.ssl.version if env.ssl.version
  84. 163 ssl_options[:verify_depth] = env.ssl.verify_depth if env.ssl.verify_depth
  85. 163 ssl_options[:min_version] = env.ssl.min_version if env.ssl.min_version
  86. 163 ssl_options[:max_version] = env.ssl.max_version if env.ssl.max_version
  87. 163 ssl_options
  88. end
  89. else
  90. def ssl_options_from_env(*)
  91. {}
  92. end
  93. end
  94. end
  95. 7 include RequestMixin
  96. 7 module OnDataPlugin
  97. 7 module RequestMethods
  98. 7 attr_writer :response_on_data
  99. 7 def response=(response)
  100. 12 super
  101. 12 return if response.is_a?(::HTTPX::ErrorResponse)
  102. 12 response.body.on_data = @response_on_data
  103. end
  104. end
  105. 7 module ResponseBodyMethods
  106. 7 attr_writer :on_data
  107. 7 def write(chunk)
  108. 24 return super unless @on_data
  109. 24 @on_data.call(chunk, chunk.bytesize)
  110. end
  111. end
  112. end
  113. 7 module ReasonPlugin
  114. 7 def self.load_dependencies(*)
  115. 163 require "net/http/status"
  116. end
  117. 7 module ResponseMethods
  118. 7 def reason
  119. 133 Net::HTTP::STATUS_CODES.fetch(@status)
  120. end
  121. end
  122. end
  123. 7 class ParallelManager
  124. 7 class ResponseHandler < SimpleDelegator
  125. 7 attr_reader :env
  126. 7 def initialize(env)
  127. 24 @env = env
  128. 24 super
  129. end
  130. 7 def on_response(&blk)
  131. 48 if blk
  132. 24 @on_response = ->(response) do
  133. 24 blk.call(response)
  134. end
  135. 24 self
  136. else
  137. 24 @on_response
  138. end
  139. end
  140. 7 def on_complete(&blk)
  141. 48 if blk
  142. 24 @on_complete = blk
  143. 24 self
  144. else
  145. 24 @on_complete
  146. end
  147. end
  148. end
  149. 7 include RequestMixin
  150. 7 def initialize(options)
  151. 24 @handlers = []
  152. 24 @connection_options = options
  153. end
  154. 7 def enqueue(request)
  155. 24 handler = ResponseHandler.new(request)
  156. 24 @handlers << handler
  157. 24 handler
  158. end
  159. 7 def run
  160. 24 return unless @handlers.last
  161. 18 env = @handlers.last.env
  162. 18 connect(env) do |session|
  163. 42 requests = @handlers.map { |handler| session.build_request(*build_request(handler.env)) }
  164. 18 if env.request.stream_response?
  165. 6 requests.each do |request|
  166. 6 request.response_on_data = env.request.on_data
  167. end
  168. end
  169. 18 responses = session.request(*requests)
  170. 18 Array(responses).each_with_index do |response, index|
  171. 24 handler = @handlers[index]
  172. 24 handler.on_response.call(response)
  173. 24 handler.on_complete.call(handler.env)
  174. end
  175. end
  176. rescue ::HTTPX::TimeoutError => e
  177. raise Faraday::TimeoutError, e
  178. end
  179. # from Faraday::Adapter#connection
  180. 7 def connection(env)
  181. 18 conn = build_connection(env)
  182. 18 return conn unless block_given?
  183. 18 yield conn
  184. end
  185. 7 private
  186. # from Faraday::Adapter#request_timeout
  187. 7 def request_timeout(type, options)
  188. 54 key = Faraday::Adapter::TIMEOUT_KEYS[type]
  189. 54 options[key] || options[:timeout]
  190. end
  191. end
  192. 7 self.supports_parallel = true
  193. 7 class << self
  194. 7 def setup_parallel_manager(options = {})
  195. 24 ParallelManager.new(options)
  196. end
  197. end
  198. 7 def call(env)
  199. 169 super
  200. 169 if parallel?(env)
  201. 24 handler = env[:parallel_manager].enqueue(env)
  202. 24 handler.on_response do |response|
  203. 24 if response.is_a?(::HTTPX::Response)
  204. 18 save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
  205. 18 response_headers.merge!(response.headers)
  206. end
  207. else
  208. 5 env[:error] = response.error
  209. 6 save_response(env, 0, "", {}, nil)
  210. end
  211. end
  212. 20 return handler
  213. end
  214. 145 response = connect_and_request(env)
  215. 115 save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
  216. 115 response_headers.merge!(response.headers)
  217. end
  218. 115 @app.call(env)
  219. end
  220. 7 private
  221. 7 def connect_and_request(env)
  222. 145 connect(env) do |session|
  223. 145 request = session.build_request(*build_request(env))
  224. 145 request.response_on_data = env.request.on_data if env.request.stream_response?
  225. 145 response = session.request(request)
  226. # do not call #raise_for_status for HTTP 4xx or 5xx, as faraday has a middleware for that.
  227. 145 response.raise_for_status unless response.is_a?(::HTTPX::Response)
  228. 115 response
  229. end
  230. rescue ::HTTPX::TimeoutError => e
  231. 18 raise Faraday::TimeoutError, e
  232. end
  233. 7 def parallel?(env)
  234. 169 env[:parallel_manager]
  235. end
  236. end
  237. 7 register_middleware httpx: HTTPX
  238. end
  239. end

lib/httpx/adapters/sentry.rb

100.0% lines covered

62 relevant lines. 62 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 5 require "sentry-ruby"
  3. 5 module HTTPX::Plugins
  4. 5 module Sentry
  5. 5 module Tracer
  6. 5 module_function
  7. 5 def call(request)
  8. 88 sentry_span = start_sentry_span
  9. 88 return unless sentry_span
  10. 88 set_sentry_trace_header(request, sentry_span)
  11. 88 request.on(:response, &method(:finish_sentry_span).curry(3)[sentry_span, request])
  12. end
  13. 5 def start_sentry_span
  14. 88 return unless ::Sentry.initialized? && (span = ::Sentry.get_current_scope.get_span)
  15. 88 return if span.sampled == false
  16. 88 span.start_child(op: "httpx.client", start_timestamp: ::Sentry.utc_now.to_f)
  17. end
  18. 5 def set_sentry_trace_header(request, sentry_span)
  19. 88 return unless sentry_span
  20. 88 config = ::Sentry.configuration
  21. 88 url = request.uri.to_s
  22. 176 return unless config.propagate_traces && config.trace_propagation_targets.any? { |target| url.match?(target) }
  23. 88 trace = ::Sentry.get_current_client.generate_sentry_trace(sentry_span)
  24. 88 request.headers[::Sentry::SENTRY_TRACE_HEADER_NAME] = trace if trace
  25. end
  26. 5 def finish_sentry_span(span, request, response)
  27. 96 return unless ::Sentry.initialized?
  28. 96 record_sentry_breadcrumb(request, response)
  29. 96 record_sentry_span(request, response, span)
  30. end
  31. 5 def record_sentry_breadcrumb(req, res)
  32. 96 return unless ::Sentry.configuration.breadcrumbs_logger.include?(:http_logger)
  33. 96 request_info = extract_request_info(req)
  34. 96 data = if res.is_a?(HTTPX::ErrorResponse)
  35. 9 { error: res.error.message, **request_info }
  36. else
  37. 87 { status: res.status, **request_info }
  38. end
  39. 96 crumb = ::Sentry::Breadcrumb.new(
  40. level: :info,
  41. category: "httpx",
  42. type: :info,
  43. data: data
  44. )
  45. 96 ::Sentry.add_breadcrumb(crumb)
  46. end
  47. 5 def record_sentry_span(req, res, sentry_span)
  48. 96 return unless sentry_span
  49. 96 request_info = extract_request_info(req)
  50. 96 sentry_span.set_description("#{request_info[:method]} #{request_info[:url]}")
  51. 96 if res.is_a?(HTTPX::ErrorResponse)
  52. 9 sentry_span.set_data(:error, res.error.message)
  53. else
  54. 87 sentry_span.set_data(:status, res.status)
  55. end
  56. 96 sentry_span.set_timestamp(::Sentry.utc_now.to_f)
  57. end
  58. 5 def extract_request_info(req)
  59. 192 uri = req.uri
  60. result = {
  61. 192 method: req.verb,
  62. }
  63. 192 if ::Sentry.configuration.send_default_pii
  64. 20 uri += "?#{req.query}" unless req.query.empty?
  65. 20 result[:body] = req.body.to_s unless req.body.empty? || req.body.unbounded_body?
  66. end
  67. 192 result[:url] = uri.to_s
  68. 192 result
  69. end
  70. end
  71. 5 module RequestMethods
  72. 5 def __sentry_enable_trace!
  73. 96 return if @__sentry_enable_trace
  74. 88 Tracer.call(self)
  75. 88 @__sentry_enable_trace = true
  76. end
  77. end
  78. 5 module ConnectionMethods
  79. 5 def send(request)
  80. 96 request.__sentry_enable_trace!
  81. 96 super
  82. end
  83. end
  84. end
  85. end
  86. 5 Sentry.register_patch(:httpx) do
  87. 25 sentry_session = HTTPX.plugin(HTTPX::Plugins::Sentry)
  88. 25 HTTPX.send(:remove_const, :Session)
  89. 25 HTTPX.send(:const_set, :Session, sentry_session.class)
  90. end

lib/httpx/adapters/webmock.rb

100.0% lines covered

84 relevant lines. 84 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module WebMock
  3. 6 module HttpLibAdapters
  4. 6 require "net/http/status"
  5. 6 HTTP_REASONS = Net::HTTP::STATUS_CODES
  6. #
  7. # HTTPX plugin for webmock.
  8. #
  9. # Requests are "hijacked" at the session, before they're distributed to a connection.
  10. #
  11. 6 module Plugin
  12. 6 class << self
  13. 6 def build_webmock_request_signature(request)
  14. 126 uri = WebMock::Util::URI.heuristic_parse(request.uri)
  15. 126 uri.query = request.query
  16. 126 uri.path = uri.normalized_path.gsub("[^:]//", "/")
  17. 126 WebMock::RequestSignature.new(
  18. request.verb.downcase.to_sym,
  19. uri.to_s,
  20. body: request.body.each.to_a.join,
  21. headers: request.headers.to_h
  22. )
  23. end
  24. 6 def build_webmock_response(_request, response)
  25. 5 webmock_response = WebMock::Response.new
  26. 5 webmock_response.status = [response.status, HTTP_REASONS[response.status]]
  27. 5 webmock_response.body = response.body.to_s
  28. 5 webmock_response.headers = response.headers.to_h
  29. 5 webmock_response
  30. end
  31. 6 def build_from_webmock_response(request, webmock_response)
  32. 116 return build_error_response(request, HTTPX::TimeoutError.new(1, "Timed out")) if webmock_response.should_timeout
  33. 101 return build_error_response(request, webmock_response.exception) if webmock_response.exception
  34. 96 request.options.response_class.new(request,
  35. webmock_response.status[0],
  36. "2.0",
  37. webmock_response.headers).tap do |res|
  38. 96 res.mocked = true
  39. end
  40. end
  41. 6 def build_error_response(request, exception)
  42. 20 HTTPX::ErrorResponse.new(request, exception, request.options)
  43. end
  44. end
  45. 6 module InstanceMethods
  46. 6 def init_connection(*)
  47. 101 connection = super
  48. 101 connection.once(:unmock_connection) do
  49. 5 unless connection.addresses
  50. 5 connection.__send__(:callbacks)[:connect_error].clear
  51. 5 pool.__send__(:unregister_connection, connection)
  52. end
  53. 5 pool.__send__(:resolve_connection, connection)
  54. end
  55. 101 connection
  56. end
  57. end
  58. 6 module ResponseMethods
  59. 6 attr_accessor :mocked
  60. 6 def initialize(*)
  61. 101 super
  62. 101 @mocked = false
  63. end
  64. end
  65. 6 module ResponseBodyMethods
  66. 6 def decode_chunk(chunk)
  67. 50 return chunk if @response.mocked
  68. 5 super
  69. end
  70. end
  71. 6 module ConnectionMethods
  72. 6 def initialize(*)
  73. 101 super
  74. 101 @mocked = true
  75. end
  76. 6 def open?
  77. 106 return true if @mocked
  78. 5 super
  79. end
  80. 6 def interests
  81. 4313 return if @mocked
  82. 40 super
  83. end
  84. 6 def send(request)
  85. 126 request_signature = Plugin.build_webmock_request_signature(request)
  86. 126 WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)
  87. 126 if (mock_response = WebMock::StubRegistry.instance.response_for_request(request_signature))
  88. 116 response = Plugin.build_from_webmock_response(request, mock_response)
  89. 116 WebMock::CallbackRegistry.invoke_callbacks({ lib: :httpx }, request_signature, mock_response)
  90. 116 log { "mocking #{request.uri} with #{mock_response.inspect}" }
  91. 116 request.response = response
  92. 116 request.emit(:response, response)
  93. 116 response << mock_response.body.dup unless response.is_a?(HTTPX::ErrorResponse)
  94. 10 elsif WebMock.net_connect_allowed?(request_signature.uri)
  95. 5 if WebMock::CallbackRegistry.any_callbacks?
  96. 5 request.on(:response) do |resp|
  97. 5 unless resp.is_a?(HTTPX::ErrorResponse)
  98. 5 webmock_response = Plugin.build_webmock_response(request, resp)
  99. 5 WebMock::CallbackRegistry.invoke_callbacks(
  100. { lib: :httpx, real_request: true }, request_signature,
  101. webmock_response
  102. )
  103. end
  104. end
  105. end
  106. 5 @mocked = false
  107. 5 emit(:unmock_connection, self)
  108. 5 super
  109. else
  110. 5 raise WebMock::NetConnectNotAllowedError, request_signature
  111. end
  112. end
  113. end
  114. end
  115. 6 class HttpxAdapter < HttpLibAdapter
  116. 6 adapter_for :httpx
  117. 6 class << self
  118. 6 def enable!
  119. 247 @original_session ||= HTTPX::Session
  120. 247 webmock_session = HTTPX.plugin(Plugin)
  121. 247 HTTPX.send(:remove_const, :Session)
  122. 247 HTTPX.send(:const_set, :Session, webmock_session.class)
  123. end
  124. 6 def disable!
  125. 247 return unless @original_session
  126. 241 HTTPX.send(:remove_const, :Session)
  127. 241 HTTPX.send(:const_set, :Session, @original_session)
  128. end
  129. end
  130. end
  131. end
  132. end

lib/httpx/altsvc.rb

96.39% lines covered

83 relevant lines. 80 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "strscan"
  3. 23 module HTTPX
  4. 23 module AltSvc
  5. # makes connections able to accept requests destined to primary service.
  6. 23 module ConnectionMixin
  7. 23 using URIExtensions
  8. 23 def send(request)
  9. 6 request.headers["alt-used"] = @origin.authority if @parser && !@write_buffer.full? && match_altsvcs?(request.uri)
  10. 6 super
  11. end
  12. 23 def match?(uri, options)
  13. 6 return false if !used? && (@state == :closing || @state == :closed)
  14. 6 match_altsvcs?(uri) && match_altsvc_options?(uri, options)
  15. end
  16. 23 private
  17. # checks if this is connection is an alternative service of
  18. # +uri+
  19. 23 def match_altsvcs?(uri)
  20. 18 @origins.any? { |origin| altsvc_match?(uri, origin) } ||
  21. AltSvc.cached_altsvc(@origin).any? do |altsvc|
  22. origin = altsvc["origin"]
  23. altsvc_match?(origin, uri.origin)
  24. end
  25. end
  26. 23 def match_altsvc_options?(uri, options)
  27. 6 return @options == options unless @options.ssl.all? do |k, v|
  28. 6 v == (k == :hostname ? uri.host : options.ssl[k])
  29. end
  30. 6 @options.options_equals?(options, Options::REQUEST_BODY_IVARS + %i[@ssl])
  31. end
  32. 23 def altsvc_match?(uri, other_uri)
  33. 12 other_uri = URI(other_uri)
  34. 12 uri.origin == other_uri.origin || begin
  35. 5 case uri.scheme
  36. when "h2"
  37. (other_uri.scheme == "https" || other_uri.scheme == "h2") &&
  38. uri.host == other_uri.host &&
  39. uri.port == other_uri.port
  40. else
  41. 6 false
  42. end
  43. end
  44. end
  45. end
  46. 23 @altsvc_mutex = Thread::Mutex.new
  47. 38 @altsvcs = Hash.new { |h, k| h[k] = [] }
  48. 23 module_function
  49. 23 def cached_altsvc(origin)
  50. 30 now = Utils.now
  51. 30 @altsvc_mutex.synchronize do
  52. 30 lookup(origin, now)
  53. end
  54. end
  55. 23 def cached_altsvc_set(origin, entry)
  56. 18 now = Utils.now
  57. 18 @altsvc_mutex.synchronize do
  58. 18 return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
  59. 18 entry["TTL"] = Integer(entry["ma"]) + now if entry.key?("ma")
  60. 18 @altsvcs[origin] << entry
  61. 18 entry
  62. end
  63. end
  64. 23 def lookup(origin, ttl)
  65. 30 return [] unless @altsvcs.key?(origin)
  66. 20 @altsvcs[origin] = @altsvcs[origin].select do |entry|
  67. 18 !entry.key?("TTL") || entry["TTL"] > ttl
  68. end
  69. 36 @altsvcs[origin].reject { |entry| entry["noop"] }
  70. end
  71. 23 def emit(request, response)
  72. 5555 return unless response.respond_to?(:headers)
  73. # Alt-Svc
  74. 5534 return unless response.headers.key?("alt-svc")
  75. 64 origin = request.origin
  76. 64 host = request.uri.host
  77. 64 altsvc = response.headers["alt-svc"]
  78. # https://datatracker.ietf.org/doc/html/rfc7838#section-3
  79. # A field value containing the special value "clear" indicates that the
  80. # origin requests all alternatives for that origin to be invalidated
  81. # (including those specified in the same response, in case of an
  82. # invalid reply containing both "clear" and alternative services).
  83. 64 if altsvc == "clear"
  84. 6 @altsvc_mutex.synchronize do
  85. 6 @altsvcs[origin].clear
  86. end
  87. 5 return
  88. end
  89. 58 parse(altsvc) do |alt_origin, alt_params|
  90. 6 alt_origin.host ||= host
  91. 6 yield(alt_origin, origin, alt_params)
  92. end
  93. end
  94. 23 def parse(altsvc)
  95. 142 return enum_for(__method__, altsvc) unless block_given?
  96. 100 scanner = StringScanner.new(altsvc)
  97. 106 until scanner.eos?
  98. 100 alt_service = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
  99. 100 alt_params = []
  100. 100 loop do
  101. 118 alt_param = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
  102. 118 alt_params << alt_param.strip if alt_param
  103. 118 scanner.skip(/;/)
  104. 118 break if scanner.eos? || scanner.scan(/ *, */)
  105. end
  106. 200 alt_params = Hash[alt_params.map { |field| field.split("=") }]
  107. 100 alt_proto, alt_authority = alt_service.split("=")
  108. 100 alt_origin = parse_altsvc_origin(alt_proto, alt_authority)
  109. 100 return unless alt_origin
  110. 36 yield(alt_origin, alt_params.merge("proto" => alt_proto))
  111. end
  112. end
  113. 23 def parse_altsvc_scheme(alt_proto)
  114. 101 case alt_proto
  115. when "h2c"
  116. 6 "http"
  117. when "h2"
  118. 42 "https"
  119. end
  120. end
  121. 23 def parse_altsvc_origin(alt_proto, alt_origin)
  122. 100 alt_scheme = parse_altsvc_scheme(alt_proto)
  123. 100 return unless alt_scheme
  124. 36 alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
  125. 36 URI.parse("#{alt_scheme}://#{alt_origin}")
  126. end
  127. end
  128. end

lib/httpx/buffer.rb

100.0% lines covered

21 relevant lines. 21 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "forwardable"
  3. 23 module HTTPX
  4. # Internal class to abstract a string buffer, by wrapping a string and providing the
  5. # minimum possible API and functionality required.
  6. #
  7. # buffer = Buffer.new(640)
  8. # buffer.full? #=> false
  9. # buffer << "aa"
  10. # buffer.capacity #=> 638
  11. #
  12. 23 class Buffer
  13. 23 extend Forwardable
  14. 23 def_delegator :@buffer, :<<
  15. 23 def_delegator :@buffer, :to_s
  16. 23 def_delegator :@buffer, :to_str
  17. 23 def_delegator :@buffer, :empty?
  18. 23 def_delegator :@buffer, :bytesize
  19. 23 def_delegator :@buffer, :clear
  20. 23 def_delegator :@buffer, :replace
  21. 23 attr_reader :limit
  22. 23 def initialize(limit)
  23. 10783 @buffer = "".b
  24. 10783 @limit = limit
  25. end
  26. 23 def full?
  27. 1568611 @buffer.bytesize >= @limit
  28. end
  29. 23 def capacity
  30. 10 @limit - @buffer.bytesize
  31. end
  32. 23 def shift!(fin)
  33. 15896 @buffer = @buffer.byteslice(fin..-1) || "".b
  34. end
  35. end
  36. end

lib/httpx/callbacks.rb

100.0% lines covered

22 relevant lines. 22 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 module Callbacks
  4. 23 def on(type, &action)
  5. 208140 callbacks(type) << action
  6. 208140 self
  7. end
  8. 23 def once(type, &block)
  9. 82623 on(type) do |*args, &callback|
  10. 81732 block.call(*args, &callback)
  11. 81684 :delete
  12. end
  13. 82623 self
  14. end
  15. 23 def only(type, &block)
  16. 15381 callbacks(type).clear
  17. 15381 on(type, &block)
  18. end
  19. 23 def emit(type, *args)
  20. 230405 callbacks(type).delete_if { |pr| :delete == pr.call(*args) } # rubocop:disable Style/YodaCondition
  21. end
  22. 23 def callbacks_for?(type)
  23. 2270 @callbacks.key?(type) && @callbacks[type].any?
  24. end
  25. 23 protected
  26. 23 def callbacks(type = nil)
  27. 324790 return @callbacks unless type
  28. 468615 @callbacks ||= Hash.new { |h, k| h[k] = [] }
  29. 324707 @callbacks[type]
  30. end
  31. end
  32. end

lib/httpx/chainable.rb

95.35% lines covered

43 relevant lines. 41 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. # Session mixin, implements most of the APIs that the users call.
  4. # delegates to a default session when extended.
  5. 23 module Chainable
  6. 23 %w[head get post put delete trace options connect patch].each do |meth|
  7. 207 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  8. 9 def #{meth}(*uri, **options) # def get(*uri, **options)
  9. 9 request("#{meth.upcase}", uri, **options) # request("GET", uri, **options)
  10. end # end
  11. MOD
  12. end
  13. # delegates to the default session (see HTTPX::Session#request).
  14. 23 def request(*args, **options)
  15. 1827 branch(default_options).request(*args, **options)
  16. end
  17. 23 def accept(type)
  18. 12 with(headers: { "accept" => String(type) })
  19. end
  20. # delegates to the default session (see HTTPX::Session#wrap).
  21. 23 def wrap(&blk)
  22. 73 branch(default_options).wrap(&blk)
  23. end
  24. # returns a new instance loaded with the +pl+ plugin and +options+.
  25. 23 def plugin(pl, options = nil, &blk)
  26. 3376 klass = is_a?(S) ? self.class : Session
  27. 3376 klass = Class.new(klass)
  28. 3376 klass.instance_variable_set(:@default_options, klass.default_options.merge(default_options))
  29. 3376 klass.plugin(pl, options, &blk).new
  30. end
  31. # returns a new instance loaded with +options+.
  32. 23 def with(options, &blk)
  33. 1843 branch(default_options.merge(options), &blk)
  34. end
  35. 23 private
  36. # returns default instance of HTTPX::Options.
  37. 23 def default_options
  38. 7161 @options || Session.default_options
  39. end
  40. # returns a default instance of HTTPX::Session.
  41. 23 def branch(options, &blk)
  42. 3743 return self.class.new(options, &blk) if is_a?(S)
  43. 2170 Session.new(options, &blk)
  44. end
  45. 23 def method_missing(meth, *args, **options, &blk)
  46. 462 case meth
  47. when /\Awith_(.+)/
  48. 539 option = Regexp.last_match(1)
  49. 539 return super unless option
  50. 539 with(option.to_sym => args.first || options)
  51. when /\Aon_(.+)/
  52. 7 callback = Regexp.last_match(1)
  53. 7 return super unless %w[
  54. connection_opened connection_closed
  55. request_error
  56. request_started request_body_chunk request_completed
  57. response_started response_body_chunk response_completed
  58. ].include?(callback)
  59. 7 warn "DEPRECATION WARNING: calling `.#{meth}` on plain HTTPX sessions is deprecated. " \
  60. 1 "Use HTTPX.plugin(:callbacks).#{meth} instead."
  61. 7 plugin(:callbacks).__send__(meth, *args, **options, &blk)
  62. else
  63. super
  64. end
  65. end
  66. 23 def respond_to_missing?(meth, *)
  67. 35 case meth
  68. when /\Awith_(.+)/
  69. 30 option = Regexp.last_match(1)
  70. 30 default_options.respond_to?(option) || super
  71. when /\Aon_(.+)/
  72. 12 callback = Regexp.last_match(1)
  73. 12 %w[
  74. connection_opened connection_closed
  75. request_error
  76. request_started request_body_chunk request_completed
  77. response_started response_body_chunk response_completed
  78. ].include?(callback) || super
  79. else
  80. super
  81. end
  82. end
  83. end
  84. end

lib/httpx/connection.rb

96.22% lines covered

370 relevant lines. 356 lines covered and 14 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "resolv"
  3. 23 require "forwardable"
  4. 23 require "httpx/io"
  5. 23 require "httpx/buffer"
  6. 23 module HTTPX
  7. # The Connection can be watched for IO events.
  8. #
  9. # It contains the +io+ object to read/write from, and knows what to do when it can.
  10. #
  11. # It defers connecting until absolutely necessary. Connection should be triggered from
  12. # the IO selector (until then, any request will be queued).
  13. #
  14. # A connection boots up its parser after connection is established. All pending requests
  15. # will be redirected there after connection.
  16. #
  17. # A connection can be prevented from closing by the parser, that is, if there are pending
  18. # requests. This will signal that the connection was prematurely closed, due to a possible
  19. # number of conditions:
  20. #
  21. # * Remote peer closed the connection ("Connection: close");
  22. # * Remote peer doesn't support pipelining;
  23. #
  24. # A connection may also route requests for a different host for which the +io+ was connected
  25. # to, provided that the IP is the same and the port and scheme as well. This will allow to
  26. # share the same socket to send HTTP/2 requests to different hosts.
  27. #
  28. 23 class Connection
  29. 23 extend Forwardable
  30. 23 include Loggable
  31. 23 include Callbacks
  32. 23 using URIExtensions
  33. 23 require "httpx/connection/http2"
  34. 23 require "httpx/connection/http1"
  35. 23 def_delegator :@io, :closed?
  36. 23 def_delegator :@write_buffer, :empty?
  37. 23 attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session
  38. 23 attr_writer :timers
  39. 23 attr_accessor :family
  40. 23 def initialize(uri, options)
  41. 5117 @origins = [uri.origin]
  42. 5117 @origin = Utils.to_uri(uri.origin)
  43. 5117 @options = Options.new(options)
  44. 5117 @type = initialize_type(uri, @options)
  45. 5117 @window_size = @options.window_size
  46. 5117 @read_buffer = Buffer.new(@options.buffer_size)
  47. 5117 @write_buffer = Buffer.new(@options.buffer_size)
  48. 5117 @pending = []
  49. 5117 on(:error, &method(:on_error))
  50. 5117 if @options.io
  51. # if there's an already open IO, get its
  52. # peer address, and force-initiate the parser
  53. 52 transition(:already_open)
  54. 52 @io = build_socket
  55. 52 parser
  56. else
  57. 5065 transition(:idle)
  58. end
  59. 5117 @inflight = 0
  60. 5117 @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
  61. 5117 @intervals = []
  62. 5117 self.addresses = @options.addresses if @options.addresses
  63. end
  64. # this is a semi-private method, to be used by the resolver
  65. # to initiate the io object.
  66. 23 def addresses=(addrs)
  67. 4949 if @io
  68. 173 @io.add_addresses(addrs)
  69. else
  70. 4776 @io = build_socket(addrs)
  71. end
  72. end
  73. 23 def addresses
  74. 23543 @io && @io.addresses
  75. end
  76. 23 def match?(uri, options)
  77. 12153 return false if !used? && (@state == :closing || @state == :closed)
  78. 476 (
  79. 12153 @origins.include?(uri.origin) &&
  80. # if there is more than one origin to match, it means that this connection
  81. # was the result of coalescing. To prevent blind trust in the case where the
  82. # origin came from an ORIGIN frame, we're going to verify the hostname with the
  83. # SSL certificate
  84. 2564 (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
  85. ) && @options == options
  86. end
  87. 23 def expired?
  88. return false unless @io
  89. @io.expired?
  90. end
  91. 23 def mergeable?(connection)
  92. 9333 return false if @state == :closing || @state == :closed || !@io
  93. 6787 return false unless connection.addresses
  94. 285 (
  95. 6787 (open? && @origin == connection.origin) ||
  96. 6607 !(@io.addresses & (connection.addresses || [])).empty?
  97. ) && @options == connection.options
  98. end
  99. # coalescable connections need to be mergeable!
  100. # but internally, #mergeable? is called before #coalescable?
  101. 23 def coalescable?(connection)
  102. 13 if @io.protocol == "h2" &&
  103. @origin.scheme == "https" &&
  104. connection.origin.scheme == "https" &&
  105. @io.can_verify_peer?
  106. 6 @io.verify_hostname(connection.origin.host)
  107. else
  108. 7 @origin == connection.origin
  109. end
  110. end
  111. 23 def create_idle(options = {})
  112. 6 self.class.new(@origin, @options.merge(options))
  113. end
  114. 23 def merge(connection)
  115. 21 @origins |= connection.instance_variable_get(:@origins)
  116. 23 if connection.ssl_session
  117. 6 @ssl_session = connection.ssl_session
  118. @io.session_new_cb do |sess|
  119. @ssl_session = sess
  120. 6 end if @io
  121. end
  122. 23 connection.purge_pending do |req|
  123. 6 send(req)
  124. end
  125. end
  126. 23 def purge_pending(&block)
  127. 23 pendings = []
  128. 23 if @parser
  129. 10 @inflight -= @parser.pending.size
  130. 12 pendings << @parser.pending
  131. end
  132. 23 pendings << @pending
  133. 23 pendings.each do |pending|
  134. 35 pending.reject!(&block)
  135. end
  136. end
  137. 23 def connecting?
  138. 2022814 @state == :idle
  139. end
  140. 23 def inflight?
  141. 4892 @parser && !@parser.empty? && !@write_buffer.empty?
  142. end
  143. 23 def interests
  144. # connecting
  145. 2014859 if connecting?
  146. 7701 connect
  147. 7701 return @io.interests if connecting?
  148. end
  149. # if the write buffer is full, we drain it
  150. 2008111 return :w unless @write_buffer.empty?
  151. 1977474 return @parser.interests if @parser
  152. 6 nil
  153. end
  154. 23 def to_io
  155. 17291 @io.to_io
  156. end
  157. 23 def call
  158. 12755 case @state
  159. when :idle
  160. 6622 connect
  161. 6610 consume
  162. when :closed
  163. return
  164. when :closing
  165. consume
  166. transition(:closed)
  167. when :open
  168. 7504 consume
  169. end
  170. 1515 nil
  171. end
  172. 23 def close
  173. 4921 transition(:active) if @state == :inactive
  174. 4921 @parser.close if @parser
  175. end
  176. 23 def terminate
  177. 4921 @connected_at = nil if @state == :closed
  178. 4921 close
  179. end
  180. # bypasses the state machine to force closing of connections still connecting.
  181. # **only** used for Happy Eyeballs v2.
  182. 23 def force_reset
  183. 118 @state = :closing
  184. 118 transition(:closed)
  185. end
  186. 23 def reset
  187. 7860 return if @state == :closing || @state == :closed
  188. 5359 transition(:closing)
  189. 5359 transition(:closed)
  190. end
  191. 23 def send(request)
  192. 6292 if @parser && !@write_buffer.full?
  193. 302 if @response_received_at && @keep_alive_timeout &&
  194. Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
  195. # when pushing a request into an existing connection, we have to check whether there
  196. # is the possibility that the connection might have extended the keep alive timeout.
  197. # for such cases, we want to ping for availability before deciding to shovel requests.
  198. 6 log(level: 3) { "keep alive timeout expired, pinging connection..." }
  199. 6 @pending << request
  200. 6 transition(:active) if @state == :inactive
  201. 6 parser.ping
  202. 5 return
  203. end
  204. 296 send_request_to_parser(request)
  205. else
  206. 5990 @pending << request
  207. end
  208. end
  209. 23 def timeout
  210. 11692674 return @timeout if @timeout
  211. 5930505 return @options.timeout[:connect_timeout] if @state == :idle
  212. 5930505 @options.timeout[:operation_timeout]
  213. end
  214. 23 def idling
  215. 497 purge_after_closed
  216. 497 @write_buffer.clear
  217. 497 transition(:idle)
  218. 497 @parser = nil if @parser
  219. end
  220. 23 def used?
  221. 18029 @connected_at
  222. end
  223. 23 def deactivate
  224. 809 transition(:inactive)
  225. end
  226. 23 def open?
  227. 11750 @state == :open || @state == :inactive
  228. end
  229. 23 def handle_socket_timeout(interval)
  230. 340 @intervals.delete_if(&:elapsed?)
  231. 340 unless @intervals.empty?
  232. # remove the intervals which will elapse
  233. 264 return
  234. end
  235. 24 error = HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
  236. 24 error.set_backtrace(caller)
  237. 24 on_error(error)
  238. end
  239. 23 private
  240. 23 def connect
  241. 13517 transition(:open)
  242. end
  243. 23 def consume
  244. 16363 return unless @io
  245. 16363 catch(:called) do
  246. 16363 epiped = false
  247. 16363 loop do
  248. # connection may have
  249. 31115 return if @state == :idle
  250. 28961 parser.consume
  251. # we exit if there's no more requests to process
  252. #
  253. # this condition takes into account:
  254. #
  255. # * the number of inflight requests
  256. # * the number of pending requests
  257. # * whether the write buffer has bytes (i.e. for close handshake)
  258. 28949 if @pending.empty? && @inflight.zero? && @write_buffer.empty?
  259. 2053 log(level: 3) { "NO MORE REQUESTS..." }
  260. 2041 return
  261. end
  262. 26908 @timeout = @current_timeout
  263. 26908 read_drained = false
  264. 26908 write_drained = nil
  265. #
  266. # tight read loop.
  267. #
  268. # read as much of the socket as possible.
  269. #
  270. # this tight loop reads all the data it can from the socket and pipes it to
  271. # its parser.
  272. #
  273. 2057 loop do
  274. 34358 siz = @io.read(@window_size, @read_buffer)
  275. 34449 log(level: 3, color: :cyan) { "IO READ: #{siz} bytes..." }
  276. 34358 unless siz
  277. 10 ex = EOFError.new("descriptor closed")
  278. 10 ex.set_backtrace(caller)
  279. 10 on_error(ex)
  280. 10 return
  281. end
  282. # socket has been drained. mark and exit the read loop.
  283. 34348 if siz.zero?
  284. 6416 read_drained = @read_buffer.empty?
  285. 6416 epiped = false
  286. 6416 break
  287. end
  288. 27932 parser << @read_buffer.to_s
  289. # continue reading if possible.
  290. 24978 break if interests == :w && !epiped
  291. # exit the read loop if connection is preparing to be closed
  292. 19435 break if @state == :closing || @state == :closed
  293. # exit #consume altogether if all outstanding requests have been dealt with
  294. 19429 return if @pending.empty? && @inflight.zero?
  295. 26908 end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
  296. #
  297. # tight write loop.
  298. #
  299. # flush as many bytes as the sockets allow.
  300. #
  301. 2170 loop do
  302. # buffer has been drainned, mark and exit the write loop.
  303. 17882 if @write_buffer.empty?
  304. # we only mark as drained on the first loop
  305. 2497 write_drained = write_drained.nil? && @inflight.positive?
  306. 2497 break
  307. end
  308. begin
  309. 15385 siz = @io.write(@write_buffer)
  310. rescue Errno::EPIPE
  311. # this can happen if we still have bytes in the buffer to send to the server, but
  312. # the server wants to respond immediately with some message, or an error. An example is
  313. # when one's uploading a big file to an unintended endpoint, and the server stops the
  314. # consumption, and responds immediately with an authorization of even method not allowed error.
  315. # at this point, we have to let the connection switch to read-mode.
  316. 14 log(level: 2) { "pipe broken, could not flush buffer..." }
  317. 14 epiped = true
  318. 14 read_drained = false
  319. 14 break
  320. end
  321. 15431 log(level: 3, color: :cyan) { "IO WRITE: #{siz} bytes..." }
  322. 15365 unless siz
  323. ex = EOFError.new("descriptor closed")
  324. ex.set_backtrace(caller)
  325. on_error(ex)
  326. return
  327. end
  328. # socket closed for writing. mark and exit the write loop.
  329. 15365 if siz.zero?
  330. 22 write_drained = !@write_buffer.empty?
  331. 22 break
  332. end
  333. # exit write loop if marked to consume from peer, or is closing.
  334. 15343 break if interests == :r || @state == :closing || @state == :closed
  335. 2462 write_drained = false
  336. 21823 end unless (ints = interests) == :r
  337. 21817 send_pending if @state == :open
  338. # return if socket is drained
  339. 21817 next unless (ints != :r || read_drained) && (ints != :w || write_drained)
  340. # gotta go back to the event loop. It happens when:
  341. #
  342. # * the socket is drained of bytes or it's not the interest of the conn to read;
  343. # * theres nothing more to write, or it's not in the interest of the conn to write;
  344. 7093 log(level: 3) { "(#{ints}): WAITING FOR EVENTS..." }
  345. 7065 return
  346. end
  347. end
  348. end
  349. 23 def send_pending
  350. 61845 while !@write_buffer.full? && (request = @pending.shift)
  351. 16210 send_request_to_parser(request)
  352. end
  353. end
  354. 23 def parser
  355. 79341 @parser ||= build_parser
  356. end
  357. 23 def send_request_to_parser(request)
  358. 15681 @inflight += 1
  359. 16506 request.peer_address = @io.ip
  360. 16506 parser.send(request)
  361. 16506 set_request_timeouts(request)
  362. 16506 return unless @state == :inactive
  363. 53 transition(:active)
  364. end
  365. 23 def build_parser(protocol = @io.protocol)
  366. 5056 parser = self.class.parser_type(protocol).new(@write_buffer, @options)
  367. 5056 set_parser_callbacks(parser)
  368. 5056 parser
  369. end
  370. 23 def set_parser_callbacks(parser)
  371. 5152 parser.on(:response) do |request, response|
  372. 5549 AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
  373. 6 emit(:altsvc, alt_origin, origin, alt_params)
  374. end
  375. 5549 @response_received_at = Utils.now
  376. 4798 @inflight -= 1
  377. 5549 request.emit(:response, response)
  378. end
  379. 5152 parser.on(:altsvc) do |alt_origin, origin, alt_params|
  380. emit(:altsvc, alt_origin, origin, alt_params)
  381. end
  382. 5152 parser.on(:pong, &method(:send_pending))
  383. 5152 parser.on(:promise) do |request, stream|
  384. 18 request.emit(:promise, parser, stream)
  385. end
  386. 5152 parser.on(:exhausted) do
  387. 6 @pending.concat(parser.pending)
  388. 6 emit(:exhausted)
  389. end
  390. 5152 parser.on(:origin) do |origin|
  391. @origins |= [origin]
  392. end
  393. 5152 parser.on(:close) do |force|
  394. 4562 if force
  395. 4562 reset
  396. 4556 emit(:terminate)
  397. end
  398. end
  399. 5152 parser.on(:close_handshake) do
  400. 6 consume
  401. end
  402. 5152 parser.on(:reset) do
  403. 2671 @pending.concat(parser.pending) unless parser.empty?
  404. 2671 reset
  405. 2665 idling unless @pending.empty?
  406. end
  407. 5152 parser.on(:current_timeout) do
  408. 2189 @current_timeout = @timeout = parser.timeout
  409. end
  410. 5152 parser.on(:timeout) do |tout|
  411. 2074 @timeout = tout
  412. end
  413. 5152 parser.on(:error) do |request, ex|
  414. 285 case ex
  415. when MisdirectedRequestError
  416. 6 emit(:misdirected, request)
  417. else
  418. 334 response = ErrorResponse.new(request, ex, @options)
  419. 334 request.response = response
  420. 334 request.emit(:response, response)
  421. end
  422. end
  423. end
  424. 23 def transition(nextstate)
  425. 32394 handle_transition(nextstate)
  426. rescue Errno::ECONNABORTED,
  427. Errno::ECONNREFUSED,
  428. Errno::ECONNRESET,
  429. Errno::EADDRNOTAVAIL,
  430. Errno::EHOSTUNREACH,
  431. Errno::EINVAL,
  432. Errno::ENETUNREACH,
  433. Errno::EPIPE,
  434. Errno::ENOENT,
  435. SocketError,
  436. IOError => e
  437. # connect errors, exit gracefully
  438. 60 error = ConnectionError.new(e.message)
  439. 60 error.set_backtrace(e.backtrace)
  440. 60 connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, error) : handle_error(error)
  441. 60 @state = :closed
  442. 60 emit(:close)
  443. rescue TLSError, HTTP2Next::Error::ProtocolError, HTTP2Next::Error::HandshakeError => e
  444. # connect errors, exit gracefully
  445. 19 handle_error(e)
  446. 19 connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, e) : handle_error(e)
  447. 19 @state = :closed
  448. 19 emit(:close)
  449. end
  450. 23 def handle_transition(nextstate)
  451. 27507 case nextstate
  452. when :idle
  453. 5574 @timeout = @current_timeout = @options.timeout[:connect_timeout]
  454. 5574 @connected_at = nil
  455. when :open
  456. 13761 return if @state == :closed
  457. 13761 @io.connect
  458. 13683 emit(:tcp_open, self) if @io.state == :connected
  459. 13683 return unless @io.connected?
  460. 5070 @connected_at = Utils.now
  461. 5070 send_pending
  462. 5070 @timeout = @current_timeout = parser.timeout
  463. 5070 emit(:open)
  464. when :inactive
  465. 809 return unless @state == :open
  466. when :closing
  467. 5409 return unless @state == :idle || @state == :open
  468. 5384 unless @write_buffer.empty?
  469. # preset state before handshake, as error callbacks
  470. # may take it back here.
  471. 1971 @state = nextstate
  472. # handshakes, try sending
  473. 1971 consume
  474. 1970 @write_buffer.clear
  475. 1970 return
  476. end
  477. when :closed
  478. 5483 return unless @state == :closing
  479. 5463 return unless @write_buffer.empty?
  480. 5445 purge_after_closed
  481. 5445 emit(:close) if @pending.empty?
  482. when :already_open
  483. 52 nextstate = :open
  484. # the first check for given io readiness must still use a timeout.
  485. # connect is the reasonable choice in such a case.
  486. 52 @timeout = @options.timeout[:connect_timeout]
  487. 52 send_pending
  488. when :active
  489. 460 return unless @state == :inactive
  490. 460 nextstate = :open
  491. 460 emit(:activate)
  492. end
  493. 20935 @state = nextstate
  494. end
  495. 23 def purge_after_closed
  496. 5948 @io.close if @io
  497. 5948 @read_buffer.clear
  498. 5948 @timeout = nil
  499. end
  500. 23 def initialize_type(uri, options)
  501. 4834 options.transport || begin
  502. 4109 case uri.scheme
  503. when "http"
  504. 2633 "tcp"
  505. when "https"
  506. 2177 "ssl"
  507. else
  508. raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
  509. end
  510. end
  511. end
  512. 23 def build_socket(addrs = nil)
  513. 4096 case @type
  514. when "tcp"
  515. 2709 TCP.new(@origin, addrs, @options)
  516. when "ssl"
  517. 2095 SSL.new(@origin, addrs, @options) do |sock|
  518. 2078 sock.ssl_session = @ssl_session
  519. 2078 sock.session_new_cb do |sess|
  520. 3999 @ssl_session = sess
  521. 3999 sock.ssl_session = sess
  522. end
  523. end
  524. when "unix"
  525. 24 UNIX.new(@origin, addrs, @options)
  526. else
  527. raise Error, "unsupported transport (#{@type})"
  528. end
  529. end
  530. 23 def on_error(error)
  531. 615 if error.instance_of?(TimeoutError)
  532. # inactive connections do not contribute to the select loop, therefore
  533. # they should not fail due to such errors.
  534. 24 return if @state == :inactive
  535. 24 if @timeout
  536. 20 @timeout -= error.timeout
  537. 24 return unless @timeout <= 0
  538. end
  539. 24 error = error.to_connection_error if connecting?
  540. end
  541. 615 handle_error(error)
  542. 615 reset
  543. end
  544. 23 def handle_error(error)
  545. 713 parser.handle_error(error) if @parser && parser.respond_to?(:handle_error)
  546. 1666 while (request = @pending.shift)
  547. 333 response = ErrorResponse.new(request, error, request.options)
  548. 333 request.response = response
  549. 333 request.emit(:response, response)
  550. end
  551. end
  552. 23 def set_request_timeouts(request)
  553. 16506 write_timeout = request.write_timeout
  554. 16506 read_timeout = request.read_timeout
  555. 16506 request_timeout = request.request_timeout
  556. 16506 unless write_timeout.nil? || write_timeout.infinite?
  557. 16506 set_request_timeout(request, write_timeout, :headers, %i[done response]) do
  558. 18 write_timeout_callback(request, write_timeout)
  559. end
  560. end
  561. 16506 unless read_timeout.nil? || read_timeout.infinite?
  562. 16293 set_request_timeout(request, read_timeout, :done, :response) do
  563. 18 read_timeout_callback(request, read_timeout)
  564. end
  565. end
  566. 16506 return if request_timeout.nil? || request_timeout.infinite?
  567. 348 set_request_timeout(request, request_timeout, :headers, :response) do
  568. 260 read_timeout_callback(request, request_timeout, RequestTimeoutError)
  569. end
  570. end
  571. 23 def write_timeout_callback(request, write_timeout)
  572. 18 return if request.state == :done
  573. 18 @write_buffer.clear
  574. 18 error = WriteTimeoutError.new(request, nil, write_timeout)
  575. 18 on_error(error)
  576. end
  577. 23 def read_timeout_callback(request, read_timeout, error_type = ReadTimeoutError)
  578. 278 response = request.response
  579. 278 return if response && response.finished?
  580. 278 @write_buffer.clear
  581. 278 error = error_type.new(request, request.response, read_timeout)
  582. 278 on_error(error)
  583. end
  584. 23 def set_request_timeout(request, timeout, start_event, finish_events, &callback)
  585. 33207 request.once(start_event) do
  586. 32685 interval = @timers.after(timeout, callback)
  587. 32685 Array(finish_events).each do |event|
  588. # clean up request timeouts if the connection errors out
  589. 49002 request.once(event) do
  590. 48888 if @intervals.include?(interval)
  591. 48589 interval.delete(callback)
  592. 48589 @intervals.delete(interval) if interval.no_callbacks?
  593. end
  594. end
  595. end
  596. 32685 @intervals << interval
  597. end
  598. end
  599. 23 class << self
  600. 23 def parser_type(protocol)
  601. 4403 case protocol
  602. 2194 when "h2" then HTTP2
  603. 2984 when "http/1.1" then HTTP1
  604. else
  605. raise Error, "unsupported protocol (##{protocol})"
  606. end
  607. end
  608. end
  609. end
  610. end

lib/httpx/connection/http1.rb

90.91% lines covered

220 relevant lines. 200 lines covered and 20 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "httpx/parser/http1"
  3. 23 module HTTPX
  4. 23 class Connection::HTTP1
  5. 23 include Callbacks
  6. 23 include Loggable
  7. 23 MAX_REQUESTS = 200
  8. 23 CRLF = "\r\n"
  9. 23 attr_reader :pending, :requests
  10. 23 attr_accessor :max_concurrent_requests
  11. 23 def initialize(buffer, options)
  12. 2984 @options = Options.new(options)
  13. 2984 @max_concurrent_requests = @options.max_concurrent_requests || MAX_REQUESTS
  14. 2984 @max_requests = @options.max_requests
  15. 2984 @parser = Parser::HTTP1.new(self)
  16. 2984 @buffer = buffer
  17. 2984 @version = [1, 1]
  18. 2984 @pending = []
  19. 2984 @requests = []
  20. 2984 @handshake_completed = false
  21. end
  22. 23 def timeout
  23. 2891 @options.timeout[:operation_timeout]
  24. end
  25. 23 def interests
  26. # this means we're processing incoming response already
  27. 453232 return :r if @request
  28. 450635 return if @requests.empty?
  29. 18733 request = @requests.first
  30. 18733 return unless request
  31. 18733 return :w if request.interests == :w || !@buffer.empty?
  32. 15848 :r
  33. end
  34. 23 def reset
  35. 5210 @max_requests = @options.max_requests || MAX_REQUESTS
  36. 5210 @parser.reset!
  37. 5210 @handshake_completed = false
  38. 5210 @pending.concat(@requests) unless @requests.empty?
  39. end
  40. 23 def close
  41. 2503 reset
  42. 2503 emit(:close, true)
  43. end
  44. 23 def exhausted?
  45. 539 !@max_requests.positive?
  46. end
  47. 23 def empty?
  48. # this means that for every request there's an available
  49. # partial response, so there are no in-flight requests waiting.
  50. 5148 @requests.empty? || (
  51. # checking all responses can be time-consuming. Alas, as in HTTP/1, responses
  52. # do not come out of order, we can get away with checking first and last.
  53. 276 !@requests.first.response.nil? &&
  54. 97 (@requests.size == 1 || !@requests.last.response.nil?)
  55. )
  56. end
  57. 23 def <<(data)
  58. 4903 @parser << data
  59. end
  60. 23 def send(request)
  61. 14031 unless @max_requests.positive?
  62. @pending << request
  63. return
  64. end
  65. 14031 return if @requests.include?(request)
  66. 14031 @requests << request
  67. 14031 @pipelining = true if @requests.size > 1
  68. end
  69. 23 def consume
  70. 11459 requests_limit = [@max_requests, @requests.size].min
  71. 11459 concurrent_requests_limit = [@max_concurrent_requests, requests_limit].min
  72. 11459 @requests.each_with_index do |request, idx|
  73. 13572 break if idx >= concurrent_requests_limit
  74. 11403 next if request.state == :done
  75. 4691 handle(request)
  76. end
  77. end
  78. # HTTP Parser callbacks
  79. #
  80. # must be public methods, or else they won't be reachable
  81. 23 def on_start
  82. 3374 log(level: 2) { "parsing begins" }
  83. end
  84. 23 def on_headers(h)
  85. 3356 @request = @requests.first
  86. 3356 return if @request.response
  87. 3374 log(level: 2) { "headers received" }
  88. 3356 headers = @request.options.headers_class.new(h)
  89. 3356 response = @request.options.response_class.new(@request,
  90. @parser.status_code,
  91. @parser.http_version.join("."),
  92. headers)
  93. 3374 log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
  94. 3518 log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
  95. 3356 @request.response = response
  96. 3350 on_complete if response.finished?
  97. end
  98. 23 def on_trailers(h)
  99. 6 return unless @request
  100. 6 response = @request.response
  101. 6 log(level: 2) { "trailer headers received" }
  102. 6 log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{v.join(", ")}" }.join("\n") }
  103. 6 response.merge_headers(h)
  104. end
  105. 23 def on_data(chunk)
  106. 3789 request = @request
  107. 3789 return unless request
  108. 3807 log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
  109. 3807 log(level: 2, color: :green) { "-> #{chunk.inspect}" }
  110. 3789 response = request.response
  111. 3789 response << chunk
  112. rescue StandardError => e
  113. 11 error_response = ErrorResponse.new(request, e, request.options)
  114. 11 request.response = error_response
  115. 11 dispatch
  116. end
  117. 23 def on_complete
  118. 3333 request = @request
  119. 3333 return unless request
  120. 3351 log(level: 2) { "parsing complete" }
  121. 3333 dispatch
  122. end
  123. 23 def dispatch
  124. 3344 request = @request
  125. 3344 if request.expects?
  126. 54 @parser.reset!
  127. 45 return handle(request)
  128. end
  129. 3290 @request = nil
  130. 3290 @requests.shift
  131. 3290 response = request.response
  132. 3290 response.finish! unless response.is_a?(ErrorResponse)
  133. 3290 emit(:response, request, response)
  134. 3246 if @parser.upgrade?
  135. 24 response << @parser.upgrade_data
  136. 24 throw(:called)
  137. end
  138. 3222 @parser.reset!
  139. 2810 @max_requests -= 1
  140. 3222 if response.is_a?(ErrorResponse)
  141. 11 disable
  142. else
  143. 3211 manage_connection(request, response)
  144. end
  145. 539 if exhausted?
  146. @pending.concat(@requests)
  147. @requests.clear
  148. emit(:exhausted)
  149. else
  150. 539 send(@pending.shift) unless @pending.empty?
  151. end
  152. end
  153. 23 def handle_error(ex)
  154. 158 if (ex.is_a?(EOFError) || ex.is_a?(TimeoutError)) && @request && @request.response &&
  155. !@request.response.headers.key?("content-length") &&
  156. !@request.response.headers.key?("transfer-encoding")
  157. # if the response does not contain a content-length header, the server closing the
  158. # connnection is the indicator of response consumed.
  159. # https://greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4
  160. 12 catch(:called) { on_complete }
  161. 5 return
  162. end
  163. 152 if @pipelining
  164. catch(:called) { disable }
  165. else
  166. 152 @requests.each do |request|
  167. 146 emit(:error, request, ex)
  168. end
  169. 152 @pending.each do |request|
  170. emit(:error, request, ex)
  171. end
  172. end
  173. end
  174. 23 def ping
  175. reset
  176. emit(:reset)
  177. emit(:exhausted)
  178. end
  179. 23 private
  180. 23 def manage_connection(request, response)
  181. 3211 connection = response.headers["connection"]
  182. 2800 case connection
  183. when /keep-alive/i
  184. 539 if @handshake_completed
  185. if @max_requests.zero?
  186. @pending.concat(@requests)
  187. @requests.clear
  188. emit(:exhausted)
  189. end
  190. return
  191. end
  192. 539 keep_alive = response.headers["keep-alive"]
  193. 539 return unless keep_alive
  194. 112 parameters = Hash[keep_alive.split(/ *, */).map do |pair|
  195. 112 pair.split(/ *= */)
  196. end]
  197. 112 @max_requests = parameters["max"].to_i - 1 if parameters.key?("max")
  198. 112 if parameters.key?("timeout")
  199. keep_alive_timeout = parameters["timeout"].to_i
  200. emit(:timeout, keep_alive_timeout)
  201. end
  202. 112 @handshake_completed = true
  203. when /close/i
  204. 2672 disable
  205. when nil
  206. # In HTTP/1.1, it's keep alive by default
  207. return if response.version == "1.1" && request.headers["connection"] != "close"
  208. disable
  209. end
  210. end
  211. 23 def disable
  212. 2683 disable_pipelining
  213. 2683 reset
  214. 2683 emit(:reset)
  215. 2677 throw(:called)
  216. end
  217. 23 def disable_pipelining
  218. 2683 return if @requests.empty?
  219. # do not disable pipelining if already set to 1 request at a time
  220. 161 return if @max_concurrent_requests == 1
  221. 17 @requests.each do |r|
  222. 17 r.transition(:idle)
  223. # when we disable pipelining, we still want to try keep-alive.
  224. # only when keep-alive with one request fails, do we fallback to
  225. # connection: close.
  226. 17 r.headers["connection"] = "close" if @max_concurrent_requests == 1
  227. end
  228. # server doesn't handle pipelining, and probably
  229. # doesn't support keep-alive. Fallback to send only
  230. # 1 keep alive request.
  231. 17 @max_concurrent_requests = 1
  232. 17 @pipelining = false
  233. end
  234. 23 def set_protocol_headers(request)
  235. 3471 if !request.headers.key?("content-length") &&
  236. request.body.bytesize == Float::INFINITY
  237. 24 request.body.chunk!
  238. end
  239. 3471 extra_headers = {}
  240. 3471 unless request.headers.key?("connection")
  241. 3453 connection_value = if request.persistent?
  242. # when in a persistent connection, the request can't be at
  243. # the edge of a renegotiation
  244. 803 if @requests.index(request) + 1 < @max_requests
  245. 693 "keep-alive"
  246. else
  247. 110 "close"
  248. end
  249. else
  250. # when it's not a persistent connection, it sets "Connection: close" always
  251. # on the last request of the possible batch (either allowed max requests,
  252. # or if smaller, the size of the batch itself)
  253. 2650 requests_limit = [@max_requests, @requests.size].min
  254. 2650 if request == @requests[requests_limit - 1]
  255. 2615 "close"
  256. else
  257. 35 "keep-alive"
  258. end
  259. end
  260. 3005 extra_headers["connection"] = connection_value
  261. end
  262. 3471 extra_headers["host"] = request.authority unless request.headers.key?("host")
  263. 3471 extra_headers
  264. end
  265. 23 def handle(request)
  266. 4745 catch(:buffer_full) do
  267. 4745 request.transition(:headers)
  268. 4739 join_headers(request) if request.state == :headers
  269. 4739 request.transition(:body)
  270. 4739 join_body(request) if request.state == :body
  271. 3611 request.transition(:trailers)
  272. # HTTP/1.1 trailers should only work for chunked encoding
  273. 3611 join_trailers(request) if request.body.chunked? && request.state == :trailers
  274. 3611 request.transition(:done)
  275. end
  276. end
  277. 23 def join_headline(request)
  278. 2969 "#{request.verb} #{request.path} HTTP/#{@version.join(".")}"
  279. end
  280. 23 def join_headers(request)
  281. 3471 headline = join_headline(request)
  282. 3471 @buffer << headline << CRLF
  283. 3489 log(color: :yellow) { "<- HEADLINE: #{headline.chomp.inspect}" }
  284. 3471 extra_headers = set_protocol_headers(request)
  285. 3471 join_headers2(request.headers.each(extra_headers))
  286. 3489 log { "<- " }
  287. 3471 @buffer << CRLF
  288. end
  289. 23 def join_body(request)
  290. 4581 return if request.body.empty?
  291. 5596 while (chunk = request.drain_body)
  292. 2679 log(color: :green) { "<- DATA: #{chunk.bytesize} bytes..." }
  293. 2679 log(level: 2, color: :green) { "<- #{chunk.inspect}" }
  294. 2679 @buffer << chunk
  295. 2679 throw(:buffer_full, request) if @buffer.full?
  296. end
  297. 1077 return unless (error = request.drain_error)
  298. raise error
  299. end
  300. 23 def join_trailers(request)
  301. 72 return unless request.trailers? && request.callbacks_for?(:trailers)
  302. 24 join_headers2(request.trailers)
  303. 24 log { "<- " }
  304. 24 @buffer << CRLF
  305. end
  306. 23 def join_headers2(headers)
  307. 3495 headers.each do |field, value|
  308. 20928 buffer = "#{capitalized(field)}: #{value}#{CRLF}"
  309. 21018 log(color: :yellow) { "<- HEADER: #{buffer.chomp}" }
  310. 20928 @buffer << buffer
  311. end
  312. end
  313. 23 UPCASED = {
  314. "www-authenticate" => "WWW-Authenticate",
  315. "http2-settings" => "HTTP2-Settings",
  316. }.freeze
  317. 23 def capitalized(field)
  318. 20928 UPCASED[field] || field.split("-").map(&:capitalize).join("-")
  319. end
  320. end
  321. end

lib/httpx/connection/http2.rb

95.93% lines covered

246 relevant lines. 236 lines covered and 10 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "securerandom"
  3. 23 require "http/2/next"
  4. 23 module HTTPX
  5. 23 class Connection::HTTP2
  6. 23 include Callbacks
  7. 23 include Loggable
  8. 23 MAX_CONCURRENT_REQUESTS = HTTP2Next::DEFAULT_MAX_CONCURRENT_STREAMS
  9. 23 class Error < Error
  10. 23 def initialize(id, code)
  11. 28 super("stream #{id} closed with error: #{code}")
  12. end
  13. end
  14. 23 class GoawayError < Error
  15. 23 def initialize
  16. 12 super(0, :no_error)
  17. end
  18. end
  19. 23 attr_reader :streams, :pending
  20. 23 def initialize(buffer, options)
  21. 2212 @options = Options.new(options)
  22. 2212 @settings = @options.http2_settings
  23. 2212 @pending = []
  24. 2212 @streams = {}
  25. 2212 @drains = {}
  26. 2212 @pings = []
  27. 2212 @buffer = buffer
  28. 2212 @handshake_completed = false
  29. 2212 @wait_for_handshake = @settings.key?(:wait_for_handshake) ? @settings.delete(:wait_for_handshake) : true
  30. 2212 @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
  31. 2212 @max_requests = @options.max_requests
  32. 2212 init_connection
  33. end
  34. 23 def timeout
  35. 4368 return @options.timeout[:operation_timeout] if @handshake_completed
  36. 2179 @options.timeout[:settings_timeout]
  37. end
  38. 23 def interests
  39. # waiting for WINDOW_UPDATE frames
  40. 1524182 return :r if @buffer.full?
  41. 1524182 if @connection.state == :closed
  42. 2084 return unless @handshake_completed
  43. 1750 return :w
  44. end
  45. 1522098 unless @connection.state == :connected && @handshake_completed
  46. 7705 return @buffer.empty? ? :r : :rw
  47. end
  48. 1513072 return :w if !@pending.empty? && can_buffer_more_requests?
  49. 1513072 return :w unless @drains.empty?
  50. 1512424 if @buffer.empty?
  51. 1512424 return if @streams.empty? && @pings.empty?
  52. 25125 return :r
  53. end
  54. :rw
  55. end
  56. 23 def close
  57. 2085 unless @connection.state == :closed
  58. 2074 @connection.goaway
  59. 2074 emit(:timeout, @options.timeout[:close_handshake_timeout])
  60. end
  61. 2085 emit(:close, true)
  62. end
  63. 23 def empty?
  64. 2050 @connection.state == :closed || @streams.empty?
  65. end
  66. 23 def exhausted?
  67. 2203 !@max_requests.positive?
  68. end
  69. 23 def <<(data)
  70. 22801 @connection << data
  71. end
  72. 23 def can_buffer_more_requests?
  73. 5218 (@handshake_completed || !@wait_for_handshake) &&
  74. @streams.size < @max_concurrent_requests &&
  75. @streams.size < @max_requests
  76. end
  77. 23 def send(request)
  78. 4854 unless can_buffer_more_requests?
  79. 2360 @pending << request
  80. 2360 return
  81. end
  82. 2494 unless (stream = @streams[request])
  83. 2494 stream = @connection.new_stream
  84. 2494 handle_stream(stream, request)
  85. 2128 @streams[request] = stream
  86. 2128 @max_requests -= 1
  87. end
  88. 2494 handle(request, stream)
  89. 2482 true
  90. rescue HTTP2Next::Error::StreamLimitExceeded
  91. @pending.unshift(request)
  92. end
  93. 23 def consume
  94. 16917 @streams.each do |request, stream|
  95. 6403 next if request.state == :done
  96. 739 handle(request, stream)
  97. end
  98. end
  99. 23 def handle_error(ex)
  100. 207 if ex.instance_of?(TimeoutError) && !@handshake_completed && @connection.state != :closed
  101. 6 @connection.goaway(:settings_timeout, "closing due to settings timeout")
  102. 6 emit(:close_handshake)
  103. 6 settings_ex = SettingsTimeoutError.new(ex.timeout, ex.message)
  104. 6 settings_ex.set_backtrace(ex.backtrace)
  105. 6 ex = settings_ex
  106. end
  107. 207 @streams.each_key do |request|
  108. 161 emit(:error, request, ex)
  109. end
  110. 207 @pending.each do |request|
  111. 27 emit(:error, request, ex)
  112. end
  113. end
  114. 23 def ping
  115. 6 ping = SecureRandom.gen_random(8)
  116. 6 @connection.ping(ping)
  117. ensure
  118. 6 @pings << ping
  119. end
  120. 23 private
  121. 23 def send_pending
  122. 6258 while (request = @pending.shift)
  123. # TODO: this request should go back to top of stack
  124. 2269 break unless send(request)
  125. end
  126. end
  127. 23 def handle(request, stream)
  128. 3275 catch(:buffer_full) do
  129. 3275 request.transition(:headers)
  130. 3269 join_headers(stream, request) if request.state == :headers
  131. 3269 request.transition(:body)
  132. 3269 join_body(stream, request) if request.state == :body
  133. 2615 request.transition(:trailers)
  134. 2615 join_trailers(stream, request) if request.state == :trailers && !request.body.empty?
  135. 2615 request.transition(:done)
  136. end
  137. end
  138. 23 def init_connection
  139. 2212 @connection = HTTP2Next::Client.new(@settings)
  140. 2212 @connection.on(:frame, &method(:on_frame))
  141. 2212 @connection.on(:frame_sent, &method(:on_frame_sent))
  142. 2212 @connection.on(:frame_received, &method(:on_frame_received))
  143. 2212 @connection.on(:origin, &method(:on_origin))
  144. 2212 @connection.on(:promise, &method(:on_promise))
  145. 2212 @connection.on(:altsvc) { |frame| on_altsvc(frame[:origin], frame) }
  146. 2212 @connection.on(:settings_ack, &method(:on_settings))
  147. 2212 @connection.on(:ack, &method(:on_pong))
  148. 2212 @connection.on(:goaway, &method(:on_close))
  149. #
  150. # Some servers initiate HTTP/2 negotiation right away, some don't.
  151. # As such, we have to check the socket buffer. If there is something
  152. # to read, the server initiated the negotiation. If not, we have to
  153. # initiate it.
  154. #
  155. 2212 @connection.send_connection_preface
  156. end
  157. 23 alias_method :reset, :init_connection
  158. 23 public :reset
  159. 23 def handle_stream(stream, request)
  160. 2506 request.on(:refuse, &method(:on_stream_refuse).curry(3)[stream, request])
  161. 2506 stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
  162. 2506 stream.on(:half_close) do
  163. 2488 log(level: 2) { "#{stream.id}: waiting for response..." }
  164. end
  165. 2506 stream.on(:altsvc, &method(:on_altsvc).curry(2)[request.origin])
  166. 2506 stream.on(:headers, &method(:on_stream_headers).curry(3)[stream, request])
  167. 2506 stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
  168. end
  169. 23 def set_protocol_headers(request)
  170. 365 {
  171. 2123 ":scheme" => request.scheme,
  172. ":method" => request.verb,
  173. ":path" => request.path,
  174. ":authority" => request.authority,
  175. }
  176. end
  177. 23 def join_headers(stream, request)
  178. 2488 extra_headers = set_protocol_headers(request)
  179. 2488 if request.headers.key?("host")
  180. 6 log { "forbidden \"host\" header found (#{request.headers["host"]}), will use it as authority..." }
  181. 5 extra_headers[":authority"] = request.headers["host"]
  182. end
  183. 2488 log(level: 1, color: :yellow) do
  184. 92 request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
  185. end
  186. 2488 stream.headers(request.headers.each(extra_headers), end_stream: request.body.empty?)
  187. end
  188. 23 def join_trailers(stream, request)
  189. 1024 unless request.trailers?
  190. 1018 stream.data("", end_stream: true) if request.callbacks_for?(:trailers)
  191. 877 return
  192. end
  193. 6 log(level: 1, color: :yellow) do
  194. 11 request.trailers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
  195. end
  196. 6 stream.headers(request.trailers.each, end_stream: true)
  197. end
  198. 23 def join_body(stream, request)
  199. 3142 return if request.body.empty?
  200. 1678 chunk = @drains.delete(request) || request.drain_body
  201. 1850 while chunk
  202. 1848 next_chunk = request.drain_body
  203. 1863 log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
  204. 1863 log(level: 2, color: :green) { "#{stream.id}: -> #{chunk.inspect}" }
  205. 1848 stream.data(chunk, end_stream: !(next_chunk || request.trailers? || request.callbacks_for?(:trailers)))
  206. 1848 if next_chunk && (@buffer.full? || request.body.unbounded_body?)
  207. 550 @drains[request] = next_chunk
  208. 654 throw(:buffer_full)
  209. end
  210. 1194 chunk = next_chunk
  211. end
  212. 1024 return unless (error = request.drain_error)
  213. 10 on_stream_refuse(stream, request, error)
  214. end
  215. ######
  216. # HTTP/2 Callbacks
  217. ######
  218. 23 def on_stream_headers(stream, request, h)
  219. 2476 response = request.response
  220. 2476 if response.is_a?(Response) && response.version == "2.0"
  221. 95 on_stream_trailers(stream, response, h)
  222. 95 return
  223. end
  224. 2381 log(color: :yellow) do
  225. 92 h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
  226. end
  227. 2381 _, status = h.shift
  228. 2381 headers = request.options.headers_class.new(h)
  229. 2381 response = request.options.response_class.new(request, status, "2.0", headers)
  230. 2381 request.response = response
  231. 2027 @streams[request] = stream
  232. 2375 handle(request, stream) if request.expects?
  233. end
  234. 23 def on_stream_trailers(stream, response, h)
  235. 95 log(color: :yellow) do
  236. h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
  237. end
  238. 95 response.merge_headers(h)
  239. end
  240. 23 def on_stream_data(stream, request, data)
  241. 4219 log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
  242. 4219 log(level: 2, color: :green) { "#{stream.id}: <- #{data.inspect}" }
  243. 4204 request.response << data
  244. end
  245. 23 def on_stream_refuse(stream, request, error)
  246. 10 on_stream_close(stream, request, error)
  247. 10 stream.close
  248. end
  249. 23 def on_stream_close(stream, request, error)
  250. 2325 return if error == :stream_closed && !@streams.key?(request)
  251. 2325 log(level: 2) { "#{stream.id}: closing stream" }
  252. 2315 @drains.delete(request)
  253. 2315 @streams.delete(request)
  254. 2315 if error
  255. 10 ex = Error.new(stream.id, error)
  256. 10 ex.set_backtrace(caller)
  257. 10 response = ErrorResponse.new(request, ex, request.options)
  258. 10 request.response = response
  259. 10 emit(:response, request, response)
  260. else
  261. 2305 response = request.response
  262. 2305 if response && response.is_a?(Response) && response.status == 421
  263. 6 ex = MisdirectedRequestError.new(response)
  264. 6 ex.set_backtrace(caller)
  265. 6 emit(:error, request, ex)
  266. else
  267. 2299 emit(:response, request, response)
  268. end
  269. end
  270. 2309 send(@pending.shift) unless @pending.empty?
  271. 2309 return unless @streams.empty? && exhausted?
  272. 6 close
  273. 6 emit(:exhausted) unless @pending.empty?
  274. end
  275. 23 def on_frame(bytes)
  276. 13921 @buffer << bytes
  277. end
  278. 23 def on_settings(*)
  279. 2189 @handshake_completed = true
  280. 2189 emit(:current_timeout)
  281. 2189 @max_concurrent_requests = [@max_concurrent_requests, @connection.remote_settings[:settings_max_concurrent_streams]].min
  282. 2189 send_pending
  283. end
  284. 23 def on_close(_last_frame, error, _payload)
  285. 18 is_connection_closed = @connection.state == :closed
  286. 18 if error
  287. 18 @buffer.clear if is_connection_closed
  288. 18 if error == :no_error
  289. 12 ex = GoawayError.new
  290. 12 @pending.unshift(*@streams.keys)
  291. 12 @drains.clear
  292. 12 @streams.clear
  293. else
  294. 6 ex = Error.new(0, error)
  295. end
  296. 18 ex.set_backtrace(caller)
  297. 18 handle_error(ex)
  298. end
  299. 18 return unless is_connection_closed && @streams.empty?
  300. 18 emit(:close, is_connection_closed)
  301. end
  302. 23 def on_frame_sent(frame)
  303. 11763 log(level: 2) { "#{frame[:stream]}: frame was sent!" }
  304. 11703 log(level: 2, color: :blue) do
  305. 72 payload = frame
  306. 72 payload = payload.merge(payload: frame[:payload].bytesize) if frame[:type] == :data
  307. 60 "#{frame[:stream]}: #{payload}"
  308. end
  309. end
  310. 23 def on_frame_received(frame)
  311. 12124 log(level: 2) { "#{frame[:stream]}: frame was received!" }
  312. 12079 log(level: 2, color: :magenta) do
  313. 54 payload = frame
  314. 54 payload = payload.merge(payload: frame[:payload].bytesize) if frame[:type] == :data
  315. 45 "#{frame[:stream]}: #{payload}"
  316. end
  317. end
  318. 23 def on_altsvc(origin, frame)
  319. log(level: 2) { "#{frame[:stream]}: altsvc frame was received" }
  320. log(level: 2) { "#{frame[:stream]}: #{frame.inspect}" }
  321. alt_origin = URI.parse("#{frame[:proto]}://#{frame[:host]}:#{frame[:port]}")
  322. params = { "ma" => frame[:max_age] }
  323. emit(:altsvc, origin, alt_origin, origin, params)
  324. end
  325. 23 def on_promise(stream)
  326. 18 emit(:promise, @streams.key(stream.parent), stream)
  327. end
  328. 23 def on_origin(origin)
  329. emit(:origin, origin)
  330. end
  331. 23 def on_pong(ping)
  332. 6 if @pings.delete(ping.to_s)
  333. 6 emit(:pong)
  334. else
  335. close(:protocol_error, "ping payload did not match")
  336. end
  337. end
  338. end
  339. end

lib/httpx/domain_name.rb

95.45% lines covered

44 relevant lines. 42 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. #
  3. # domain_name.rb - Domain Name manipulation library for Ruby
  4. #
  5. # Copyright (C) 2011-2017 Akinori MUSHA, All rights reserved.
  6. #
  7. # Redistribution and use in source and binary forms, with or without
  8. # modification, are permitted provided that the following conditions
  9. # are met:
  10. # 1. Redistributions of source code must retain the above copyright
  11. # notice, this list of conditions and the following disclaimer.
  12. # 2. Redistributions in binary form must reproduce the above copyright
  13. # notice, this list of conditions and the following disclaimer in the
  14. # documentation and/or other materials provided with the distribution.
  15. #
  16. # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17. # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19. # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20. # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24. # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25. # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26. # SUCH DAMAGE.
  27. 23 require "ipaddr"
  28. 23 module HTTPX
  29. # Represents a domain name ready for extracting its registered domain
  30. # and TLD.
  31. 23 class DomainName
  32. 23 include Comparable
  33. # The full host name normalized, ASCII-ized and downcased using the
  34. # Unicode NFC rules and the Punycode algorithm. If initialized with
  35. # an IP address, the string representation of the IP address
  36. # suitable for opening a connection to.
  37. 23 attr_reader :hostname
  38. # The Unicode representation of the #hostname property.
  39. #
  40. # :attr_reader: hostname_idn
  41. # The least "universally original" domain part of this domain name.
  42. # For example, "example.co.uk" for "www.sub.example.co.uk". This
  43. # may be nil if the hostname does not have one, like when it is an
  44. # IP address, an effective TLD or higher itself, or of a
  45. # non-canonical domain.
  46. 23 attr_reader :domain
  47. 23 class << self
  48. 23 def new(domain)
  49. 642 return domain if domain.is_a?(self)
  50. 594 super(domain)
  51. end
  52. # Normalizes a _domain_ using the Punycode algorithm as necessary.
  53. # The result will be a downcased, ASCII-only string.
  54. 23 def normalize(domain)
  55. 570 unless domain.ascii_only?
  56. domain = domain.chomp(".").unicode_normalize(:nfc)
  57. domain = Punycode.encode_hostname(domain)
  58. end
  59. 570 domain.downcase
  60. end
  61. end
  62. # Parses _hostname_ into a DomainName object. An IP address is also
  63. # accepted. An IPv6 address may be enclosed in square brackets.
  64. 23 def initialize(hostname)
  65. 594 hostname = String(hostname)
  66. 594 raise ArgumentError, "domain name must not start with a dot: #{hostname}" if hostname.start_with?(".")
  67. begin
  68. 594 @ipaddr = IPAddr.new(hostname)
  69. 24 @hostname = @ipaddr.to_s
  70. 24 return
  71. rescue IPAddr::Error
  72. 570 nil
  73. end
  74. 570 @hostname = DomainName.normalize(hostname)
  75. 570 tld = if (last_dot = @hostname.rindex("."))
  76. 138 @hostname[(last_dot + 1)..-1]
  77. else
  78. 432 @hostname
  79. end
  80. # unknown/local TLD
  81. 570 @domain = if last_dot
  82. # fallback - accept cookies down to second level
  83. # cf. http://www.dkim-reputation.org/regdom-libs/
  84. 138 if (penultimate_dot = @hostname.rindex(".", last_dot - 1))
  85. 36 @hostname[(penultimate_dot + 1)..-1]
  86. else
  87. 102 @hostname
  88. end
  89. else
  90. # no domain part - must be a local hostname
  91. 432 tld
  92. end
  93. end
  94. # Checks if the server represented by this domain is qualified to
  95. # send and receive cookies with a domain attribute value of
  96. # _domain_. A true value given as the second argument represents
  97. # cookies without a domain attribute value, in which case only
  98. # hostname equality is checked.
  99. 23 def cookie_domain?(domain, host_only = false)
  100. # RFC 6265 #5.3
  101. # When the user agent "receives a cookie":
  102. 24 return self == @domain if host_only
  103. 24 domain = DomainName.new(domain)
  104. # RFC 6265 #5.1.3
  105. # Do not perform subdomain matching against IP addresses.
  106. 24 @hostname == domain.hostname if @ipaddr
  107. # RFC 6265 #4.1.1
  108. # Domain-value must be a subdomain.
  109. 24 @domain && self <= domain && domain <= @domain
  110. end
  111. 23 def <=>(other)
  112. 36 other = DomainName.new(other)
  113. 36 othername = other.hostname
  114. 36 if othername == @hostname
  115. 12 0
  116. 24 elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == "."
  117. # The other is higher
  118. 12 -1
  119. else
  120. # The other is lower
  121. 12 1
  122. end
  123. end
  124. end
  125. end

lib/httpx/errors.rb

97.56% lines covered

41 relevant lines. 40 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. # the default exception class for exceptions raised by HTTPX.
  4. 23 class Error < StandardError; end
  5. 23 class UnsupportedSchemeError < Error; end
  6. 23 class ConnectionError < Error; end
  7. # Error raised when there was a timeout. Its subclasses allow for finer-grained
  8. # control of which timeout happened.
  9. 23 class TimeoutError < Error
  10. # The timeout value which caused this error to be raised.
  11. 23 attr_reader :timeout
  12. # initializes the timeout exception with the +timeout+ causing the error, and the
  13. # error +message+ for it.
  14. 23 def initialize(timeout, message)
  15. 370 @timeout = timeout
  16. 370 super(message)
  17. end
  18. # clones this error into a HTTPX::ConnectionTimeoutError.
  19. 23 def to_connection_error
  20. 18 ex = ConnectTimeoutError.new(@timeout, message)
  21. 18 ex.set_backtrace(backtrace)
  22. 18 ex
  23. end
  24. end
  25. # Error raised when there was a timeout establishing the connection to a server.
  26. # This may be raised due to timeouts during TCP and TLS (when applicable) connection
  27. # establishment.
  28. 23 class ConnectTimeoutError < TimeoutError; end
  29. # Error raised when there was a timeout while sending a request, or receiving a response
  30. # from the server.
  31. 23 class RequestTimeoutError < TimeoutError
  32. # The HTTPX::Request request object this exception refers to.
  33. 23 attr_reader :request
  34. # initializes the exception with the +request+ and +response+ it refers to, and the
  35. # +timeout+ causing the error, and the
  36. 23 def initialize(request, response, timeout)
  37. 296 @request = request
  38. 296 @response = response
  39. 296 super(timeout, "Timed out after #{timeout} seconds")
  40. end
  41. 23 def marshal_dump
  42. [message]
  43. end
  44. end
  45. # Error raised when there was a timeout while receiving a response from the server.
  46. 23 class ReadTimeoutError < RequestTimeoutError; end
  47. # Error raised when there was a timeout while sending a request from the server.
  48. 23 class WriteTimeoutError < RequestTimeoutError; end
  49. # Error raised when there was a timeout while waiting for the HTTP/2 settings frame from the server.
  50. 23 class SettingsTimeoutError < TimeoutError; end
  51. # Error raised when there was a timeout while resolving a domain to an IP.
  52. 23 class ResolveTimeoutError < TimeoutError; end
  53. # Error raised when there was an error while resolving a domain to an IP.
  54. 23 class ResolveError < Error; end
  55. # Error raised when there was an error while resolving a domain to an IP
  56. # using a HTTPX::Resolver::Native resolver.
  57. 23 class NativeResolveError < ResolveError
  58. 23 attr_reader :connection, :host
  59. # initializes the exception with the +connection+ it refers to, the +host+ domain
  60. # which failed to resolve, and the error +message+.
  61. 23 def initialize(connection, host, message = "Can't resolve #{host}")
  62. 94 @connection = connection
  63. 94 @host = host
  64. 94 super(message)
  65. end
  66. end
  67. # The exception class for HTTP responses with 4xx or 5xx status.
  68. 23 class HTTPError < Error
  69. # The HTTPX::Response response object this exception refers to.
  70. 23 attr_reader :response
  71. # Creates the instance and assigns the HTTPX::Response +response+.
  72. 23 def initialize(response)
  73. 73 @response = response
  74. 73 super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
  75. end
  76. # The HTTP response status.
  77. #
  78. # error.status #=> 404
  79. 23 def status
  80. 12 @response.status
  81. end
  82. end
  83. # error raised when a request was sent a server which can't reproduce a response, and
  84. # has therefore returned an HTTP response using the 421 status code.
  85. 23 class MisdirectedRequestError < HTTPError; end
  86. end

lib/httpx/extensions.rb

67.86% lines covered

28 relevant lines. 19 lines covered and 9 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "uri"
  3. 23 module HTTPX
  4. 23 module ArrayExtensions
  5. 23 module FilterMap
  6. refine Array do
  7. # Ruby 2.7 backport
  8. def filter_map
  9. return to_enum(:filter_map) unless block_given?
  10. each_with_object([]) do |item, res|
  11. processed = yield(item)
  12. res << processed if processed
  13. end
  14. end
  15. 23 end unless Array.method_defined?(:filter_map)
  16. end
  17. 23 module Intersect
  18. refine Array do
  19. # Ruby 3.1 backport
  20. 4 def intersect?(arr)
  21. if size < arr.size
  22. smaller = self
  23. else
  24. smaller, arr = arr, self
  25. end
  26. (arr & smaller).size > 0
  27. end
  28. 23 end unless Array.method_defined?(:intersect?)
  29. end
  30. end
  31. 23 module URIExtensions
  32. # uri 0.11 backport, ships with ruby 3.1
  33. 23 refine URI::Generic do
  34. 23 def non_ascii_hostname
  35. 335 @non_ascii_hostname
  36. end
  37. 23 def non_ascii_hostname=(hostname)
  38. 24 @non_ascii_hostname = hostname
  39. end
  40. def authority
  41. 5176 return host if port == default_port
  42. 333 "#{host}:#{port}"
  43. 23 end unless URI::HTTP.method_defined?(:authority)
  44. def origin
  45. 4232 "#{scheme}://#{authority}"
  46. 23 end unless URI::HTTP.method_defined?(:origin)
  47. end
  48. end
  49. end

lib/httpx/headers.rb

100.0% lines covered

71 relevant lines. 71 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 class Headers
  4. 23 class << self
  5. 23 def new(headers = nil)
  6. 18449 return headers if headers.is_a?(self)
  7. 8981 super
  8. end
  9. end
  10. 23 def initialize(headers = nil)
  11. 8981 @headers = {}
  12. 8981 return unless headers
  13. 8847 headers.each do |field, value|
  14. 43601 array_value(value).each do |v|
  15. 43859 add(downcased(field), v)
  16. end
  17. end
  18. end
  19. # cloned initialization
  20. 23 def initialize_clone(orig)
  21. 6 super
  22. 6 @headers = orig.instance_variable_get(:@headers).clone
  23. end
  24. # dupped initialization
  25. 23 def initialize_dup(orig)
  26. 10124 super
  27. 10124 @headers = orig.instance_variable_get(:@headers).dup
  28. end
  29. # freezes the headers hash
  30. 23 def freeze
  31. 11264 @headers.freeze
  32. 11264 super
  33. end
  34. 23 def same_headers?(headers)
  35. 24 @headers.empty? || begin
  36. 24 headers.each do |k, v|
  37. 54 next unless key?(k)
  38. 54 return false unless v == self[k]
  39. end
  40. 12 true
  41. end
  42. end
  43. # merges headers with another header-quack.
  44. # the merge rule is, if the header already exists,
  45. # ignore what the +other+ headers has. Otherwise, set
  46. #
  47. 23 def merge(other)
  48. 3197 headers = dup
  49. 3197 other.each do |field, value|
  50. 2336 headers[downcased(field)] = value
  51. end
  52. 3197 headers
  53. end
  54. # returns the comma-separated values of the header field
  55. # identified by +field+, or nil otherwise.
  56. #
  57. 23 def [](field)
  58. 63060 a = @headers[downcased(field)] || return
  59. 18802 a.join(", ")
  60. end
  61. # sets +value+ (if not nil) as single value for the +field+ header.
  62. #
  63. 23 def []=(field, value)
  64. 27691 return unless value
  65. 23984 @headers[downcased(field)] = array_value(value)
  66. end
  67. # deletes all values associated with +field+ header.
  68. #
  69. 23 def delete(field)
  70. 118 canonical = downcased(field)
  71. 118 @headers.delete(canonical) if @headers.key?(canonical)
  72. end
  73. # adds additional +value+ to the existing, for header +field+.
  74. #
  75. 23 def add(field, value)
  76. 44183 (@headers[downcased(field)] ||= []) << String(value)
  77. end
  78. # helper to be used when adding an header field as a value to another field
  79. #
  80. # h2_headers.add_header("vary", "accept-encoding")
  81. # h2_headers["vary"] #=> "accept-encoding"
  82. # h1_headers.add_header("vary", "accept-encoding")
  83. # h1_headers["vary"] #=> "Accept-Encoding"
  84. #
  85. 23 alias_method :add_header, :add
  86. # returns the enumerable headers store in pairs of header field + the values in
  87. # the comma-separated string format
  88. #
  89. 23 def each(extra_headers = nil)
  90. 46332 return enum_for(__method__, extra_headers) { @headers.size } unless block_given?
  91. 24949 @headers.each do |field, value|
  92. 31912 yield(field, value.join(", ")) unless value.empty?
  93. end
  94. 5959 extra_headers.each do |field, value|
  95. 16943 yield(field, value) unless value.empty?
  96. 24937 end if extra_headers
  97. end
  98. 23 def ==(other)
  99. 14388 other == to_hash
  100. end
  101. # the headers store in Hash format
  102. 23 def to_hash
  103. 15263 Hash[to_a]
  104. end
  105. 23 alias_method :to_h, :to_hash
  106. # the headers store in array of pairs format
  107. 23 def to_a
  108. 15280 Array(each)
  109. end
  110. # headers as string
  111. 23 def to_s
  112. 1427 @headers.to_s
  113. end
  114. skipped # :nocov:
  115. skipped def inspect
  116. skipped to_hash.inspect
  117. skipped end
  118. skipped # :nocov:
  119. # this is internal API and doesn't abide to other public API
  120. # guarantees, like downcasing strings.
  121. # Please do not use this outside of core!
  122. #
  123. 23 def key?(downcased_key)
  124. 43836 @headers.key?(downcased_key)
  125. end
  126. # returns the values for the +field+ header in array format.
  127. # This method is more internal, and for this reason doesn't try
  128. # to "correct" the user input, i.e. it doesn't downcase the key.
  129. #
  130. 23 def get(field)
  131. 198 @headers[field] || EMPTY
  132. end
  133. 23 private
  134. 23 def array_value(value)
  135. 61446 case value
  136. when Array
  137. 67758 value.map { |val| String(val).strip }
  138. else
  139. 40916 [String(value).strip]
  140. end
  141. end
  142. 23 def downcased(field)
  143. 181554 String(field).downcase
  144. end
  145. end
  146. end

lib/httpx/io.rb

100.0% lines covered

5 relevant lines. 5 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "socket"
  3. 23 require "httpx/io/udp"
  4. 23 require "httpx/io/tcp"
  5. 23 require "httpx/io/unix"
  6. begin
  7. 23 require "httpx/io/ssl"
  8. rescue LoadError
  9. end

lib/httpx/io/ssl.rb

89.41% lines covered

85 relevant lines. 76 lines covered and 9 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "openssl"
  3. 23 module HTTPX
  4. 23 TLSError = OpenSSL::SSL::SSLError
  5. 23 class SSL < TCP
  6. # rubocop:disable Style/MutableConstant
  7. 23 TLS_OPTIONS = { alpn_protocols: %w[h2 http/1.1].freeze }
  8. # https://github.com/jruby/jruby-openssl/issues/284
  9. 23 TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby"
  10. # rubocop:enable Style/MutableConstant
  11. 23 TLS_OPTIONS.freeze
  12. 23 attr_writer :ssl_session
  13. 23 def initialize(_, _, options)
  14. 2163 super
  15. 2163 ctx_options = TLS_OPTIONS.merge(options.ssl)
  16. 2163 @sni_hostname = ctx_options.delete(:hostname) || @hostname
  17. 2163 if @keep_open && @io.is_a?(OpenSSL::SSL::SSLSocket)
  18. # externally initiated ssl socket
  19. 17 @ctx = @io.context
  20. 17 @state = :negotiated
  21. else
  22. 2146 @ctx = OpenSSL::SSL::SSLContext.new
  23. 2146 @ctx.set_params(ctx_options) unless ctx_options.empty?
  24. 2146 unless @ctx.session_cache_mode.nil? # a dummy method on JRuby
  25. 2146 @ctx.session_cache_mode =
  26. OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
  27. end
  28. 2146 yield(self) if block_given?
  29. end
  30. 2163 @verify_hostname = @ctx.verify_hostname
  31. end
  32. 23 if OpenSSL::SSL::SSLContext.method_defined?(:session_new_cb=)
  33. 23 def session_new_cb(&pr)
  34. 6077 @ctx.session_new_cb = proc { |_, sess| pr.call(sess) }
  35. end
  36. else
  37. # session_new_cb not implemented under JRuby
  38. def session_new_cb; end
  39. end
  40. 23 def protocol
  41. 2231 @io.alpn_protocol || super
  42. rescue StandardError
  43. 8 super
  44. end
  45. 23 if RUBY_ENGINE == "jruby"
  46. # in jruby, alpn_protocol may return ""
  47. # https://github.com/jruby/jruby-openssl/issues/287
  48. def protocol
  49. proto = @io.alpn_protocol
  50. return super if proto.nil? || proto.empty?
  51. proto
  52. rescue StandardError
  53. super
  54. end
  55. end
  56. 23 def can_verify_peer?
  57. 11 @ctx.verify_mode == OpenSSL::SSL::VERIFY_PEER
  58. end
  59. 23 def verify_hostname(host)
  60. 13 return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
  61. 13 return false if !@io.respond_to?(:peer_cert) || @io.peer_cert.nil?
  62. 13 OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
  63. end
  64. 23 def connected?
  65. 7929 @state == :negotiated
  66. end
  67. 23 def expired?
  68. super || ssl_session_expired?
  69. end
  70. 23 def ssl_session_expired?
  71. 2197 @ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (@ssl_session.time.to_f + @ssl_session.timeout)
  72. end
  73. 23 def connect
  74. 7968 super
  75. 7947 return if @state == :negotiated ||
  76. @state != :connected
  77. 5549 unless @io.is_a?(OpenSSL::SSL::SSLSocket)
  78. 2197 if (hostname_is_ip = (@ip == @sni_hostname))
  79. # IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
  80. 24 @sni_hostname = @ip.to_string
  81. # IP addresses in SNI is not valid per RFC 6066, section 3.
  82. 24 @ctx.verify_hostname = false
  83. end
  84. 2197 @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
  85. 2197 @io.hostname = @sni_hostname unless hostname_is_ip
  86. 2197 @io.session = @ssl_session unless ssl_session_expired?
  87. 2197 @io.sync_close = true
  88. end
  89. 5549 try_ssl_connect
  90. end
  91. 23 def try_ssl_connect
  92. 5549 ret = @io.connect_nonblock(exception: false)
  93. 5559 log(level: 3, color: :cyan) { "TLS CONNECT: #{ret}..." }
  94. 4899 case ret
  95. when :wait_readable
  96. 3370 @interests = :r
  97. 3370 return
  98. when :wait_writable
  99. @interests = :w
  100. return
  101. end
  102. 2161 @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && @verify_hostname
  103. 2161 transition(:negotiated)
  104. 2161 @interests = :w
  105. end
  106. 23 private
  107. 23 def transition(nextstate)
  108. 7286 case nextstate
  109. when :negotiated
  110. 2161 return unless @state == :connected
  111. when :closed
  112. 2144 return unless @state == :negotiated ||
  113. @state == :connected
  114. end
  115. 8584 do_transition(nextstate)
  116. end
  117. 23 def log_transition_state(nextstate)
  118. 53 return super unless nextstate == :negotiated
  119. 12 server_cert = @io.peer_cert
  120. 10 "#{super}\n\n" \
  121. 2 "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
  122. 2 "ALPN, server accepted to use #{protocol}\n" \
  123. "Server certificate:\n " \
  124. 2 "subject: #{server_cert.subject}\n " \
  125. 2 "start date: #{server_cert.not_before}\n " \
  126. 2 "expire date: #{server_cert.not_after}\n " \
  127. 2 "issuer: #{server_cert.issuer}\n " \
  128. "SSL certificate verify ok."
  129. end
  130. end
  131. end

lib/httpx/io/tcp.rb

91.07% lines covered

112 relevant lines. 102 lines covered and 10 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "resolv"
  3. 23 require "ipaddr"
  4. 23 module HTTPX
  5. 23 class TCP
  6. 23 include Loggable
  7. 23 using URIExtensions
  8. 23 attr_reader :ip, :port, :addresses, :state, :interests
  9. 23 alias_method :host, :ip
  10. 23 def initialize(origin, addresses, options)
  11. 4887 @state = :idle
  12. 4887 @addresses = []
  13. 4887 @hostname = origin.host
  14. 4887 @options = Options.new(options)
  15. 4887 @fallback_protocol = @options.fallback_protocol
  16. 4887 @port = origin.port
  17. 4887 @interests = :w
  18. 4887 if @options.io
  19. 40 @io = case @options.io
  20. when Hash
  21. 12 @options.io[origin.authority]
  22. else
  23. 28 @options.io
  24. end
  25. 40 raise Error, "Given IO objects do not match the request authority" unless @io
  26. 40 _, _, _, @ip = @io.addr
  27. 40 @addresses << @ip
  28. 40 @keep_open = true
  29. 40 @state = :connected
  30. else
  31. 4847 add_addresses(addresses)
  32. end
  33. 4887 @ip_index = @addresses.size - 1
  34. end
  35. 23 def socket
  36. 147 @io
  37. end
  38. 23 def add_addresses(addrs)
  39. 5020 return if addrs.empty?
  40. 14874 addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
  41. 5020 ip_index = @ip_index || (@addresses.size - 1)
  42. 5020 if addrs.first.ipv6?
  43. # should be the next in line
  44. 173 @addresses = [*@addresses[0, ip_index], *addrs, *@addresses[ip_index..-1]]
  45. else
  46. 4847 @addresses.unshift(*addrs)
  47. 4847 @ip_index += addrs.size if @ip_index
  48. end
  49. end
  50. 23 def to_io
  51. 17394 @io.to_io
  52. end
  53. 23 def protocol
  54. 3008 @fallback_protocol
  55. end
  56. 23 def connect
  57. 17396 return unless closed?
  58. 13868 if !@io || @io.closed?
  59. 5271 transition(:idle)
  60. 5271 @io = build_socket
  61. end
  62. 13868 try_connect
  63. rescue Errno::ECONNREFUSED,
  64. Errno::EADDRNOTAVAIL,
  65. Errno::EHOSTUNREACH,
  66. SocketError,
  67. IOError => e
  68. 312 raise e if @ip_index <= 0
  69. 273 log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
  70. 255 @ip_index -= 1
  71. 263 @io = build_socket
  72. 263 retry
  73. rescue Errno::ETIMEDOUT => e
  74. raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index <= 0
  75. log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
  76. @ip_index -= 1
  77. @io = build_socket
  78. retry
  79. end
  80. 23 def try_connect
  81. 13868 ret = @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s), exception: false)
  82. 10816 log(level: 3, color: :cyan) { "TCP CONNECT: #{ret}..." }
  83. 9158 case ret
  84. when :wait_readable
  85. @interests = :r
  86. return
  87. when :wait_writable
  88. 5519 @interests = :w
  89. 5519 return
  90. end
  91. 5217 transition(:connected)
  92. 5217 @interests = :w
  93. rescue Errno::EALREADY
  94. 2815 @interests = :w
  95. end
  96. 23 private :try_connect
  97. 23 def read(size, buffer)
  98. 34383 ret = @io.read_nonblock(size, buffer, exception: false)
  99. 34383 if ret == :wait_readable
  100. 6416 buffer.clear
  101. 5864 return 0
  102. end
  103. 27967 return if ret.nil?
  104. 28020 log { "READ: #{buffer.bytesize} bytes..." }
  105. 27957 buffer.bytesize
  106. end
  107. 23 def write(buffer)
  108. 15398 siz = @io.write_nonblock(buffer, exception: false)
  109. 15380 return 0 if siz == :wait_writable
  110. 15358 return if siz.nil?
  111. 15424 log { "WRITE: #{siz} bytes..." }
  112. 15358 buffer.shift!(siz)
  113. 15358 siz
  114. end
  115. 23 def close
  116. 5787 return if @keep_open || closed?
  117. begin
  118. 5148 @io.close
  119. ensure
  120. 5148 transition(:closed)
  121. end
  122. end
  123. 23 def connected?
  124. 9144 @state == :connected
  125. end
  126. 23 def closed?
  127. 23149 @state == :idle || @state == :closed
  128. end
  129. 23 def expired?
  130. # do not mess with external sockets
  131. return false if @options.io
  132. return true unless @addresses
  133. resolver_addresses = Resolver.nolookup_resolve(@hostname)
  134. (Array(resolver_addresses) & @addresses).empty?
  135. end
  136. skipped # :nocov:
  137. skipped def inspect
  138. skipped "#<#{self.class}: #{@ip}:#{@port} (state: #{@state})>"
  139. skipped end
  140. skipped # :nocov:
  141. 23 private
  142. 23 def build_socket
  143. 5534 @ip = @addresses[@ip_index]
  144. 5534 Socket.new(@ip.family, :STREAM, 0)
  145. end
  146. 23 def transition(nextstate)
  147. 7852 case nextstate
  148. # when :idle
  149. when :connected
  150. 3100 return unless @state == :idle
  151. when :closed
  152. 3004 return unless @state == :connected
  153. end
  154. 9231 do_transition(nextstate)
  155. end
  156. 23 def do_transition(nextstate)
  157. 17927 log(level: 1) { log_transition_state(nextstate) }
  158. 17815 @state = nextstate
  159. end
  160. 23 def log_transition_state(nextstate)
  161. 95 case nextstate
  162. when :connected
  163. 30 "Connected to #{host} (##{@io.fileno})"
  164. else
  165. 70 "#{host} #{@state} -> #{nextstate}"
  166. end
  167. end
  168. end
  169. end

lib/httpx/io/udp.rb

85.71% lines covered

35 relevant lines. 30 lines covered and 5 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "ipaddr"
  3. 23 module HTTPX
  4. 23 class UDP
  5. 23 include Loggable
  6. 23 def initialize(ip, port, options)
  7. 499 @host = ip
  8. 499 @port = port
  9. 499 @io = UDPSocket.new(IPAddr.new(ip).family)
  10. 499 @options = options
  11. end
  12. 23 def to_io
  13. 984 @io.to_io
  14. end
  15. 23 def connect; end
  16. 23 def connected?
  17. 499 true
  18. end
  19. 23 def close
  20. 729 @io.close
  21. end
  22. 23 if RUBY_ENGINE == "jruby"
  23. # In JRuby, sendmsg_nonblock is not implemented
  24. def write(buffer)
  25. siz = @io.send(buffer.to_s, 0, @host, @port)
  26. log { "WRITE: #{siz} bytes..." }
  27. buffer.shift!(siz)
  28. siz
  29. end
  30. else
  31. 23 def write(buffer)
  32. 538 siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s), exception: false)
  33. 538 return 0 if siz == :wait_writable
  34. 538 return if siz.nil?
  35. 538 log { "WRITE: #{siz} bytes..." }
  36. 538 buffer.shift!(siz)
  37. 538 siz
  38. end
  39. end
  40. 23 def read(size, buffer)
  41. 725 ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
  42. 725 return 0 if ret == :wait_readable
  43. 491 return if ret.nil?
  44. 491 log { "READ: #{buffer.bytesize} bytes..." }
  45. 491 buffer.bytesize
  46. rescue IOError
  47. end
  48. end
  49. end

lib/httpx/io/unix.rb

96.97% lines covered

33 relevant lines. 32 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 class UNIX < TCP
  4. 23 using URIExtensions
  5. 23 attr_reader :path
  6. 23 alias_method :host, :path
  7. 23 def initialize(origin, addresses, options)
  8. 24 @addresses = []
  9. 24 @hostname = origin.host
  10. 24 @state = :idle
  11. 24 @options = Options.new(options)
  12. 24 @fallback_protocol = @options.fallback_protocol
  13. 24 if @options.io
  14. 12 @io = case @options.io
  15. when Hash
  16. 6 @options.io[origin.authority]
  17. else
  18. 6 @options.io
  19. end
  20. 12 raise Error, "Given IO objects do not match the request authority" unless @io
  21. 12 @path = @io.path
  22. 12 @keep_open = true
  23. 12 @state = :connected
  24. else
  25. 12 @path = addresses.first
  26. end
  27. 24 @io ||= build_socket
  28. end
  29. 23 def connect
  30. 18 return unless closed?
  31. begin
  32. 18 if @io.closed?
  33. 6 transition(:idle)
  34. 6 @io = build_socket
  35. end
  36. 18 @io.connect_nonblock(Socket.sockaddr_un(@path))
  37. rescue Errno::EISCONN
  38. end
  39. 12 transition(:connected)
  40. rescue Errno::EINPROGRESS,
  41. Errno::EALREADY,
  42. ::IO::WaitReadable
  43. end
  44. 23 def expired?
  45. false
  46. end
  47. skipped # :nocov:
  48. skipped def inspect
  49. skipped "#<#{self.class}(path: #{@path}): (state: #{@state})>"
  50. skipped end
  51. skipped # :nocov:
  52. 23 private
  53. 23 def build_socket
  54. 18 Socket.new(Socket::PF_UNIX, :STREAM, 0)
  55. end
  56. end
  57. end

lib/httpx/loggable.rb

100.0% lines covered

14 relevant lines. 14 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 module Loggable
  4. 23 COLORS = {
  5. black: 30,
  6. red: 31,
  7. green: 32,
  8. yellow: 33,
  9. blue: 34,
  10. magenta: 35,
  11. cyan: 36,
  12. white: 37,
  13. }.freeze
  14. 23 def log(level: @options.debug_level, color: nil, &msg)
  15. 272256 return unless @options.debug
  16. 1255 return unless @options.debug_level >= level
  17. 1255 debug_stream = @options.debug
  18. 1255 message = (+"" << msg.call << "\n")
  19. 1255 message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
  20. 1255 debug_stream << message
  21. end
  22. 23 def log_exception(ex, level: @options.debug_level, color: nil)
  23. 818 return unless @options.debug
  24. 10 return unless @options.debug_level >= level
  25. 20 log(level: level, color: color) { ex.full_message }
  26. end
  27. end
  28. end

lib/httpx/options.rb

98.7% lines covered

154 relevant lines. 152 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "socket"
  3. 23 module HTTPX
  4. # Contains a set of options which are passed and shared across from session to its requests or
  5. # responses.
  6. 23 class Options
  7. 23 BUFFER_SIZE = 1 << 14
  8. 23 WINDOW_SIZE = 1 << 14 # 16K
  9. 23 MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
  10. 23 KEEP_ALIVE_TIMEOUT = 20
  11. 23 SETTINGS_TIMEOUT = 10
  12. 23 CLOSE_HANDSHAKE_TIMEOUT = 10
  13. 23 CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
  14. 23 REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
  15. # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
  16. 1 ip_address_families = begin
  17. 23 list = Socket.ip_address_list
  18. 116 if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? && !a.ipv6_unique_local? }
  19. [Socket::AF_INET6, Socket::AF_INET]
  20. else
  21. 23 [Socket::AF_INET]
  22. end
  23. rescue NotImplementedError
  24. [Socket::AF_INET]
  25. end
  26. 1 DEFAULT_OPTIONS = {
  27. 22 :max_requests => Float::INFINITY,
  28. 23 :debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
  29. 23 :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
  30. :ssl => {},
  31. :http2_settings => { settings_enable_push: 0 },
  32. :fallback_protocol => "http/1.1",
  33. :supported_compression_formats => %w[gzip deflate],
  34. :decompress_response_body => true,
  35. :compress_request_body => true,
  36. :timeout => {
  37. connect_timeout: CONNECT_TIMEOUT,
  38. settings_timeout: SETTINGS_TIMEOUT,
  39. close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
  40. operation_timeout: OPERATION_TIMEOUT,
  41. keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
  42. read_timeout: READ_TIMEOUT,
  43. write_timeout: WRITE_TIMEOUT,
  44. request_timeout: REQUEST_TIMEOUT,
  45. },
  46. :headers_class => Class.new(Headers),
  47. :headers => {},
  48. :window_size => WINDOW_SIZE,
  49. :buffer_size => BUFFER_SIZE,
  50. :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
  51. :request_class => Class.new(Request),
  52. :response_class => Class.new(Response),
  53. :request_body_class => Class.new(Request::Body),
  54. :response_body_class => Class.new(Response::Body),
  55. :connection_class => Class.new(Connection),
  56. :options_class => Class.new(self),
  57. :transport => nil,
  58. :addresses => nil,
  59. :persistent => false,
  60. 23 :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
  61. :resolver_options => { cache: true },
  62. :ip_families => ip_address_families,
  63. }.freeze
  64. 23 class << self
  65. 23 def new(options = {})
  66. # let enhanced options go through
  67. 26100 return options if self == Options && options.class < self
  68. 17478 return options if options.is_a?(self)
  69. 3147 super
  70. end
  71. 23 def method_added(meth)
  72. 13348 super
  73. 13348 return unless meth =~ /^option_(.+)$/
  74. 6192 optname = Regexp.last_match(1).to_sym
  75. 6192 attr_reader(optname)
  76. end
  77. end
  78. # creates a new options instance from a given hash, which optionally define the following:
  79. #
  80. # :debug :: an object which log messages are written to (must respond to <tt><<</tt>)
  81. # :debug_level :: the log level of messages (can be 1, 2, or 3).
  82. # :ssl :: a hash of options which can be set as params of OpenSSL::SSL::SSLContext (see HTTPX::IO::SSL)
  83. # :http2_settings :: a hash of options to be passed to a HTTP2Next::Connection (ex: <tt>{ max_concurrent_streams: 2 }</tt>)
  84. # :fallback_protocol :: version of HTTP protocol to use by default in the absence of protocol negotiation
  85. # like ALPN (defaults to <tt>"http/1.1"</tt>)
  86. # :supported_compression_formats :: list of compressions supported by the transcoder layer (defaults to <tt>%w[gzip deflate]</tt>).
  87. # :decompress_response_body :: whether to auto-decompress response body (defaults to <tt>true</tt>).
  88. # :compress_request_body :: whether to auto-decompress response body (defaults to <tt>true</tt>)
  89. # :timeout :: hash of timeout configurations (supports <tt>:connect_timeout</tt>, <tt>:settings_timeout</tt>,
  90. # <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>, <tt>:read_timeout</tt>, <tt>:write_timeout</tt>
  91. # and <tt>:request_timeout</tt>
  92. # :headers :: hash of HTTP headers (ex: <tt>{ "x-custom-foo" => "bar" }</tt>)
  93. # :window_size :: number of bytes to read from a socket
  94. # :buffer_size :: internal read and write buffer size in bytes
  95. # :body_threshold_size :: maximum size in bytes of response payload that is buffered in memory.
  96. # :request_class :: class used to instantiate a request
  97. # :response_class :: class used to instantiate a response
  98. # :headers_class :: class used to instantiate headers
  99. # :request_body_class :: class used to instantiate a request body
  100. # :response_body_class :: class used to instantiate a response body
  101. # :connection_class :: class used to instantiate connections
  102. # :options_class :: class used to instantiate options
  103. # :transport :: type of transport to use (set to "unix" for UNIX sockets)
  104. # :addresses :: bucket of peer addresses (can be a list of IP addresses, a hash of domain to list of adddresses;
  105. # paths should be used for UNIX sockets instead)
  106. # :io :: open socket, or domain/ip-to-socket hash, which requests should be sent to
  107. # :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
  108. # :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
  109. # using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class)
  110. # :resolver_options :: hash of options passed to the resolver
  111. # :ip_families :: which socket families are supported (system-dependent)
  112. # :origin :: HTTP origin to set on requests with relative path (ex: "https://api.serv.com")
  113. # :base_path :: path to prefix given relative paths with (ex: "/v2")
  114. # :max_concurrent_requests :: max number of requests which can be set concurrently
  115. # :max_requests :: max number of requests which can be made on socket before it reconnects.
  116. # :params :: hash or array of key-values which will be encoded and set in the query string of request uris.
  117. # :form :: hash of array of key-values which will be form-or-multipart-encoded in requests body payload.
  118. # :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
  119. # :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
  120. #
  121. # This list of options are enhanced with each loaded plugin, see the plugin docs for details.
  122. 23 def initialize(options = {})
  123. 3147 do_initialize(options)
  124. 3135 freeze
  125. end
  126. 23 def freeze
  127. 7985 super
  128. 7985 @origin.freeze
  129. 7985 @base_path.freeze
  130. 7985 @timeout.freeze
  131. 7985 @headers.freeze
  132. 7985 @addresses.freeze
  133. 7985 @supported_compression_formats.freeze
  134. end
  135. 23 def option_origin(value)
  136. 480 URI(value)
  137. end
  138. 23 def option_base_path(value)
  139. 24 String(value)
  140. end
  141. 23 def option_headers(value)
  142. 6159 headers_class.new(value)
  143. end
  144. 23 def option_timeout(value)
  145. 5775 Hash[value]
  146. end
  147. 23 def option_supported_compression_formats(value)
  148. 5190 Array(value).map(&:to_s)
  149. end
  150. 23 def option_max_concurrent_requests(value)
  151. 672 raise TypeError, ":max_concurrent_requests must be positive" unless value.positive?
  152. 672 value
  153. end
  154. 23 def option_max_requests(value)
  155. 5178 raise TypeError, ":max_requests must be positive" unless value.positive?
  156. 5178 value
  157. end
  158. 23 def option_window_size(value)
  159. 5182 value = Integer(value)
  160. 5182 raise TypeError, ":window_size must be positive" unless value.positive?
  161. 5182 value
  162. end
  163. 23 def option_buffer_size(value)
  164. 5182 value = Integer(value)
  165. 5182 raise TypeError, ":buffer_size must be positive" unless value.positive?
  166. 5182 value
  167. end
  168. 23 def option_body_threshold_size(value)
  169. 5170 bytes = Integer(value)
  170. 5170 raise TypeError, ":body_threshold_size must be positive" unless bytes.positive?
  171. 5170 bytes
  172. end
  173. 23 def option_transport(value)
  174. 42 transport = value.to_s
  175. 42 raise TypeError, "#{transport} is an unsupported transport type" unless %w[unix].include?(transport)
  176. 42 transport
  177. end
  178. 23 def option_addresses(value)
  179. 35 Array(value)
  180. end
  181. 23 def option_ip_families(value)
  182. 5170 Array(value)
  183. end
  184. 23 %i[
  185. params form json xml body ssl http2_settings
  186. request_class response_class headers_class request_body_class
  187. response_body_class connection_class options_class
  188. io fallback_protocol debug debug_level resolver_class resolver_options
  189. compress_request_body decompress_response_body
  190. persistent
  191. ].each do |method_name|
  192. 529 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  193. 23 def option_#{method_name}(v); v; end # def option_smth(v); v; end
  194. OUT
  195. end
  196. 23 REQUEST_BODY_IVARS = %i[@headers @params @form @xml @json @body].freeze
  197. 23 def ==(other)
  198. 3576 super || options_equals?(other)
  199. end
  200. 23 def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
  201. # headers and other request options do not play a role, as they are
  202. # relevant only for the request.
  203. 2302 ivars = instance_variables - ignore_ivars
  204. 2302 other_ivars = other.instance_variables - ignore_ivars
  205. 2302 return false if ivars.size != other_ivars.size
  206. 1188 return false if ivars.sort != other_ivars.sort
  207. 1144 ivars.all? do |ivar|
  208. 13966 instance_variable_get(ivar) == other.instance_variable_get(ivar)
  209. end
  210. end
  211. 23 OTHER_LOOKUP = ->(obj, k, ivar_map) {
  212. 201271 case obj
  213. when Hash
  214. 23171 obj[ivar_map[k]]
  215. else
  216. 209318 obj.instance_variable_get(k)
  217. end
  218. }
  219. 23 def merge(other)
  220. 27235 ivar_map = nil
  221. 27235 other_ivars = case other
  222. when Hash
  223. 33819 ivar_map = other.keys.to_h { |k| [:"@#{k}", k] }
  224. 19940 ivar_map.keys
  225. else
  226. 7295 other.instance_variables
  227. end
  228. 27235 return self if other_ivars.empty?
  229. 184721 return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == OTHER_LOOKUP[other, ivar, ivar_map] }
  230. 10999 opts = dup
  231. 10999 other_ivars.each do |ivar|
  232. 64165 v = OTHER_LOOKUP[other, ivar, ivar_map]
  233. 64165 unless v
  234. 2488 opts.instance_variable_set(ivar, v)
  235. 2488 next
  236. end
  237. 61677 v = opts.__send__(:"option_#{ivar[1..-1]}", v)
  238. 61677 orig_v = instance_variable_get(ivar)
  239. 61677 v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
  240. 61677 opts.instance_variable_set(ivar, v)
  241. end
  242. 10999 opts
  243. end
  244. 23 def to_hash
  245. 2143 instance_variables.each_with_object({}) do |ivar, hs|
  246. 44534 hs[ivar[1..-1].to_sym] = instance_variable_get(ivar)
  247. end
  248. end
  249. 23 def extend_with_plugin_classes(pl)
  250. 4815 if defined?(pl::RequestMethods) || defined?(pl::RequestClassMethods)
  251. 1403 @request_class = @request_class.dup
  252. 1403 @request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
  253. 1403 @request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
  254. end
  255. 4815 if defined?(pl::ResponseMethods) || defined?(pl::ResponseClassMethods)
  256. 1489 @response_class = @response_class.dup
  257. 1489 @response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
  258. 1489 @response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
  259. end
  260. 4815 if defined?(pl::HeadersMethods) || defined?(pl::HeadersClassMethods)
  261. 114 @headers_class = @headers_class.dup
  262. 114 @headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
  263. 114 @headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
  264. end
  265. 4815 if defined?(pl::RequestBodyMethods) || defined?(pl::RequestBodyClassMethods)
  266. 141 @request_body_class = @request_body_class.dup
  267. 141 @request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
  268. 141 @request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
  269. end
  270. 4815 if defined?(pl::ResponseBodyMethods) || defined?(pl::ResponseBodyClassMethods)
  271. 402 @response_body_class = @response_body_class.dup
  272. 402 @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
  273. 402 @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
  274. end
  275. 4815 if defined?(pl::ConnectionMethods)
  276. 1994 @connection_class = @connection_class.dup
  277. 1994 @connection_class.__send__(:include, pl::ConnectionMethods)
  278. end
  279. 4815 return unless defined?(pl::OptionsMethods)
  280. 1815 @options_class = @options_class.dup
  281. 1815 @options_class.__send__(:include, pl::OptionsMethods)
  282. end
  283. 23 private
  284. 23 def do_initialize(options = {})
  285. 3147 defaults = DEFAULT_OPTIONS.merge(options)
  286. 3147 defaults.each do |k, v|
  287. 86000 next if v.nil?
  288. 76559 option_method_name = :"option_#{k}"
  289. 76559 raise Error, "unknown option: #{k}" unless respond_to?(option_method_name)
  290. 76553 value = __send__(option_method_name, v)
  291. 76547 instance_variable_set(:"@#{k}", value)
  292. end
  293. end
  294. end
  295. end

lib/httpx/parser/http1.rb

100.0% lines covered

109 relevant lines. 109 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 module Parser
  4. 23 class Error < Error; end
  5. 23 class HTTP1
  6. 23 VERSIONS = %w[1.0 1.1].freeze
  7. 23 attr_reader :status_code, :http_version, :headers
  8. 23 def initialize(observer)
  9. 3134 @observer = observer
  10. 3134 @state = :idle
  11. 3134 @buffer = "".b
  12. 3134 @headers = {}
  13. end
  14. 23 def <<(chunk)
  15. 5053 @buffer << chunk
  16. 5053 parse
  17. end
  18. 23 def reset!
  19. 9065 @state = :idle
  20. 9065 @headers.clear
  21. 9065 @content_length = nil
  22. 9065 @_has_trailers = nil
  23. end
  24. 23 def upgrade?
  25. 3246 @upgrade
  26. end
  27. 23 def upgrade_data
  28. 24 @buffer
  29. end
  30. 23 private
  31. 23 def parse
  32. 5053 loop do
  33. 10902 state = @state
  34. 9455 case @state
  35. when :idle
  36. 3506 parse_headline
  37. when :headers, :trailers
  38. 3573 parse_headers
  39. when :data
  40. 3823 parse_data
  41. end
  42. 8115 return if @buffer.empty? || state == @state
  43. end
  44. end
  45. 23 def parse_headline
  46. 3506 idx = @buffer.index("\n")
  47. 3506 return unless idx
  48. 3506 (m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
  49. raise(Error, "wrong head line format")
  50. 3500 version, code, _ = m.captures
  51. 3500 raise(Error, "unsupported HTTP version (HTTP/#{version})") unless version && VERSIONS.include?(version)
  52. 3494 @http_version = version.split(".").map(&:to_i)
  53. 3494 @status_code = code.to_i
  54. 3494 raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
  55. 3488 @buffer = @buffer.byteslice((idx + 1)..-1)
  56. 3488 nextstate(:headers)
  57. end
  58. 23 def parse_headers
  59. 3573 headers = @headers
  60. 3573 buffer = @buffer
  61. 30294 while (idx = buffer.index("\n"))
  62. 27117 line = buffer.byteslice(0..idx)
  63. 27117 raise Error, "wrong header format" if line.start_with?("\s", "\t")
  64. 27111 line.lstrip!
  65. 27111 buffer = @buffer = buffer.byteslice((idx + 1)..-1)
  66. 27111 if line.empty?
  67. 3033 case @state
  68. when :headers
  69. 3476 prepare_data(headers)
  70. 3476 @observer.on_headers(headers)
  71. 3007 return unless @state == :headers
  72. # state might have been reset
  73. # in the :headers callback
  74. 2951 nextstate(:data)
  75. 2951 headers.clear
  76. when :trailers
  77. 12 @observer.on_trailers(headers)
  78. 12 headers.clear
  79. 12 nextstate(:complete)
  80. end
  81. 2963 return
  82. end
  83. 23623 separator_index = line.index(":")
  84. 23623 raise Error, "wrong header format" unless separator_index
  85. 23617 key = line.byteslice(0..(separator_index - 1))
  86. 23617 key.rstrip! # was lstripped previously!
  87. 23617 value = line.byteslice((separator_index + 1)..-1)
  88. 23617 value.strip!
  89. 23617 raise Error, "wrong header format" if value.nil?
  90. 23617 (headers[key.downcase] ||= []) << value
  91. end
  92. end
  93. 23 def parse_data
  94. 3823 if @buffer.respond_to?(:each)
  95. 144 @buffer.each do |chunk|
  96. 158 @observer.on_data(chunk)
  97. end
  98. 3679 elsif @content_length
  99. 3655 data = @buffer.byteslice(0, @content_length)
  100. 3655 @buffer = @buffer.byteslice(@content_length..-1) || "".b
  101. 3162 @content_length -= data.bytesize
  102. 3655 @observer.on_data(data)
  103. 3644 data.clear
  104. else
  105. 24 @observer.on_data(@buffer)
  106. 24 @buffer.clear
  107. end
  108. 3806 return unless no_more_data?
  109. 2850 @buffer = @buffer.to_s
  110. 2850 if @_has_trailers
  111. 12 nextstate(:trailers)
  112. else
  113. 2838 nextstate(:complete)
  114. end
  115. end
  116. 23 def prepare_data(headers)
  117. 3476 @upgrade = headers.key?("upgrade")
  118. 3476 @_has_trailers = headers.key?("trailer")
  119. 3476 if (tr_encodings = headers["transfer-encoding"])
  120. 80 tr_encodings.reverse_each do |tr_encoding|
  121. 80 tr_encoding.split(/ *, */).each do |encoding|
  122. 67 case encoding
  123. when "chunked"
  124. 80 @buffer = Transcoder::Chunker::Decoder.new(@buffer, @_has_trailers)
  125. end
  126. end
  127. end
  128. else
  129. 3396 @content_length = headers["content-length"][0].to_i if headers.key?("content-length")
  130. end
  131. end
  132. 23 def no_more_data?
  133. 3806 if @content_length
  134. 3644 @content_length <= 0
  135. 162 elsif @buffer.respond_to?(:finished?)
  136. 138 @buffer.finished?
  137. else
  138. 24 false
  139. end
  140. end
  141. 23 def nextstate(state)
  142. 9301 @state = state
  143. 8100 case state
  144. when :headers
  145. 3488 @observer.on_start
  146. when :complete
  147. 2850 @observer.on_complete
  148. 579 reset!
  149. 579 nextstate(:idle) unless @buffer.empty?
  150. end
  151. end
  152. end
  153. end
  154. end

lib/httpx/plugins/auth.rb

100.0% lines covered

9 relevant lines. 9 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin adds a shim +authorization+ method to the session, which will fill
  6. # the HTTP Authorization header, and another, +bearer_auth+, which fill the "Bearer " prefix
  7. # in its value.
  8. #
  9. # https://gitlab.com/os85/httpx/wikis/Auth#auth
  10. #
  11. 6 module Auth
  12. 6 module InstanceMethods
  13. 6 def authorization(token)
  14. 108 with(headers: { "authorization" => token })
  15. end
  16. 6 def bearer_auth(token)
  17. 12 authorization("Bearer #{token}")
  18. end
  19. end
  20. end
  21. 6 register_plugin :auth, Auth
  22. end
  23. end

lib/httpx/plugins/auth/basic.rb

100.0% lines covered

10 relevant lines. 10 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 7 require "httpx/base64"
  3. 7 module HTTPX
  4. 7 module Plugins
  5. 7 module Authentication
  6. 7 class Basic
  7. 7 def initialize(user, password, **)
  8. 205 @user = user
  9. 205 @password = password
  10. end
  11. 7 def authenticate(*)
  12. 181 "Basic #{Base64.strict_encode64("#{@user}:#{@password}")}"
  13. end
  14. end
  15. end
  16. end
  17. end

lib/httpx/plugins/auth/digest.rb

100.0% lines covered

59 relevant lines. 59 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "time"
  3. 6 require "securerandom"
  4. 6 require "digest"
  5. 6 module HTTPX
  6. 6 module Plugins
  7. 6 module Authentication
  8. 6 class Digest
  9. 6 def initialize(user, password, hashed: false, **)
  10. 144 @user = user
  11. 144 @password = password
  12. 144 @nonce = 0
  13. 144 @hashed = hashed
  14. end
  15. 6 def can_authenticate?(authenticate)
  16. 120 authenticate && /Digest .*/.match?(authenticate)
  17. end
  18. 6 def authenticate(request, authenticate)
  19. 120 "Digest #{generate_header(request.verb, request.path, authenticate)}"
  20. end
  21. 6 private
  22. 6 def generate_header(meth, uri, authenticate)
  23. # discard first token, it's Digest
  24. 120 auth_info = authenticate[/^(\w+) (.*)/, 2]
  25. 120 params = auth_info.split(/ *, */)
  26. 1248 .to_h { |val| val.split("=") }.transform_values { |v| v.delete("\"") }
  27. 120 nonce = params["nonce"]
  28. 120 nc = next_nonce
  29. # verify qop
  30. 120 qop = params["qop"]
  31. 120 if params["algorithm"] =~ /(.*?)(-sess)?$/
  32. 108 alg = Regexp.last_match(1)
  33. 108 algorithm = ::Digest.const_get(alg)
  34. 108 raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
  35. 108 sess = Regexp.last_match(2)
  36. else
  37. 12 algorithm = ::Digest::MD5
  38. end
  39. 120 if qop || sess
  40. 120 cnonce = make_cnonce
  41. 120 nc = format("%<nonce>08x", nonce: nc)
  42. end
  43. 120 a1 = if sess
  44. 4 [
  45. 24 (@hashed ? @password : algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}")),
  46. nonce,
  47. cnonce,
  48. ].join ":"
  49. else
  50. 96 @hashed ? @password : "#{@user}:#{params["realm"]}:#{@password}"
  51. end
  52. 120 ha1 = algorithm.hexdigest(a1)
  53. 120 ha2 = algorithm.hexdigest("#{meth}:#{uri}")
  54. 120 request_digest = [ha1, nonce]
  55. 120 request_digest.push(nc, cnonce, qop) if qop
  56. 120 request_digest << ha2
  57. 120 request_digest = request_digest.join(":")
  58. 20 header = [
  59. 120 %(username="#{@user}"),
  60. 20 %(nonce="#{nonce}"),
  61. 20 %(uri="#{uri}"),
  62. 20 %(response="#{algorithm.hexdigest(request_digest)}"),
  63. ]
  64. 120 header << %(realm="#{params["realm"]}") if params.key?("realm")
  65. 120 header << %(algorithm=#{params["algorithm"]}) if params.key?("algorithm")
  66. 120 header << %(cnonce="#{cnonce}") if cnonce
  67. 120 header << %(nc=#{nc})
  68. 120 header << %(qop=#{qop}) if qop
  69. 120 header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
  70. 120 header.join ", "
  71. end
  72. 6 def make_cnonce
  73. 120 ::Digest::MD5.hexdigest [
  74. Time.now.to_i,
  75. Process.pid,
  76. SecureRandom.random_number(2**32),
  77. ].join ":"
  78. end
  79. 6 def next_nonce
  80. 100 @nonce += 1
  81. end
  82. end
  83. end
  84. end
  85. end

lib/httpx/plugins/auth/ntlm.rb

100.0% lines covered

19 relevant lines. 19 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 5 require "httpx/base64"
  3. 5 require "ntlm"
  4. 5 module HTTPX
  5. 5 module Plugins
  6. 5 module Authentication
  7. 5 class Ntlm
  8. 5 def initialize(user, password, domain: nil)
  9. 4 @user = user
  10. 4 @password = password
  11. 4 @domain = domain
  12. end
  13. 5 def can_authenticate?(authenticate)
  14. 2 authenticate && /NTLM .*/.match?(authenticate)
  15. end
  16. 5 def negotiate
  17. 4 "NTLM #{NTLM.negotiate(domain: @domain).to_base64}"
  18. end
  19. 5 def authenticate(_req, www)
  20. 2 challenge = www[/NTLM (.*)/, 1]
  21. 2 challenge = Base64.decode64(challenge)
  22. 2 ntlm_challenge = NTLM.authenticate(challenge, @user, @domain, @password).to_base64
  23. 2 "NTLM #{ntlm_challenge}"
  24. end
  25. end
  26. end
  27. end
  28. end

lib/httpx/plugins/auth/socks5.rb

100.0% lines covered

11 relevant lines. 11 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. 8 module Authentication
  5. 8 class Socks5
  6. 8 def initialize(user, password, **)
  7. 36 @user = user
  8. 36 @password = password
  9. end
  10. 8 def can_authenticate?(*)
  11. 36 @user && @password
  12. end
  13. 8 def authenticate(*)
  14. 36 [0x01, @user.bytesize, @user, @password.bytesize, @password].pack("CCA*CA*")
  15. end
  16. end
  17. end
  18. end
  19. end

lib/httpx/plugins/aws_sdk_authentication.rb

100.0% lines covered

43 relevant lines. 43 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin applies AWS Sigv4 to requests, using the AWS SDK credentials and configuration.
  6. #
  7. # It requires the "aws-sdk-core" gem.
  8. #
  9. 6 module AwsSdkAuthentication
  10. # Mock configuration, to be used only when resolving credentials
  11. 6 class Configuration
  12. 6 attr_reader :profile
  13. 6 def initialize(profile)
  14. 24 @profile = profile
  15. end
  16. 6 def respond_to_missing?(*)
  17. 12 true
  18. end
  19. 6 def method_missing(*); end
  20. end
  21. #
  22. # encapsulates access to an AWS SDK credentials store.
  23. #
  24. 6 class Credentials
  25. 6 def initialize(aws_credentials)
  26. 12 @aws_credentials = aws_credentials
  27. end
  28. 6 def username
  29. 12 @aws_credentials.access_key_id
  30. end
  31. 6 def password
  32. 12 @aws_credentials.secret_access_key
  33. end
  34. 6 def security_token
  35. 12 @aws_credentials.session_token
  36. end
  37. end
  38. 6 class << self
  39. 6 def load_dependencies(_klass)
  40. 12 require "aws-sdk-core"
  41. end
  42. 6 def configure(klass)
  43. 12 klass.plugin(:aws_sigv4)
  44. end
  45. 6 def extra_options(options)
  46. 12 options.merge(max_concurrent_requests: 1)
  47. end
  48. 6 def credentials(profile)
  49. 12 mock_configuration = Configuration.new(profile)
  50. 12 Credentials.new(Aws::CredentialProviderChain.new(mock_configuration).resolve)
  51. end
  52. 6 def region(profile)
  53. # https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb#L62
  54. 12 keys = %w[AWS_REGION AMAZON_REGION AWS_DEFAULT_REGION]
  55. 12 env_region = ENV.values_at(*keys).compact.first
  56. 12 env_region = nil if env_region == ""
  57. 12 cfg_region = Aws.shared_config.region(profile: profile)
  58. 12 env_region || cfg_region
  59. end
  60. end
  61. 6 module OptionsMethods
  62. 6 def option_aws_profile(value)
  63. 60 String(value)
  64. end
  65. end
  66. 6 module InstanceMethods
  67. #
  68. # aws_authentication
  69. # aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
  70. # aws_authentication()
  71. #
  72. 6 def aws_sdk_authentication(
  73. credentials: AwsSdkAuthentication.credentials(@options.aws_profile),
  74. region: AwsSdkAuthentication.region(@options.aws_profile),
  75. **options
  76. )
  77. 12 aws_sigv4_authentication(
  78. credentials: credentials,
  79. region: region,
  80. provider_prefix: "aws",
  81. header_provider_field: "amz",
  82. **options
  83. )
  84. end
  85. 6 alias_method :aws_auth, :aws_sdk_authentication
  86. end
  87. end
  88. 6 register_plugin :aws_sdk_authentication, AwsSdkAuthentication
  89. end
  90. end

lib/httpx/plugins/aws_sigv4.rb

100.0% lines covered

114 relevant lines. 114 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin adds AWS Sigv4 authentication.
  6. #
  7. # https://docs.aws.amazon.com/IAM/latest/UserGuide/signing-elements.html
  8. #
  9. # https://gitlab.com/os85/httpx/wikis/AWS-SigV4
  10. #
  11. 6 module AWSSigV4
  12. 6 Credentials = Struct.new(:username, :password, :security_token)
  13. 6 class Signer
  14. 6 def initialize(
  15. service:,
  16. region:,
  17. credentials: nil,
  18. username: nil,
  19. password: nil,
  20. security_token: nil,
  21. provider_prefix: "aws",
  22. header_provider_field: "amz",
  23. unsigned_headers: [],
  24. apply_checksum_header: true,
  25. algorithm: "SHA256"
  26. )
  27. 96 @credentials = credentials || Credentials.new(username, password, security_token)
  28. 96 @service = service
  29. 96 @region = region
  30. 96 @unsigned_headers = Set.new(unsigned_headers.map(&:downcase))
  31. 96 @unsigned_headers << "authorization"
  32. 96 @unsigned_headers << "x-amzn-trace-id"
  33. 96 @unsigned_headers << "expect"
  34. 96 @apply_checksum_header = apply_checksum_header
  35. 96 @provider_prefix = provider_prefix
  36. 96 @header_provider_field = header_provider_field
  37. 96 @algorithm = algorithm
  38. end
  39. 6 def sign!(request)
  40. 96 lower_provider_prefix = "#{@provider_prefix}4"
  41. 96 upper_provider_prefix = lower_provider_prefix.upcase
  42. 96 downcased_algorithm = @algorithm.downcase
  43. 96 datetime = (request.headers["x-#{@header_provider_field}-date"] ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ"))
  44. 96 date = datetime[0, 8]
  45. 96 content_hashed = request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] || hexdigest(request.body)
  46. 96 request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] ||= content_hashed if @apply_checksum_header
  47. 96 request.headers["x-#{@header_provider_field}-security-token"] ||= @credentials.security_token if @credentials.security_token
  48. 96 signature_headers = request.headers.each.reject do |k, _|
  49. 636 @unsigned_headers.include?(k)
  50. end
  51. # aws sigv4 needs to declare the host, regardless of protocol version
  52. 96 signature_headers << ["host", request.authority] unless request.headers.key?("host")
  53. 96 signature_headers.sort_by!(&:first)
  54. 96 signed_headers = signature_headers.map(&:first).join(";")
  55. 96 canonical_headers = signature_headers.map do |k, v|
  56. # eliminate whitespace between value fields, unless it's a quoted value
  57. 520 "#{k}:#{v.start_with?("\"") && v.end_with?("\"") ? v : v.gsub(/\s+/, " ").strip}\n"
  58. end.join
  59. # canonical request
  60. 96 creq = "#{request.verb}" \
  61. 16 "\n#{request.canonical_path}" \
  62. 16 "\n#{request.canonical_query}" \
  63. 16 "\n#{canonical_headers}" \
  64. 16 "\n#{signed_headers}" \
  65. 16 "\n#{content_hashed}"
  66. 96 credential_scope = "#{date}" \
  67. 16 "/#{@region}" \
  68. 16 "/#{@service}" \
  69. 16 "/#{lower_provider_prefix}_request"
  70. 96 algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
  71. # string to sign
  72. 96 sts = "#{algo_line}" \
  73. 16 "\n#{datetime}" \
  74. 16 "\n#{credential_scope}" \
  75. 16 "\n#{hexdigest(creq)}"
  76. # signature
  77. 96 k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
  78. 96 k_region = hmac(k_date, @region)
  79. 96 k_service = hmac(k_region, @service)
  80. 96 k_credentials = hmac(k_service, "#{lower_provider_prefix}_request")
  81. 96 sig = hexhmac(k_credentials, sts)
  82. 96 credential = "#{@credentials.username}/#{credential_scope}"
  83. # apply signature
  84. 80 request.headers["authorization"] =
  85. 16 "#{algo_line} " \
  86. 16 "Credential=#{credential}, " \
  87. 16 "SignedHeaders=#{signed_headers}, " \
  88. 16 "Signature=#{sig}"
  89. end
  90. 6 private
  91. 6 def hexdigest(value)
  92. 186 if value.respond_to?(:to_path)
  93. # files, pathnames
  94. 6 OpenSSL::Digest.new(@algorithm).file(value.to_path).hexdigest
  95. 180 elsif value.respond_to?(:each)
  96. 84 digest = OpenSSL::Digest.new(@algorithm)
  97. 84 mb_buffer = value.each.with_object("".b) do |chunk, buffer|
  98. 30 buffer << chunk
  99. 30 break if buffer.bytesize >= 1024 * 1024
  100. end
  101. 84 digest.update(mb_buffer)
  102. 84 value.rewind
  103. 84 digest.hexdigest
  104. else
  105. 96 OpenSSL::Digest.new(@algorithm).hexdigest(value)
  106. end
  107. end
  108. 6 def hmac(key, value)
  109. 384 OpenSSL::HMAC.digest(OpenSSL::Digest.new(@algorithm), key, value)
  110. end
  111. 6 def hexhmac(key, value)
  112. 96 OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new(@algorithm), key, value)
  113. end
  114. end
  115. 6 class << self
  116. 6 def load_dependencies(*)
  117. 96 require "set"
  118. 96 require "digest/sha2"
  119. 96 require "openssl"
  120. end
  121. 6 def configure(klass)
  122. 96 klass.plugin(:expect)
  123. end
  124. end
  125. 6 module OptionsMethods
  126. 6 def option_sigv4_signer(value)
  127. 204 value.is_a?(Signer) ? value : Signer.new(value)
  128. end
  129. end
  130. 6 module InstanceMethods
  131. 6 def aws_sigv4_authentication(**options)
  132. 96 with(sigv4_signer: Signer.new(**options))
  133. end
  134. 6 def build_request(*, _)
  135. 96 request = super
  136. 96 return request if request.headers.key?("authorization")
  137. 96 signer = request.options.sigv4_signer
  138. 96 return request unless signer
  139. 96 signer.sign!(request)
  140. 96 request
  141. end
  142. end
  143. 6 module RequestMethods
  144. 6 def canonical_path
  145. 96 path = uri.path.dup
  146. 96 path << "/" if path.empty?
  147. 120 path.gsub(%r{[^/]+}) { |part| CGI.escape(part.encode("UTF-8")).gsub("+", "%20").gsub("%7E", "~") }
  148. end
  149. 6 def canonical_query
  150. 120 params = query.split("&")
  151. # params = params.map { |p| p.match(/=/) ? p : p + '=' }
  152. # From: https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html#create-canonical-request
  153. # Sort the parameter names by character code point in ascending order.
  154. # Parameters with duplicate names should be sorted by value.
  155. #
  156. # Default sort <=> in JRuby will swap members
  157. # occasionally when <=> is 0 (considered still sorted), but this
  158. # causes our normalized query string to not match the sent querystring.
  159. # When names match, we then sort by their values. When values also
  160. # match then we sort by their original order
  161. 120 params.each.with_index.sort do |a, b|
  162. 48 a, a_offset = a
  163. 48 b, b_offset = b
  164. 48 a_name, a_value = a.split("=")
  165. 48 b_name, b_value = b.split("=")
  166. 48 if a_name == b_name
  167. 24 if a_value == b_value
  168. 12 a_offset <=> b_offset
  169. else
  170. 12 a_value <=> b_value
  171. end
  172. else
  173. 24 a_name <=> b_name
  174. end
  175. end.map(&:first).join("&")
  176. end
  177. end
  178. end
  179. 6 register_plugin :aws_sigv4, AWSSigV4
  180. end
  181. end

lib/httpx/plugins/basic_auth.rb

100.0% lines covered

12 relevant lines. 12 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin adds helper methods to implement HTTP Basic Auth (https://datatracker.ietf.org/doc/html/rfc7617)
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Auth#basic-auth
  8. #
  9. 6 module BasicAuth
  10. 6 class << self
  11. 6 def load_dependencies(_klass)
  12. 84 require_relative "auth/basic"
  13. end
  14. 6 def configure(klass)
  15. 84 klass.plugin(:auth)
  16. end
  17. end
  18. 6 module InstanceMethods
  19. 6 def basic_auth(user, password)
  20. 96 authorization(Authentication::Basic.new(user, password).authenticate)
  21. end
  22. end
  23. end
  24. 6 register_plugin :basic_auth, BasicAuth
  25. end
  26. end

lib/httpx/plugins/brotli.rb

100.0% lines covered

25 relevant lines. 25 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 5 module HTTPX
  3. 5 module Plugins
  4. 5 module Brotli
  5. 5 class Deflater < Transcoder::Deflater
  6. 5 def deflate(chunk)
  7. 20 return unless chunk
  8. 10 ::Brotli.deflate(chunk)
  9. end
  10. end
  11. 5 module RequestBodyClassMethods
  12. 5 def initialize_deflater_body(body, encoding)
  13. 20 return Brotli.encode(body) if encoding == "br"
  14. 10 super
  15. end
  16. end
  17. 5 module ResponseBodyClassMethods
  18. 5 def initialize_inflater_by_encoding(encoding, response, **kwargs)
  19. 20 return Brotli.decode(response, **kwargs) if encoding == "br"
  20. 10 super
  21. end
  22. end
  23. 5 module_function
  24. 5 def load_dependencies(*)
  25. 20 require "brotli"
  26. end
  27. 5 def self.extra_options(options)
  28. 20 options.merge(supported_compression_formats: %w[br] + options.supported_compression_formats)
  29. end
  30. 5 def encode(body)
  31. 10 Deflater.new(body)
  32. end
  33. 5 def decode(_response, **)
  34. 10 ::Brotli.method(:inflate)
  35. end
  36. end
  37. 5 register_plugin :brotli, Brotli
  38. end
  39. end

lib/httpx/plugins/callbacks.rb

100.0% lines covered

43 relevant lines. 43 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 module Plugins
  4. #
  5. # This plugin adds suppoort for callbacks around the request/response lifecycle.
  6. #
  7. # https://gitlab.com/os85/httpx/-/wikis/Events
  8. #
  9. 23 module Callbacks
  10. # connection closed user-space errors happen after errors can be surfaced to requests,
  11. # so they need to pierce through the scheduler, which is only possible by simulating an
  12. # interrupt.
  13. 23 class CallbackError < Exception; end # rubocop:disable Lint/InheritException
  14. 23 module InstanceMethods
  15. 23 include HTTPX::Callbacks
  16. 23 %i[
  17. connection_opened connection_closed
  18. request_error
  19. request_started request_body_chunk request_completed
  20. response_started response_body_chunk response_completed
  21. ].each do |meth|
  22. 207 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  23. 9 def on_#{meth}(&blk) # def on_connection_opened(&blk)
  24. 9 on(:#{meth}, &blk) # on(:connection_opened, &blk)
  25. end # end
  26. MOD
  27. end
  28. 23 private
  29. 23 def init_connection(uri, options)
  30. 157 connection = super
  31. 157 connection.on(:open) do
  32. 147 emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
  33. end
  34. 157 connection.on(:close) do
  35. 147 emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
  36. end
  37. 157 connection
  38. end
  39. 23 def set_request_callbacks(request)
  40. 159 super
  41. 159 request.on(:headers) do
  42. 135 emit_or_callback_error(:request_started, request)
  43. end
  44. 159 request.on(:body_chunk) do |chunk|
  45. 12 emit_or_callback_error(:request_body_chunk, request, chunk)
  46. end
  47. 159 request.on(:done) do
  48. 123 emit_or_callback_error(:request_completed, request)
  49. end
  50. 159 request.on(:response_started) do |res|
  51. 123 if res.is_a?(Response)
  52. 111 emit_or_callback_error(:response_started, request, res)
  53. 99 res.on(:chunk_received) do |chunk|
  54. 121 emit_or_callback_error(:response_body_chunk, request, res, chunk)
  55. end
  56. else
  57. 12 emit_or_callback_error(:request_error, request, res.error)
  58. end
  59. end
  60. 159 request.on(:response) do |res|
  61. 99 emit_or_callback_error(:response_completed, request, res)
  62. end
  63. end
  64. 23 def emit_or_callback_error(*args)
  65. 895 emit(*args)
  66. rescue StandardError => e
  67. 90 ex = CallbackError.new(e.message)
  68. 90 ex.set_backtrace(e.backtrace)
  69. 90 raise ex
  70. end
  71. 23 def receive_requests(*)
  72. 159 super
  73. rescue CallbackError => e
  74. 84 raise e.cause
  75. end
  76. end
  77. end
  78. 23 register_plugin :callbacks, Callbacks
  79. end
  80. end

lib/httpx/plugins/circuit_breaker.rb

97.06% lines covered

68 relevant lines. 66 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin implements a circuit breaker around connection errors.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Circuit-Breaker
  8. #
  9. 6 module CircuitBreaker
  10. 6 using URIExtensions
  11. 6 def self.load_dependencies(*)
  12. 42 require_relative "circuit_breaker/circuit"
  13. 42 require_relative "circuit_breaker/circuit_store"
  14. end
  15. 6 def self.extra_options(options)
  16. 42 options.merge(
  17. circuit_breaker_max_attempts: 3,
  18. circuit_breaker_reset_attempts_in: 60,
  19. circuit_breaker_break_in: 60,
  20. circuit_breaker_half_open_drip_rate: 1
  21. )
  22. end
  23. 6 module InstanceMethods
  24. 6 include HTTPX::Callbacks
  25. 6 def initialize(*)
  26. 42 super
  27. 42 @circuit_store = CircuitStore.new(@options)
  28. end
  29. 6 def initialize_dup(orig)
  30. super
  31. @circuit_store = orig.instance_variable_get(:@circuit_store).dup
  32. end
  33. 6 %i[circuit_open].each do |meth|
  34. 6 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  35. 1 def on_#{meth}(&blk) # def on_circuit_open(&blk)
  36. 1 on(:#{meth}, &blk) # on(:circuit_open, &blk)
  37. end # end
  38. MOD
  39. end
  40. 6 private
  41. 6 def send_requests(*requests)
  42. # @type var short_circuit_responses: Array[response]
  43. 168 short_circuit_responses = []
  44. # run all requests through the circuit breaker, see if the circuit is
  45. # open for any of them.
  46. 168 real_requests = requests.each_with_index.with_object([]) do |(req, idx), real_reqs|
  47. 168 short_circuit_response = @circuit_store.try_respond(req)
  48. 168 if short_circuit_response.nil?
  49. 132 real_reqs << req
  50. 132 next
  51. end
  52. 30 short_circuit_responses[idx] = short_circuit_response
  53. end
  54. # run requests for the remainder
  55. 168 unless real_requests.empty?
  56. 132 responses = super(*real_requests)
  57. 132 real_requests.each_with_index do |request, idx|
  58. 110 short_circuit_responses[requests.index(request)] = responses[idx]
  59. end
  60. end
  61. 168 short_circuit_responses
  62. end
  63. 6 def on_response(request, response)
  64. 132 emit(:circuit_open, request) if try_circuit_open(request, response)
  65. 132 super
  66. end
  67. 6 def try_circuit_open(request, response)
  68. 132 if response.is_a?(ErrorResponse)
  69. 80 case response.error
  70. when RequestTimeoutError
  71. 60 @circuit_store.try_open(request.uri, response)
  72. else
  73. 36 @circuit_store.try_open(request.origin, response)
  74. end
  75. 36 elsif (break_on = request.options.circuit_breaker_break_on) && break_on.call(response)
  76. 12 @circuit_store.try_open(request.uri, response)
  77. else
  78. 24 @circuit_store.try_close(request.uri)
  79. 4 nil
  80. end
  81. end
  82. end
  83. 6 module OptionsMethods
  84. 6 def option_circuit_breaker_max_attempts(value)
  85. 84 attempts = Integer(value)
  86. 84 raise TypeError, ":circuit_breaker_max_attempts must be positive" unless attempts.positive?
  87. 84 attempts
  88. end
  89. 6 def option_circuit_breaker_reset_attempts_in(value)
  90. 48 timeout = Float(value)
  91. 48 raise TypeError, ":circuit_breaker_reset_attempts_in must be positive" unless timeout.positive?
  92. 48 timeout
  93. end
  94. 6 def option_circuit_breaker_break_in(value)
  95. 66 timeout = Float(value)
  96. 66 raise TypeError, ":circuit_breaker_break_in must be positive" unless timeout.positive?
  97. 66 timeout
  98. end
  99. 6 def option_circuit_breaker_half_open_drip_rate(value)
  100. 66 ratio = Float(value)
  101. 66 raise TypeError, ":circuit_breaker_half_open_drip_rate must be a number between 0 and 1" unless (0..1).cover?(ratio)
  102. 66 ratio
  103. end
  104. 6 def option_circuit_breaker_break_on(value)
  105. 12 raise TypeError, ":circuit_breaker_break_on must be called with the response" unless value.respond_to?(:call)
  106. 12 value
  107. end
  108. end
  109. end
  110. 6 register_plugin :circuit_breaker, CircuitBreaker
  111. end
  112. end

lib/httpx/plugins/circuit_breaker/circuit.rb

100.0% lines covered

47 relevant lines. 47 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins::CircuitBreaker
  4. #
  5. # A circuit is assigned to a given absoolute url or origin.
  6. #
  7. # It sets +max_attempts+, the number of attempts the circuit allows, before it is opened.
  8. # It sets +reset_attempts_in+, the time a circuit stays open at most, before it resets.
  9. # It sets +break_in+, the time that must elapse before an open circuit can transit to the half-open state.
  10. # It sets +circuit_breaker_half_open_drip_rate+, the rate of requests a circuit allows to be performed when in an half-open state.
  11. #
  12. 6 class Circuit
  13. 6 def initialize(max_attempts, reset_attempts_in, break_in, circuit_breaker_half_open_drip_rate)
  14. 42 @max_attempts = max_attempts
  15. 42 @reset_attempts_in = reset_attempts_in
  16. 42 @break_in = break_in
  17. 42 @circuit_breaker_half_open_drip_rate = circuit_breaker_half_open_drip_rate
  18. 42 @attempts = 0
  19. 42 total_real_attempts = @max_attempts * @circuit_breaker_half_open_drip_rate
  20. 42 @drip_factor = (@max_attempts / total_real_attempts).round
  21. 42 @state = :closed
  22. end
  23. 6 def respond
  24. 168 try_close
  25. 140 case @state
  26. when :closed
  27. 17 nil
  28. when :half_open
  29. 35 @attempts += 1
  30. # do real requests while drip rate valid
  31. 42 if (@real_attempts % @drip_factor).zero?
  32. 25 @real_attempts += 1
  33. 25 return
  34. end
  35. 12 @response
  36. when :open
  37. 24 @response
  38. end
  39. end
  40. 6 def try_open(response)
  41. 90 case @state
  42. when :closed
  43. 90 now = Utils.now
  44. 90 if @attempts.positive?
  45. # reset if error happened long ago
  46. 36 @attempts = 0 if now - @attempted_at > @reset_attempts_in
  47. else
  48. 54 @attempted_at = now
  49. end
  50. 75 @attempts += 1
  51. 90 return unless @attempts >= @max_attempts
  52. 48 @state = :open
  53. 48 @opened_at = now
  54. 48 @response = response
  55. when :half_open
  56. # open immediately
  57. 18 @state = :open
  58. 18 @attempted_at = @opened_at = Utils.now
  59. 18 @response = response
  60. end
  61. end
  62. 6 def try_close
  63. 160 case @state
  64. when :closed
  65. 17 nil
  66. when :half_open
  67. # do not close circuit unless attempts exhausted
  68. 36 return unless @attempts >= @max_attempts
  69. # reset!
  70. 12 @attempts = 0
  71. 12 @opened_at = @attempted_at = @response = nil
  72. 12 @state = :closed
  73. when :open
  74. 54 if Utils.elapsed_time(@opened_at) > @break_in
  75. 30 @state = :half_open
  76. 30 @attempts = 0
  77. 30 @real_attempts = 0
  78. end
  79. end
  80. end
  81. end
  82. end
  83. end

lib/httpx/plugins/circuit_breaker/circuit_store.rb

100.0% lines covered

23 relevant lines. 23 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX::Plugins::CircuitBreaker
  3. 6 using HTTPX::URIExtensions
  4. 6 class CircuitStore
  5. 6 def initialize(options)
  6. 42 @circuits = Hash.new do |h, k|
  7. 35 h[k] = Circuit.new(
  8. options.circuit_breaker_max_attempts,
  9. options.circuit_breaker_reset_attempts_in,
  10. options.circuit_breaker_break_in,
  11. options.circuit_breaker_half_open_drip_rate
  12. )
  13. end
  14. 42 @circuits_mutex = Thread::Mutex.new
  15. end
  16. 6 def try_open(uri, response)
  17. 216 circuit = @circuits_mutex.synchronize { get_circuit_for_uri(uri) }
  18. 108 circuit.try_open(response)
  19. end
  20. 6 def try_close(uri)
  21. 24 circuit = @circuits_mutex.synchronize do
  22. 24 return unless @circuits.key?(uri.origin) || @circuits.key?(uri.to_s)
  23. 24 get_circuit_for_uri(uri)
  24. end
  25. 24 circuit.try_close
  26. end
  27. # if circuit is open, it'll respond with the stored response.
  28. # if not, nil.
  29. 6 def try_respond(request)
  30. 336 circuit = @circuits_mutex.synchronize { get_circuit_for_uri(request.uri) }
  31. 168 circuit.respond
  32. end
  33. 6 private
  34. 6 def get_circuit_for_uri(uri)
  35. 300 if uri.respond_to?(:origin) && @circuits.key?(uri.origin)
  36. 216 @circuits[uri.origin]
  37. else
  38. 84 @circuits[uri.to_s]
  39. end
  40. end
  41. end
  42. end

lib/httpx/plugins/cookies.rb

100.0% lines covered

50 relevant lines. 50 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "forwardable"
  3. 6 module HTTPX
  4. 6 module Plugins
  5. #
  6. # This plugin implements a persistent cookie jar for the duration of a session.
  7. #
  8. # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
  9. #
  10. # https://gitlab.com/os85/httpx/wikis/Cookies
  11. #
  12. 6 module Cookies
  13. 6 def self.load_dependencies(*)
  14. 108 require "httpx/plugins/cookies/jar"
  15. 108 require "httpx/plugins/cookies/cookie"
  16. 108 require "httpx/plugins/cookies/set_cookie_parser"
  17. end
  18. 6 module InstanceMethods
  19. 6 extend Forwardable
  20. 6 def_delegator :@options, :cookies
  21. 6 def initialize(options = {}, &blk)
  22. 216 super({ cookies: Jar.new }.merge(options), &blk)
  23. end
  24. 6 def wrap
  25. 12 return super unless block_given?
  26. 12 super do |session|
  27. 12 old_cookies_jar = @options.cookies.dup
  28. begin
  29. 12 yield session
  30. ensure
  31. 12 @options = @options.merge(cookies: old_cookies_jar)
  32. end
  33. end
  34. end
  35. 6 private
  36. 6 def on_response(_request, response)
  37. 240 if response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
  38. 48 log { "cookies: set-cookie is over #{Cookie::MAX_LENGTH}" } if set_cookie.bytesize > Cookie::MAX_LENGTH
  39. 48 @options.cookies.parse(set_cookie)
  40. end
  41. 240 super
  42. end
  43. 6 def build_request(*, _)
  44. 240 request = super
  45. 240 request.headers.set_cookie(request.options.cookies[request.uri])
  46. 240 request
  47. end
  48. end
  49. 6 module HeadersMethods
  50. 6 def set_cookie(cookies)
  51. 240 return if cookies.empty?
  52. 204 header_value = cookies.sort.join("; ")
  53. 204 add("cookie", header_value)
  54. end
  55. end
  56. 6 module OptionsMethods
  57. 6 def option_headers(*)
  58. 240 value = super
  59. 240 merge_cookie_in_jar(value.delete("cookie"), @cookies) if defined?(@cookies) && value.key?("cookie")
  60. 240 value
  61. end
  62. 6 def option_cookies(value)
  63. 360 jar = value.is_a?(Jar) ? value : Jar.new(value)
  64. 360 merge_cookie_in_jar(@headers.delete("cookie"), jar) if defined?(@headers) && @headers.key?("cookie")
  65. 360 jar
  66. end
  67. 6 private
  68. 6 def merge_cookie_in_jar(cookies, jar)
  69. 12 cookies.each do |ck|
  70. 12 ck.split(/ *; */).each do |cookie|
  71. 24 name, value = cookie.split("=", 2)
  72. 24 jar.add(Cookie.new(name, value))
  73. end
  74. end
  75. end
  76. end
  77. end
  78. 6 register_plugin :cookies, Cookies
  79. end
  80. end

lib/httpx/plugins/cookies/cookie.rb

100.0% lines covered

77 relevant lines. 77 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins::Cookies
  4. # The HTTP Cookie.
  5. #
  6. # Contains the single cookie info: name, value and attributes.
  7. 6 class Cookie
  8. 6 include Comparable
  9. # Maximum number of bytes per cookie (RFC 6265 6.1 requires 4096 at
  10. # least)
  11. 6 MAX_LENGTH = 4096
  12. 6 attr_reader :domain, :path, :name, :value, :created_at
  13. 6 def path=(path)
  14. 138 path = String(path)
  15. 138 @path = path.start_with?("/") ? path : "/"
  16. end
  17. # See #domain.
  18. 6 def domain=(domain)
  19. 30 domain = String(domain)
  20. 30 if domain.start_with?(".")
  21. 12 @for_domain = true
  22. 12 domain = domain[1..-1]
  23. end
  24. 30 return if domain.empty?
  25. 30 @domain_name = DomainName.new(domain)
  26. # RFC 6265 5.3 5.
  27. 30 @for_domain = false if @domain_name.domain.nil? # a public suffix or IP address
  28. 30 @domain = @domain_name.hostname
  29. end
  30. # Compares the cookie with another. When there are many cookies with
  31. # the same name for a URL, the value of the smallest must be used.
  32. 6 def <=>(other)
  33. # RFC 6265 5.4
  34. # Precedence: 1. longer path 2. older creation
  35. 532 (@name <=> other.name).nonzero? ||
  36. 43 (other.path.length <=> @path.length).nonzero? ||
  37. 25 (@created_at <=> other.created_at).nonzero? || 0
  38. end
  39. 6 class << self
  40. 6 def new(cookie, *args)
  41. 378 return cookie if cookie.is_a?(self)
  42. 378 super
  43. end
  44. # Tests if +target_path+ is under +base_path+ as described in RFC
  45. # 6265 5.1.4. +base_path+ must be an absolute path.
  46. # +target_path+ may be empty, in which case it is treated as the
  47. # root path.
  48. #
  49. # e.g.
  50. #
  51. # path_match?('/admin/', '/admin/index') == true
  52. # path_match?('/admin/', '/Admin/index') == false
  53. # path_match?('/admin/', '/admin/') == true
  54. # path_match?('/admin/', '/admin') == false
  55. #
  56. # path_match?('/admin', '/admin') == true
  57. # path_match?('/admin', '/Admin') == false
  58. # path_match?('/admin', '/admins') == false
  59. # path_match?('/admin', '/admin/') == true
  60. # path_match?('/admin', '/admin/index') == true
  61. 6 def path_match?(base_path, target_path)
  62. 1014 base_path.start_with?("/") || (return false)
  63. # RFC 6265 5.1.4
  64. 1014 bsize = base_path.size
  65. 1014 tsize = target_path.size
  66. 1014 return bsize == 1 if tsize.zero? # treat empty target_path as "/"
  67. 1014 return false unless target_path.start_with?(base_path)
  68. 1008 return true if bsize == tsize || base_path.end_with?("/")
  69. 12 target_path[bsize] == "/"
  70. end
  71. end
  72. 6 def initialize(arg, *attrs)
  73. 378 @created_at = Time.now
  74. 378 if attrs.empty?
  75. 18 attr_hash = Hash.try_convert(arg)
  76. else
  77. 360 @name = arg
  78. 360 @value, attr_hash = attrs
  79. 360 attr_hash = Hash.try_convert(attr_hash)
  80. end
  81. 33 attr_hash.each do |key, val|
  82. 234 key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)
  83. 195 case key
  84. when :domain, :path
  85. 168 __send__(:"#{key}=", val)
  86. else
  87. 66 instance_variable_set(:"@#{key}", val)
  88. end
  89. 378 end if attr_hash
  90. 378 @path ||= "/"
  91. 378 raise ArgumentError, "name must be specified" if @name.nil?
  92. 378 @name = @name.to_s
  93. end
  94. 6 def expires
  95. 570 @expires || (@created_at && @max_age ? @created_at + @max_age : nil)
  96. end
  97. 6 def expired?(time = Time.now)
  98. 546 return false unless expires
  99. 24 expires <= time
  100. end
  101. # Returns a string for use in the Cookie header, i.e. `name=value`
  102. # or `name="value"`.
  103. 6 def cookie_value
  104. 345 "#{@name}=#{Scanner.quote(@value.to_s)}"
  105. end
  106. 6 alias_method :to_s, :cookie_value
  107. # Tests if it is OK to send this cookie to a given `uri`. A
  108. # RuntimeError is raised if the cookie's domain is unknown.
  109. 6 def valid_for_uri?(uri)
  110. 534 uri = URI(uri)
  111. # RFC 6265 5.4
  112. 534 return false if @secure && uri.scheme != "https"
  113. 528 acceptable_from_uri?(uri) && Cookie.path_match?(@path, uri.path)
  114. end
  115. 6 private
  116. # Tests if it is OK to accept this cookie if it is sent from a given
  117. # URI/URL, `uri`.
  118. 6 def acceptable_from_uri?(uri)
  119. 552 uri = URI(uri)
  120. 552 host = DomainName.new(uri.host)
  121. # RFC 6265 5.3
  122. 552 if host.hostname == @domain
  123. 12 true
  124. 540 elsif @for_domain # !host-only-flag
  125. 24 host.cookie_domain?(@domain_name)
  126. else
  127. 516 @domain.nil?
  128. end
  129. end
  130. 6 module Scanner
  131. 6 RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
  132. 6 module_function
  133. 6 def quote(s)
  134. 414 return s unless s.match(RE_BAD_CHAR)
  135. 6 "\"#{s.gsub(/([\\"])/, "\\\\\\1")}\""
  136. end
  137. end
  138. end
  139. end
  140. end

lib/httpx/plugins/cookies/jar.rb

100.0% lines covered

47 relevant lines. 47 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins::Cookies
  4. # The Cookie Jar
  5. #
  6. # It holds a bunch of cookies.
  7. 6 class Jar
  8. 6 using URIExtensions
  9. 6 include Enumerable
  10. 6 def initialize_dup(orig)
  11. 162 super
  12. 162 @cookies = orig.instance_variable_get(:@cookies).dup
  13. end
  14. 6 def initialize(cookies = nil)
  15. 402 @cookies = []
  16. 120 cookies.each do |elem|
  17. 132 cookie = case elem
  18. when Cookie
  19. 12 elem
  20. when Array
  21. 108 Cookie.new(*elem)
  22. else
  23. 12 Cookie.new(elem)
  24. end
  25. 132 @cookies << cookie
  26. 402 end if cookies
  27. end
  28. 6 def parse(set_cookie)
  29. 108 SetCookieParser.call(set_cookie) do |name, value, attrs|
  30. 156 add(Cookie.new(name, value, attrs))
  31. end
  32. end
  33. 6 def add(cookie, path = nil)
  34. 342 c = cookie.dup
  35. 342 c.path = path if path && c.path == "/"
  36. # If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
  37. # as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
  38. 648 @cookies.delete_if { |ck| ck.name == c.name && ck.domain == c.domain && ck.path == c.path }
  39. 342 @cookies << c
  40. end
  41. 6 def [](uri)
  42. 354 each(uri).sort
  43. end
  44. 6 def each(uri = nil, &blk)
  45. 888 return enum_for(__method__, uri) unless blk
  46. 510 return @cookies.each(&blk) unless uri
  47. 354 uri = URI(uri)
  48. 354 now = Time.now
  49. 354 tpath = uri.path
  50. 354 @cookies.delete_if do |cookie|
  51. 546 if cookie.expired?(now)
  52. 12 true
  53. else
  54. 534 yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
  55. 534 false
  56. end
  57. end
  58. end
  59. 6 def merge(other)
  60. 150 cookies_dup = dup
  61. 150 other.each do |elem|
  62. 162 cookie = case elem
  63. when Cookie
  64. 150 elem
  65. when Array
  66. 6 Cookie.new(*elem)
  67. else
  68. 6 Cookie.new(elem)
  69. end
  70. 162 cookies_dup.add(cookie)
  71. end
  72. 150 cookies_dup
  73. end
  74. end
  75. end
  76. end

lib/httpx/plugins/cookies/set_cookie_parser.rb

100.0% lines covered

70 relevant lines. 70 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "strscan"
  3. 6 require "time"
  4. 6 module HTTPX
  5. 6 module Plugins::Cookies
  6. 6 module SetCookieParser
  7. # Whitespace.
  8. 6 RE_WSP = /[ \t]+/.freeze
  9. # A pattern that matches a cookie name or attribute name which may
  10. # be empty, capturing trailing whitespace.
  11. 6 RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
  12. 6 RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
  13. # A pattern that matches the comma in a (typically date) value.
  14. 6 RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
  15. 6 module_function
  16. 6 def scan_dquoted(scanner)
  17. 12 s = +""
  18. 18 until scanner.eos?
  19. 48 break if scanner.skip(/"/)
  20. 36 if scanner.skip(/\\/)
  21. 12 s << scanner.getch
  22. 24 elsif scanner.scan(/[^"\\]+/)
  23. 24 s << scanner.matched
  24. end
  25. end
  26. 12 s
  27. end
  28. 6 def scan_value(scanner, comma_as_separator = false)
  29. 330 value = +""
  30. 388 until scanner.eos?
  31. 570 if scanner.scan(/[^,;"]+/)
  32. 324 value << scanner.matched
  33. 246 elsif scanner.skip(/"/)
  34. # RFC 6265 2.2
  35. # A cookie-value may be DQUOTE'd.
  36. 12 value << scan_dquoted(scanner)
  37. 234 elsif scanner.check(/;/)
  38. 174 break
  39. 60 elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
  40. 48 break
  41. else
  42. 12 value << scanner.getch
  43. end
  44. end
  45. 330 value.rstrip!
  46. 330 value
  47. end
  48. 6 def scan_name_value(scanner, comma_as_separator = false)
  49. 330 name = scanner.scan(RE_NAME)
  50. 330 name.rstrip! if name
  51. 330 if scanner.skip(/=/)
  52. 324 value = scan_value(scanner, comma_as_separator)
  53. else
  54. 6 scan_value(scanner, comma_as_separator)
  55. 6 value = nil
  56. end
  57. 330 [name, value]
  58. end
  59. 6 def call(set_cookie)
  60. 108 scanner = StringScanner.new(set_cookie)
  61. # RFC 6265 4.1.1 & 5.2
  62. 134 until scanner.eos?
  63. 156 start = scanner.pos
  64. 156 len = nil
  65. 156 scanner.skip(RE_WSP)
  66. 156 name, value = scan_name_value(scanner, true)
  67. 156 value = nil if name.empty?
  68. 156 attrs = {}
  69. 185 until scanner.eos?
  70. 222 if scanner.skip(/,/)
  71. # The comma is used as separator for concatenating multiple
  72. # values of a header.
  73. 48 len = (scanner.pos - 1) - start
  74. 48 break
  75. 174 elsif scanner.skip(/;/)
  76. 174 scanner.skip(RE_WSP)
  77. 174 aname, avalue = scan_name_value(scanner, true)
  78. 174 next if aname.empty? || value.nil?
  79. 174 aname.downcase!
  80. 145 case aname
  81. when "expires"
  82. # RFC 6265 5.2.1
  83. 12 (avalue &&= Time.parse(avalue)) || next
  84. when "max-age"
  85. # RFC 6265 5.2.2
  86. 6 next unless /\A-?\d+\z/.match?(avalue)
  87. 6 avalue = Integer(avalue)
  88. when "domain"
  89. # RFC 6265 5.2.3
  90. # An empty value SHOULD be ignored.
  91. 18 next if avalue.nil? || avalue.empty?
  92. when "path"
  93. # RFC 6265 5.2.4
  94. # A relative path must be ignored rather than normalizing it
  95. # to "/".
  96. 132 next unless avalue.start_with?("/")
  97. when "secure", "httponly"
  98. # RFC 6265 5.2.5, 5.2.6
  99. 6 avalue = true
  100. end
  101. 145 attrs[aname] = avalue
  102. end
  103. end
  104. 156 len ||= scanner.pos - start
  105. 156 next if len > Cookie::MAX_LENGTH
  106. 156 yield(name, value, attrs) if name && !name.empty? && value
  107. end
  108. end
  109. end
  110. end
  111. end

lib/httpx/plugins/digest_auth.rb

100.0% lines covered

29 relevant lines. 29 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin adds helper methods to implement HTTP Digest Auth (https://datatracker.ietf.org/doc/html/rfc7616)
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
  8. #
  9. 6 module DigestAuth
  10. 6 DigestError = Class.new(Error)
  11. 6 class << self
  12. 6 def extra_options(options)
  13. 120 options.merge(max_concurrent_requests: 1)
  14. end
  15. 6 def load_dependencies(*)
  16. 120 require_relative "auth/digest"
  17. end
  18. end
  19. 6 module OptionsMethods
  20. 6 def option_digest(value)
  21. 240 raise TypeError, ":digest must be a #{Authentication::Digest}" unless value.is_a?(Authentication::Digest)
  22. 240 value
  23. end
  24. end
  25. 6 module InstanceMethods
  26. 6 def digest_auth(user, password, hashed: false)
  27. 120 with(digest: Authentication::Digest.new(user, password, hashed: hashed))
  28. end
  29. 6 private
  30. 6 def send_requests(*requests)
  31. 144 requests.flat_map do |request|
  32. 144 digest = request.options.digest
  33. 144 next super(request) unless digest
  34. 240 probe_response = wrap { super(request).first }
  35. 120 return probe_response unless probe_response.is_a?(Response)
  36. 120 if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
  37. 108 request.transition(:idle)
  38. 90 request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
  39. 108 super(request)
  40. else
  41. 12 probe_response
  42. end
  43. end
  44. end
  45. end
  46. end
  47. 6 register_plugin :digest_auth, DigestAuth
  48. end
  49. end

lib/httpx/plugins/expect.rb

100.0% lines covered

56 relevant lines. 56 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin makes all HTTP/1.1 requests with a body send the "Expect: 100-continue".
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Expect#expect
  8. #
  9. 6 module Expect
  10. 6 EXPECT_TIMEOUT = 2
  11. 6 class << self
  12. 6 def no_expect_store
  13. 124 @no_expect_store ||= []
  14. end
  15. 6 def extra_options(options)
  16. 144 options.merge(expect_timeout: EXPECT_TIMEOUT)
  17. end
  18. end
  19. 6 module OptionsMethods
  20. 6 def option_expect_timeout(value)
  21. 252 seconds = Float(value)
  22. 252 raise TypeError, ":expect_timeout must be positive" unless seconds.positive?
  23. 252 seconds
  24. end
  25. 6 def option_expect_threshold_size(value)
  26. 12 bytes = Integer(value)
  27. 12 raise TypeError, ":expect_threshold_size must be positive" unless bytes.positive?
  28. 12 bytes
  29. end
  30. end
  31. 6 module RequestMethods
  32. 6 def initialize(*)
  33. 168 super
  34. 168 return if @body.empty?
  35. 108 threshold = @options.expect_threshold_size
  36. 108 return if threshold && !@body.unbounded_body? && @body.bytesize < threshold
  37. 96 return if Expect.no_expect_store.include?(origin)
  38. 75 @headers["expect"] = "100-continue"
  39. end
  40. 6 def response=(response)
  41. 132 if response.is_a?(Response) &&
  42. response.status == 100 &&
  43. !@headers.key?("expect") &&
  44. 2 (@state == :body || @state == :done)
  45. # if we're past this point, this means that we just received a 100-Continue response,
  46. # but the request doesn't have the expect flag, and is already flushing (or flushed) the body.
  47. #
  48. # this means that expect was deactivated for this request too soon, i.e. response took longer.
  49. #
  50. # so we have to reactivate it again.
  51. 7 @headers["expect"] = "100-continue"
  52. 8 @informational_status = 100
  53. 8 Expect.no_expect_store.delete(origin)
  54. end
  55. 132 super
  56. end
  57. end
  58. 6 module ConnectionMethods
  59. 6 def send_request_to_parser(request)
  60. 84 super
  61. 84 return unless request.headers["expect"] == "100-continue"
  62. 60 expect_timeout = request.options.expect_timeout
  63. 60 return if expect_timeout.nil? || expect_timeout.infinite?
  64. 60 set_request_timeout(request, expect_timeout, :expect, %i[body response]) do
  65. # expect timeout expired
  66. 20 if request.state == :expect && !request.expects?
  67. 20 Expect.no_expect_store << request.origin
  68. 20 request.headers.delete("expect")
  69. 20 consume
  70. end
  71. end
  72. end
  73. end
  74. 6 module InstanceMethods
  75. 6 def fetch_response(request, connections, options)
  76. 316 response = @responses.delete(request)
  77. 316 return unless response
  78. 84 if response.is_a?(Response) && response.status == 417 && request.headers.key?("expect")
  79. 12 response.close
  80. 12 request.headers.delete("expect")
  81. 12 request.transition(:idle)
  82. 12 send_request(request, connections, options)
  83. 10 return
  84. end
  85. 72 response
  86. end
  87. end
  88. end
  89. 6 register_plugin :expect, Expect
  90. end
  91. end

lib/httpx/plugins/follow_redirects.rb

100.0% lines covered

82 relevant lines. 82 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 12 module HTTPX
  3. 12 InsecureRedirectError = Class.new(Error)
  4. 12 module Plugins
  5. #
  6. # This plugin adds support for following redirect (status 30X) responses.
  7. #
  8. # It has an upper bound of followed redirects (see *MAX_REDIRECTS*), after which it
  9. # will return the last redirect response. It will **not** raise an exception.
  10. #
  11. # It also doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
  12. #
  13. # https://gitlab.com/os85/httpx/wikis/Follow-Redirects
  14. #
  15. 12 module FollowRedirects
  16. 12 MAX_REDIRECTS = 3
  17. 12 REDIRECT_STATUS = (300..399).freeze
  18. 12 using URIExtensions
  19. 12 module OptionsMethods
  20. 12 def option_max_redirects(value)
  21. 292 num = Integer(value)
  22. 292 raise TypeError, ":max_redirects must be positive" if num.negative?
  23. 292 num
  24. end
  25. 12 def option_follow_insecure_redirects(value)
  26. 18 value
  27. end
  28. 12 def option_allow_auth_to_other_origins(value)
  29. 18 value
  30. end
  31. 12 def option_redirect_on(value)
  32. 36 raise TypeError, ":redirect_on must be callable" unless value.respond_to?(:call)
  33. 36 value
  34. end
  35. end
  36. 12 module InstanceMethods
  37. 12 def max_redirects(n)
  38. 36 with(max_redirects: n.to_i)
  39. end
  40. 12 private
  41. 12 def fetch_response(request, connections, options)
  42. 439733 redirect_request = request.redirect_request
  43. 439733 response = super(redirect_request, connections, options)
  44. 439733 return unless response
  45. 381 max_redirects = redirect_request.max_redirects
  46. 381 return response unless response.is_a?(Response)
  47. 381 return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
  48. 249 return response unless max_redirects.positive?
  49. # build redirect request
  50. 225 redirect_uri = __get_location_from_response(response)
  51. 225 if options.redirect_on
  52. 24 redirect_allowed = options.redirect_on.call(redirect_uri)
  53. 24 return response unless redirect_allowed
  54. end
  55. 213 if response.status == 305 && options.respond_to?(:proxy)
  56. # The requested resource MUST be accessed through the proxy given by
  57. # the Location field. The Location field gives the URI of the proxy.
  58. 6 retry_options = options.merge(headers: redirect_request.headers,
  59. proxy: { uri: redirect_uri },
  60. body: redirect_request.body,
  61. max_redirects: max_redirects - 1)
  62. 6 redirect_uri = redirect_request.uri
  63. 6 options = retry_options
  64. else
  65. 207 redirect_headers = redirect_request_headers(redirect_request.uri, redirect_uri, request.headers, options)
  66. # redirects are **ALWAYS** GET
  67. 207 retry_opts = Hash[options].merge(
  68. headers: redirect_headers.to_h,
  69. body: redirect_request.body,
  70. max_redirects: max_redirects - 1
  71. )
  72. 207 retry_options = options.class.new(retry_opts)
  73. end
  74. 213 redirect_uri = Utils.to_uri(redirect_uri)
  75. 213 if !options.follow_insecure_redirects &&
  76. response.uri.scheme == "https" &&
  77. redirect_uri.scheme == "http"
  78. 6 error = InsecureRedirectError.new(redirect_uri.to_s)
  79. 6 error.set_backtrace(caller)
  80. 5 return ErrorResponse.new(request, error, options)
  81. end
  82. 207 retry_request = build_request("GET", redirect_uri, retry_options)
  83. 207 request.redirect_request = retry_request
  84. 207 retry_after = response.headers["retry-after"]
  85. 207 if retry_after
  86. # Servers send the "Retry-After" header field to indicate how long the
  87. # user agent ought to wait before making a follow-up request.
  88. # When sent with any 3xx (Redirection) response, Retry-After indicates
  89. # the minimum time that the user agent is asked to wait before issuing
  90. # the redirected request.
  91. #
  92. 12 retry_after = Utils.parse_retry_after(retry_after)
  93. 12 log { "redirecting after #{retry_after} secs..." }
  94. 12 pool.after(retry_after) do
  95. 12 send_request(retry_request, connections, options)
  96. end
  97. else
  98. 195 send_request(retry_request, connections, options)
  99. end
  100. 33 nil
  101. end
  102. 12 def redirect_request_headers(original_uri, redirect_uri, headers, options)
  103. 207 return headers if options.allow_auth_to_other_origins
  104. 201 return headers unless headers.key?("authorization")
  105. 12 unless original_uri.origin == redirect_uri.origin
  106. 6 headers = headers.dup
  107. 6 headers.delete("authorization")
  108. end
  109. 12 headers
  110. end
  111. 12 def __get_location_from_response(response)
  112. 225 location_uri = URI(response.headers["location"])
  113. 225 location_uri = response.uri.merge(location_uri) if location_uri.relative?
  114. 225 location_uri
  115. end
  116. end
  117. 12 module RequestMethods
  118. 12 attr_accessor :root_request
  119. 12 def redirect_request
  120. 439733 @redirect_request || self
  121. end
  122. 12 def redirect_request=(req)
  123. 207 @redirect_request = req
  124. 207 req.root_request = @root_request || self
  125. 207 @response = nil
  126. end
  127. 12 def response
  128. 1169 return super unless @redirect_request
  129. 10 @redirect_request.response
  130. end
  131. 12 def max_redirects
  132. 381 @options.max_redirects || MAX_REDIRECTS
  133. end
  134. end
  135. end
  136. 12 register_plugin :follow_redirects, FollowRedirects
  137. end
  138. end

lib/httpx/plugins/grpc.rb

100.0% lines covered

133 relevant lines. 133 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 5 module HTTPX
  3. 5 GRPCError = Class.new(Error) do
  4. 5 attr_reader :status, :details, :metadata
  5. 5 def initialize(status, details, metadata)
  6. 20 @status = status
  7. 20 @details = details
  8. 20 @metadata = metadata
  9. 20 super("GRPC error, code=#{status}, details=#{details}, metadata=#{metadata}")
  10. end
  11. end
  12. 5 module Plugins
  13. #
  14. # This plugin adds DSL to build GRPC interfaces.
  15. #
  16. # https://gitlab.com/os85/httpx/wikis/GRPC
  17. #
  18. 5 module GRPC
  19. 5 unless String.method_defined?(:underscore)
  20. 5 module StringExtensions
  21. 5 refine String do
  22. 5 def underscore
  23. 260 s = dup # Avoid mutating the argument, as it might be frozen.
  24. 260 s.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
  25. 260 s.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
  26. 260 s.tr!("-", "_")
  27. 260 s.downcase!
  28. 260 s
  29. end
  30. end
  31. end
  32. 5 using StringExtensions
  33. end
  34. 5 DEADLINE = 60
  35. 5 MARSHAL_METHOD = :encode
  36. 5 UNMARSHAL_METHOD = :decode
  37. 5 HEADERS = {
  38. "content-type" => "application/grpc",
  39. "te" => "trailers",
  40. "accept" => "application/grpc",
  41. # metadata fits here
  42. # ex "foo-bin" => base64("bar")
  43. }.freeze
  44. 5 class << self
  45. 5 def load_dependencies(*)
  46. 115 require "stringio"
  47. 115 require "httpx/plugins/grpc/grpc_encoding"
  48. 115 require "httpx/plugins/grpc/message"
  49. 115 require "httpx/plugins/grpc/call"
  50. end
  51. 5 def configure(klass)
  52. 115 klass.plugin(:persistent)
  53. 115 klass.plugin(:stream)
  54. end
  55. 5 def extra_options(options)
  56. 115 options.merge(
  57. fallback_protocol: "h2",
  58. grpc_rpcs: {}.freeze,
  59. grpc_compression: false,
  60. grpc_deadline: DEADLINE
  61. )
  62. end
  63. end
  64. 5 module OptionsMethods
  65. 5 def option_grpc_service(value)
  66. 100 String(value)
  67. end
  68. 5 def option_grpc_compression(value)
  69. 135 case value
  70. when true, false
  71. 115 value
  72. else
  73. 20 value.to_s
  74. end
  75. end
  76. 5 def option_grpc_rpcs(value)
  77. 930 Hash[value]
  78. end
  79. 5 def option_grpc_deadline(value)
  80. 670 raise TypeError, ":grpc_deadline must be positive" unless value.positive?
  81. 670 value
  82. end
  83. 5 def option_call_credentials(value)
  84. 15 raise TypeError, ":call_credentials must respond to #call" unless value.respond_to?(:call)
  85. 15 value
  86. end
  87. end
  88. 5 module ResponseMethods
  89. 5 attr_reader :trailing_metadata
  90. 5 def merge_headers(trailers)
  91. 95 @trailing_metadata = Hash[trailers]
  92. 95 super
  93. end
  94. end
  95. 5 module RequestBodyMethods
  96. 5 def initialize(headers, _)
  97. 105 super
  98. 105 if (compression = headers["grpc-encoding"])
  99. 10 deflater_body = self.class.initialize_deflater_body(@body, compression)
  100. 10 @body = Transcoder::GRPCEncoding.encode(deflater_body || @body, compressed: !deflater_body.nil?)
  101. else
  102. 95 @body = Transcoder::GRPCEncoding.encode(@body, compressed: false)
  103. end
  104. end
  105. end
  106. 5 module InstanceMethods
  107. 5 def with_channel_credentials(ca_path, key = nil, cert = nil, **ssl_opts)
  108. 60 ssl_params = {
  109. **ssl_opts,
  110. ca_file: ca_path,
  111. }
  112. 60 if key
  113. 60 key = File.read(key) if File.file?(key)
  114. 60 ssl_params[:key] = OpenSSL::PKey.read(key)
  115. end
  116. 60 if cert
  117. 60 cert = File.read(cert) if File.file?(cert)
  118. 60 ssl_params[:cert] = OpenSSL::X509::Certificate.new(cert)
  119. end
  120. 60 with(ssl: ssl_params)
  121. end
  122. 5 def rpc(rpc_name, input, output, **opts)
  123. 260 rpc_name = rpc_name.to_s
  124. 260 raise Error, "rpc #{rpc_name} already defined" if @options.grpc_rpcs.key?(rpc_name)
  125. rpc_opts = {
  126. 260 deadline: @options.grpc_deadline,
  127. }.merge(opts)
  128. 260 local_rpc_name = rpc_name.underscore
  129. 260 session_class = Class.new(self.class) do
  130. # define rpc method with ruby style name
  131. 260 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  132. def #{local_rpc_name}(input, **opts) # def grpc_action(input, **opts)
  133. rpc_execute("#{local_rpc_name}", input, **opts) # rpc_execute("grpc_action", input, **opts)
  134. end # end
  135. OUT
  136. # define rpc method with original name
  137. 260 unless local_rpc_name == rpc_name
  138. 10 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  139. def #{rpc_name}(input, **opts) # def grpcAction(input, **opts)
  140. rpc_execute("#{local_rpc_name}", input, **opts) # rpc_execute("grpc_action", input, **opts)
  141. end # end
  142. OUT
  143. end
  144. end
  145. 260 session_class.new(@options.merge(
  146. grpc_rpcs: @options.grpc_rpcs.merge(
  147. local_rpc_name => [rpc_name, input, output, rpc_opts]
  148. ).freeze
  149. ))
  150. end
  151. 5 def build_stub(origin, service: nil, compression: false)
  152. 115 scheme = @options.ssl.empty? ? "http" : "https"
  153. 115 origin = URI.parse("#{scheme}://#{origin}")
  154. 115 session = self
  155. 115 if service && service.respond_to?(:rpc_descs)
  156. # it's a grpc generic service
  157. 50 service.rpc_descs.each do |rpc_name, rpc_desc|
  158. rpc_opts = {
  159. 250 marshal_method: rpc_desc.marshal_method,
  160. unmarshal_method: rpc_desc.unmarshal_method,
  161. }
  162. 250 input = rpc_desc.input
  163. 250 input = input.type if input.respond_to?(:type)
  164. 250 output = rpc_desc.output
  165. 250 if output.respond_to?(:type)
  166. 100 rpc_opts[:stream] = true
  167. 100 output = output.type
  168. end
  169. 250 session = session.rpc(rpc_name, input, output, **rpc_opts)
  170. end
  171. 50 service = service.service_name
  172. end
  173. 115 session.with(origin: origin, grpc_service: service, grpc_compression: compression)
  174. end
  175. 5 def execute(rpc_method, input,
  176. deadline: DEADLINE,
  177. metadata: nil,
  178. **opts)
  179. 105 grpc_request = build_grpc_request(rpc_method, input, deadline: deadline, metadata: metadata, **opts)
  180. 105 response = request(grpc_request, **opts)
  181. 105 response.raise_for_status unless opts[:stream]
  182. 95 GRPC::Call.new(response)
  183. end
  184. 5 private
  185. 5 def rpc_execute(rpc_name, input, **opts)
  186. 50 rpc_name, input_enc, output_enc, rpc_opts = @options.grpc_rpcs[rpc_name]
  187. 50 exec_opts = rpc_opts.merge(opts)
  188. 50 marshal_method ||= exec_opts.delete(:marshal_method) || MARSHAL_METHOD
  189. 50 unmarshal_method ||= exec_opts.delete(:unmarshal_method) || UNMARSHAL_METHOD
  190. 50 messages = if input.respond_to?(:each)
  191. 20 Enumerator.new do |y|
  192. 20 input.each do |message|
  193. 40 y << input_enc.__send__(marshal_method, message)
  194. end
  195. end
  196. else
  197. 30 input_enc.__send__(marshal_method, input)
  198. end
  199. 50 call = execute(rpc_name, messages, **exec_opts)
  200. 50 call.decoder = output_enc.method(unmarshal_method)
  201. 50 call
  202. end
  203. 5 def build_grpc_request(rpc_method, input, deadline:, metadata: nil, **)
  204. 105 uri = @options.origin.dup
  205. 105 rpc_method = "/#{rpc_method}" unless rpc_method.start_with?("/")
  206. 105 rpc_method = "/#{@options.grpc_service}#{rpc_method}" if @options.grpc_service
  207. 105 uri.path = rpc_method
  208. 105 headers = HEADERS.merge(
  209. "grpc-accept-encoding" => ["identity", *@options.supported_compression_formats]
  210. )
  211. 105 unless deadline == Float::INFINITY
  212. # convert to milliseconds
  213. 105 deadline = (deadline * 1000.0).to_i
  214. 105 headers["grpc-timeout"] = "#{deadline}m"
  215. end
  216. 105 headers = headers.merge(metadata.transform_keys(&:to_s)) if metadata
  217. # prepare compressor
  218. 105 compression = @options.grpc_compression == true ? "gzip" : @options.grpc_compression
  219. 105 headers["grpc-encoding"] = compression if compression
  220. 105 headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
  221. 105 build_request("POST", uri, headers: headers, body: input)
  222. end
  223. end
  224. end
  225. 5 register_plugin :grpc, GRPC
  226. end
  227. end

lib/httpx/plugins/grpc/call.rb

90.91% lines covered

33 relevant lines. 30 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 5 module HTTPX
  3. 5 module Plugins
  4. 5 module GRPC
  5. # Encapsulates call information
  6. 5 class Call
  7. 5 attr_writer :decoder
  8. 5 def initialize(response)
  9. 95 @response = response
  10. 130 @decoder = ->(z) { z }
  11. 95 @consumed = false
  12. 95 @grpc_response = nil
  13. end
  14. 5 def inspect
  15. "#GRPC::Call(#{grpc_response})"
  16. end
  17. 5 def to_s
  18. 55 grpc_response.to_s
  19. end
  20. 5 def metadata
  21. response.headers
  22. end
  23. 5 def trailing_metadata
  24. 60 return unless @consumed
  25. 40 @response.trailing_metadata
  26. end
  27. 5 private
  28. 5 def grpc_response
  29. 155 @grpc_response ||= if @response.respond_to?(:each)
  30. 20 Enumerator.new do |y|
  31. 20 Message.stream(@response).each do |message|
  32. 40 y << @decoder.call(message)
  33. end
  34. 20 @consumed = true
  35. end
  36. else
  37. 75 @consumed = true
  38. 75 @decoder.call(Message.unary(@response))
  39. end
  40. end
  41. 5 def respond_to_missing?(meth, *args, &blk)
  42. 20 grpc_response.respond_to?(meth, *args) || super
  43. end
  44. 5 def method_missing(meth, *args, &blk)
  45. 40 return grpc_response.__send__(meth, *args, &blk) if grpc_response.respond_to?(meth)
  46. super
  47. end
  48. end
  49. end
  50. end
  51. end

lib/httpx/plugins/grpc/grpc_encoding.rb

97.83% lines covered

46 relevant lines. 45 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 5 module HTTPX
  3. 5 module Transcoder
  4. 5 module GRPCEncoding
  5. 5 class Deflater
  6. 5 extend Forwardable
  7. 5 attr_reader :content_type
  8. 5 def initialize(body, compressed:)
  9. 105 @content_type = body.content_type
  10. 105 @body = BodyReader.new(body)
  11. 105 @compressed = compressed
  12. end
  13. 5 def bytesize
  14. 335 return @body.bytesize if @body.respond_to?(:bytesize)
  15. Float::INFINITY
  16. end
  17. 5 def read(length = nil, outbuf = nil)
  18. 220 buf = @body.read(length, outbuf)
  19. 210 return unless buf
  20. 115 compressed_flag = @compressed ? 1 : 0
  21. 115 buf = outbuf if outbuf
  22. 115 buf.prepend([compressed_flag, buf.bytesize].pack("CL>"))
  23. 115 buf
  24. end
  25. end
  26. 5 class Inflater
  27. 5 def initialize(response)
  28. 75 @response = response
  29. 75 @grpc_encodings = nil
  30. end
  31. 5 def call(message, &blk)
  32. 95 data = "".b
  33. 95 until message.empty?
  34. 95 compressed, size = message.unpack("CL>")
  35. 95 encoded_data = message.byteslice(5..size + 5 - 1)
  36. 95 if compressed == 1
  37. 10 grpc_encodings.reverse_each do |encoding|
  38. 10 decoder = @response.body.class.initialize_inflater_by_encoding(encoding, @response, bytesize: encoded_data.bytesize)
  39. 10 encoded_data = decoder.call(encoded_data)
  40. 10 blk.call(encoded_data) if blk
  41. 10 data << encoded_data
  42. end
  43. else
  44. 85 blk.call(encoded_data) if blk
  45. 85 data << encoded_data
  46. end
  47. 95 message = message.byteslice((size + 5)..-1)
  48. end
  49. 95 data
  50. end
  51. 5 private
  52. 5 def grpc_encodings
  53. 10 @grpc_encodings ||= @response.headers.get("grpc-encoding")
  54. end
  55. end
  56. 5 def self.encode(*args, **kwargs)
  57. 105 Deflater.new(*args, **kwargs)
  58. end
  59. 5 def self.decode(response)
  60. 75 Inflater.new(response)
  61. end
  62. end
  63. end
  64. end

lib/httpx/plugins/grpc/message.rb

95.83% lines covered

24 relevant lines. 23 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 5 module HTTPX
  3. 5 module Plugins
  4. 5 module GRPC
  5. # Encoding module for GRPC responses
  6. #
  7. # Can encode and decode grpc messages.
  8. 5 module Message
  9. 5 module_function
  10. # decodes a unary grpc response
  11. 5 def unary(response)
  12. 75 verify_status(response)
  13. 55 decoder = Transcoder::GRPCEncoding.decode(response)
  14. 55 decoder.call(response.to_s)
  15. end
  16. # lazy decodes a grpc stream response
  17. 5 def stream(response, &block)
  18. 40 return enum_for(__method__, response) unless block
  19. 20 decoder = Transcoder::GRPCEncoding.decode(response)
  20. 20 response.each do |frame|
  21. 40 decoder.call(frame, &block)
  22. end
  23. 20 verify_status(response)
  24. end
  25. 5 def cancel(request)
  26. request.emit(:refuse, :client_cancellation)
  27. end
  28. # interprets the grpc call trailing metadata, and raises an
  29. # exception in case of error code
  30. 5 def verify_status(response)
  31. # return standard errors if need be
  32. 95 response.raise_for_status
  33. 95 status = Integer(response.headers["grpc-status"])
  34. 95 message = response.headers["grpc-message"]
  35. 95 return if status.zero?
  36. 20 response.close
  37. 20 raise GRPCError.new(status, message, response.trailing_metadata)
  38. end
  39. end
  40. end
  41. end
  42. end

lib/httpx/plugins/h2c.rb

94.44% lines covered

54 relevant lines. 51 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
  6. # (https://datatracker.ietf.org/doc/html/rfc7540#section-3.2)
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2c
  9. #
  10. 6 module H2C
  11. 6 VALID_H2C_VERBS = %w[GET OPTIONS HEAD].freeze
  12. 6 class << self
  13. 6 def load_dependencies(klass)
  14. 12 klass.plugin(:upgrade)
  15. end
  16. 6 def call(connection, request, response)
  17. 12 connection.upgrade_to_h2c(request, response)
  18. end
  19. 6 def extra_options(options)
  20. 12 options.merge(max_concurrent_requests: 1, upgrade_handlers: options.upgrade_handlers.merge("h2c" => self))
  21. end
  22. end
  23. 6 module InstanceMethods
  24. 6 def send_requests(*requests)
  25. 18 upgrade_request, *remainder = requests
  26. 18 return super unless VALID_H2C_VERBS.include?(upgrade_request.verb) && upgrade_request.scheme == "http"
  27. 18 connection = pool.find_connection(upgrade_request.uri, upgrade_request.options)
  28. 18 return super if connection && connection.upgrade_protocol == "h2c"
  29. # build upgrade request
  30. 12 upgrade_request.headers.add("connection", "upgrade")
  31. 12 upgrade_request.headers.add("connection", "http2-settings")
  32. 10 upgrade_request.headers["upgrade"] = "h2c"
  33. 10 upgrade_request.headers["http2-settings"] = HTTP2Next::Client.settings_header(upgrade_request.options.http2_settings)
  34. 12 super(upgrade_request, *remainder)
  35. end
  36. end
  37. 6 class H2CParser < Connection::HTTP2
  38. 6 def upgrade(request, response)
  39. # skip checks, it is assumed that this is the first
  40. # request in the connection
  41. 12 stream = @connection.upgrade
  42. # on_settings
  43. 12 handle_stream(stream, request)
  44. 10 @streams[request] = stream
  45. # clean up data left behind in the buffer, if the server started
  46. # sending frames
  47. 12 data = response.read
  48. 12 @connection << data
  49. end
  50. end
  51. 6 module ConnectionMethods
  52. 6 using URIExtensions
  53. 6 def upgrade_to_h2c(request, response)
  54. 12 prev_parser = @parser
  55. 12 if prev_parser
  56. 12 prev_parser.reset
  57. 10 @inflight -= prev_parser.requests.size
  58. end
  59. 12 @parser = H2CParser.new(@write_buffer, @options)
  60. 12 set_parser_callbacks(@parser)
  61. 10 @inflight += 1
  62. 12 @parser.upgrade(request, response)
  63. 12 @upgrade_protocol = "h2c"
  64. 12 prev_parser.requests.each do |req|
  65. 12 req.transition(:idle)
  66. 12 send(req)
  67. end
  68. end
  69. 6 private
  70. 6 def send_request_to_parser(request)
  71. 42 super
  72. 42 return unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
  73. 12 max_concurrent_requests = parser.max_concurrent_requests
  74. 12 return if max_concurrent_requests == 1
  75. parser.max_concurrent_requests = 1
  76. request.once(:response) do
  77. parser.max_concurrent_requests = max_concurrent_requests
  78. end
  79. end
  80. end
  81. end
  82. 6 register_plugin(:h2c, H2C)
  83. end
  84. end

lib/httpx/plugins/ntlm_auth.rb

100.0% lines covered

30 relevant lines. 30 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 5 module HTTPX
  3. 5 module Plugins
  4. #
  5. # https://gitlab.com/os85/httpx/wikis/Auth#ntlm-auth
  6. #
  7. 5 module NTLMAuth
  8. 5 class << self
  9. 5 def load_dependencies(_klass)
  10. 2 require_relative "auth/ntlm"
  11. end
  12. 5 def extra_options(options)
  13. 2 options.merge(max_concurrent_requests: 1)
  14. end
  15. end
  16. 5 module OptionsMethods
  17. 5 def option_ntlm(value)
  18. 8 raise TypeError, ":ntlm must be a #{Authentication::Ntlm}" unless value.is_a?(Authentication::Ntlm)
  19. 8 value
  20. end
  21. end
  22. 5 module InstanceMethods
  23. 5 def ntlm_auth(user, password, domain = nil)
  24. 4 with(ntlm: Authentication::Ntlm.new(user, password, domain: domain))
  25. end
  26. 5 private
  27. 5 def send_requests(*requests)
  28. 8 requests.flat_map do |request|
  29. 8 ntlm = request.options.ntlm
  30. 8 if ntlm
  31. 4 request.headers["authorization"] = ntlm.negotiate
  32. 8 probe_response = wrap { super(request).first }
  33. 4 return probe_response unless probe_response.is_a?(Response)
  34. 4 if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
  35. 2 request.transition(:idle)
  36. 2 request.headers["authorization"] = ntlm.authenticate(request, probe_response.headers["www-authenticate"])
  37. 2 super(request)
  38. else
  39. 2 probe_response
  40. end
  41. else
  42. 4 super(request)
  43. end
  44. end
  45. end
  46. end
  47. end
  48. 5 register_plugin :ntlm_auth, NTLMAuth
  49. end
  50. end

lib/httpx/plugins/oauth.rb

89.53% lines covered

86 relevant lines. 77 lines covered and 9 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # https://gitlab.com/os85/httpx/wikis/OAuth
  6. #
  7. 6 module OAuth
  8. 6 class << self
  9. 6 def load_dependencies(_klass)
  10. 60 require_relative "auth/basic"
  11. end
  12. end
  13. 6 SUPPORTED_GRANT_TYPES = %w[client_credentials refresh_token].freeze
  14. 6 SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
  15. 6 class OAuthSession
  16. 6 attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope
  17. 6 def initialize(
  18. issuer:,
  19. client_id:,
  20. client_secret:,
  21. access_token: nil,
  22. refresh_token: nil,
  23. scope: nil,
  24. token_endpoint: nil,
  25. response_type: nil,
  26. grant_type: nil,
  27. token_endpoint_auth_method: nil
  28. )
  29. 60 @issuer = URI(issuer)
  30. 60 @client_id = client_id
  31. 60 @client_secret = client_secret
  32. 60 @token_endpoint = URI(token_endpoint) if token_endpoint
  33. 60 @response_type = response_type
  34. 60 @scope = case scope
  35. when String
  36. 24 scope.split
  37. when Array
  38. 24 scope
  39. end
  40. 60 @access_token = access_token
  41. 60 @refresh_token = refresh_token
  42. 60 @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
  43. 60 @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
  44. 60 unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
  45. 12 raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
  46. end
  47. 48 return if SUPPORTED_GRANT_TYPES.include?(@grant_type)
  48. raise Error, "#{@grant_type} is not a supported grant type"
  49. end
  50. 6 def token_endpoint
  51. 48 @token_endpoint || "#{@issuer}/token"
  52. end
  53. 6 def token_endpoint_auth_method
  54. 72 @token_endpoint_auth_method || "client_secret_basic"
  55. end
  56. 6 def load(http)
  57. 24 return if @grant_type && @scope
  58. metadata = http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json
  59. @token_endpoint = metadata["token_endpoint"]
  60. @scope = metadata["scopes_supported"]
  61. @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
  62. @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
  63. SUPPORTED_AUTH_METHODS.include?(am)
  64. end
  65. nil
  66. end
  67. 6 def merge(other)
  68. 48 obj = dup
  69. 40 case other
  70. when OAuthSession
  71. 24 other.instance_variables.each do |ivar|
  72. 204 val = other.instance_variable_get(ivar)
  73. 204 next unless val
  74. 156 obj.instance_variable_set(ivar, val)
  75. end
  76. when Hash
  77. 24 other.each do |k, v|
  78. 48 obj.instance_variable_set(:"@#{k}", v) if obj.instance_variable_defined?(:"@#{k}")
  79. end
  80. end
  81. 48 obj
  82. end
  83. end
  84. 6 module OptionsMethods
  85. 6 def option_oauth_session(value)
  86. 120 case value
  87. when Hash
  88. OAuthSession.new(**value)
  89. when OAuthSession
  90. 144 value
  91. else
  92. raise TypeError, ":oauth_session must be a #{OAuthSession}"
  93. end
  94. end
  95. end
  96. 6 module InstanceMethods
  97. 6 def oauth_auth(**args)
  98. 60 with(oauth_session: OAuthSession.new(**args))
  99. end
  100. 6 def with_access_token
  101. 24 oauth_session = @options.oauth_session
  102. 24 oauth_session.load(self)
  103. 24 grant_type = oauth_session.grant_type
  104. 24 headers = {}
  105. 24 form_post = { "grant_type" => grant_type, "scope" => Array(oauth_session.scope).join(" ") }.compact
  106. # auth
  107. 20 case oauth_session.token_endpoint_auth_method
  108. when "client_secret_post"
  109. 10 form_post["client_id"] = oauth_session.client_id
  110. 10 form_post["client_secret"] = oauth_session.client_secret
  111. when "client_secret_basic"
  112. 10 headers["authorization"] = Authentication::Basic.new(oauth_session.client_id, oauth_session.client_secret).authenticate
  113. end
  114. 20 case grant_type
  115. when "client_credentials"
  116. # do nothing
  117. when "refresh_token"
  118. 10 form_post["refresh_token"] = oauth_session.refresh_token
  119. end
  120. 24 token_request = build_request("POST", oauth_session.token_endpoint, headers: headers, form: form_post)
  121. 24 token_request.headers.delete("authorization") unless oauth_session.token_endpoint_auth_method == "client_secret_basic"
  122. 24 token_response = request(token_request)
  123. 24 token_response.raise_for_status
  124. 24 payload = token_response.json
  125. 24 access_token = payload["access_token"]
  126. 24 refresh_token = payload["refresh_token"]
  127. 24 with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
  128. end
  129. 6 def build_request(*, _)
  130. 72 request = super
  131. 72 return request if request.headers.key?("authorization")
  132. 60 oauth_session = @options.oauth_session
  133. 60 return request unless oauth_session && oauth_session.access_token
  134. 40 request.headers["authorization"] = "Bearer #{oauth_session.access_token}"
  135. 48 request
  136. end
  137. end
  138. end
  139. 6 register_plugin :oauth, OAuth
  140. end
  141. end

lib/httpx/plugins/persistent.rb

100.0% lines covered

11 relevant lines. 11 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. # This plugin implements a session that persists connections over the duration of the process.
  5. #
  6. # This will improve connection reuse in a long-running process.
  7. #
  8. # One important caveat to note is, although this session might not close connections,
  9. # other sessions from the same process that don't have this plugin turned on might.
  10. #
  11. # This session will still be able to work with it, as if, when expecting a connection
  12. # terminated by a different session, it will just retry on a new one and keep it open.
  13. #
  14. # This plugin is also not recommendable when connecting to >9000 (like, a lot) different origins.
  15. # So when you use this, make sure that you don't fall into this trap.
  16. #
  17. # https://gitlab.com/os85/httpx/wikis/Persistent
  18. #
  19. 8 module Persistent
  20. 8 def self.load_dependencies(klass)
  21. 321 max_retries = if klass.default_options.respond_to?(:max_retries)
  22. 6 [klass.default_options.max_retries, 1].max
  23. else
  24. 315 1
  25. end
  26. 321 klass.plugin(:retries, max_retries: max_retries, retry_change_requests: true)
  27. end
  28. 8 def self.extra_options(options)
  29. 321 options.merge(persistent: true)
  30. end
  31. end
  32. 8 register_plugin :persistent, Persistent
  33. end
  34. end

lib/httpx/plugins/proxy.rb

97.28% lines covered

147 relevant lines. 143 lines covered and 4 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 class HTTPProxyError < ConnectionError; end
  4. 8 module Plugins
  5. #
  6. # This plugin adds support for proxies. It ships with support for:
  7. #
  8. # * HTTP proxies
  9. # * HTTPS proxies
  10. # * Socks4/4a proxies
  11. # * Socks5 proxies
  12. #
  13. # https://gitlab.com/os85/httpx/wikis/Proxy
  14. #
  15. 8 module Proxy
  16. 8 Error = HTTPProxyError
  17. 8 PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
  18. 8 class << self
  19. 8 def configure(klass)
  20. 253 klass.plugin(:"proxy/http")
  21. 253 klass.plugin(:"proxy/socks4")
  22. 253 klass.plugin(:"proxy/socks5")
  23. end
  24. 8 def extra_options(options)
  25. 253 options.merge(supported_proxy_protocols: [])
  26. end
  27. end
  28. 8 class Parameters
  29. 8 attr_reader :uri, :username, :password, :scheme
  30. 8 def initialize(uri:, scheme: nil, username: nil, password: nil, **extra)
  31. 326 @uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
  32. 326 @username = username || @uri.user
  33. 326 @password = password || @uri.password
  34. 326 return unless @username && @password
  35. 193 scheme ||= case @uri.scheme
  36. when "socks5"
  37. 36 @uri.scheme
  38. when "http", "https"
  39. 97 "basic"
  40. else
  41. 36 return
  42. end
  43. 157 @scheme = scheme
  44. 157 auth_scheme = scheme.to_s.capitalize
  45. 157 require_relative "auth/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
  46. 157 @authenticator = Authentication.const_get(auth_scheme).new(@username, @password, **extra)
  47. end
  48. 8 def can_authenticate?(*args)
  49. 138 return false unless @authenticator
  50. 48 @authenticator.can_authenticate?(*args)
  51. end
  52. 8 def authenticate(*args)
  53. 121 return unless @authenticator
  54. 121 @authenticator.authenticate(*args)
  55. end
  56. 8 def ==(other)
  57. 61 case other
  58. when Parameters
  59. 47 @uri == other.uri &&
  60. @username == other.username &&
  61. @password == other.password &&
  62. @scheme == other.scheme
  63. when URI::Generic, String
  64. 18 proxy_uri = @uri.dup
  65. 18 proxy_uri.user = @username
  66. 18 proxy_uri.password = @password
  67. 18 other_uri = other.is_a?(URI::Generic) ? other : URI.parse(other)
  68. 18 proxy_uri == other_uri
  69. else
  70. 6 super
  71. end
  72. end
  73. end
  74. 8 module OptionsMethods
  75. 8 def option_proxy(value)
  76. 805 value.is_a?(Parameters) ? value : Hash[value]
  77. end
  78. 8 def option_supported_proxy_protocols(value)
  79. 1277 raise TypeError, ":supported_proxy_protocols must be an Array" unless value.is_a?(Array)
  80. 1277 value.map(&:to_s)
  81. end
  82. end
  83. 8 module InstanceMethods
  84. 8 private
  85. 8 def find_connection(request, connections, options)
  86. 333 return super unless options.respond_to?(:proxy)
  87. 333 uri = URI(request.uri)
  88. 333 proxy_opts = if (next_proxy = uri.find_proxy)
  89. 4 { uri: next_proxy }
  90. else
  91. 329 proxy = options.proxy
  92. 329 return super unless proxy
  93. 322 return super(request, connections, options.merge(proxy: nil)) unless proxy.key?(:uri)
  94. 322 @_proxy_uris ||= Array(proxy[:uri])
  95. 322 next_proxy = @_proxy_uris.first
  96. 322 raise Error, "Failed to connect to proxy" unless next_proxy
  97. 310 next_proxy = URI(next_proxy)
  98. 1 raise Error,
  99. 310 "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
  100. 304 if proxy.key?(:no_proxy)
  101. 12 no_proxy = proxy[:no_proxy]
  102. 12 no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
  103. 12 return super(request, connections, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(uri.host, next_proxy.host,
  104. next_proxy.port, no_proxy)
  105. end
  106. 298 proxy.merge(uri: next_proxy)
  107. end
  108. 302 proxy = Parameters.new(**proxy_opts)
  109. 302 proxy_options = options.merge(proxy: proxy)
  110. 302 connection = pool.find_connection(uri, proxy_options) || init_connection(uri, proxy_options)
  111. 302 unless connections.nil? || connections.include?(connection)
  112. 283 connections << connection
  113. 283 set_connection_callbacks(connection, connections, options)
  114. end
  115. 302 connection
  116. end
  117. 8 def fetch_response(request, connections, options)
  118. 1117 response = super
  119. 1117 if response.is_a?(ErrorResponse) && proxy_error?(request, response)
  120. 72 @_proxy_uris.shift
  121. # return last error response if no more proxies to try
  122. 72 return response if @_proxy_uris.empty?
  123. 12 log { "failed connecting to proxy, trying next..." }
  124. 12 request.transition(:idle)
  125. 12 send_request(request, connections, options)
  126. 10 return
  127. end
  128. 1045 response
  129. end
  130. 8 def proxy_error?(_request, response)
  131. 108 error = response.error
  132. 90 case error
  133. when NativeResolveError
  134. 12 return false unless @_proxy_uris && !@_proxy_uris.empty?
  135. 12 proxy_uri = URI(@_proxy_uris.first)
  136. 12 origin = error.connection.origin
  137. # failed resolving proxy domain
  138. 12 origin.host == proxy_uri.host && origin.port == proxy_uri.port
  139. when ResolveError
  140. return false unless @_proxy_uris && !@_proxy_uris.empty?
  141. proxy_uri = URI(@_proxy_uris.first)
  142. error.message.end_with?(proxy_uri.to_s)
  143. when *PROXY_ERRORS
  144. # timeout errors connecting to proxy
  145. 96 true
  146. else
  147. false
  148. end
  149. end
  150. end
  151. 8 module ConnectionMethods
  152. 8 using URIExtensions
  153. 8 def initialize(*)
  154. 296 super
  155. 296 return unless @options.proxy
  156. # redefining the connection origin as the proxy's URI,
  157. # as this will be used as the tcp peer ip.
  158. 283 proxy_uri = URI(@options.proxy.uri)
  159. 283 @origin.host = proxy_uri.host
  160. 283 @origin.port = proxy_uri.port
  161. end
  162. 8 def coalescable?(connection)
  163. 10 return super unless @options.proxy
  164. 10 if @io.protocol == "h2" &&
  165. @origin.scheme == "https" &&
  166. connection.origin.scheme == "https" &&
  167. @io.can_verify_peer?
  168. # in proxied connections, .origin is the proxy ; Given names
  169. # are stored in .origins, this is what is used.
  170. 5 origin = URI(connection.origins.first)
  171. 5 @io.verify_hostname(origin.host)
  172. else
  173. 5 @origin == connection.origin
  174. end
  175. end
  176. 8 def connecting?
  177. 435310 return super unless @options.proxy
  178. 435189 super || @state == :connecting || @state == :connected
  179. end
  180. 8 def call
  181. 786 super
  182. 786 return unless @options.proxy
  183. 655 case @state
  184. when :connecting
  185. 175 consume
  186. end
  187. end
  188. 8 def reset
  189. 393 return super unless @options.proxy
  190. 367 @state = :open
  191. 367 super
  192. 367 emit(:close)
  193. end
  194. 8 private
  195. 8 def initialize_type(uri, options)
  196. 296 return super unless options.proxy
  197. 283 "tcp"
  198. end
  199. 8 def connect
  200. 832 return super unless @options.proxy
  201. 680 case @state
  202. when :idle
  203. 562 transition(:connecting)
  204. when :connected
  205. 244 transition(:open)
  206. end
  207. end
  208. 8 def handle_transition(nextstate)
  209. 1902 return super unless @options.proxy
  210. 1552 case nextstate
  211. when :closing
  212. # this is a hack so that we can use the super method
  213. # and it'll think that the current state is open
  214. 417 @state = :open if @state == :connecting
  215. end
  216. 1836 super
  217. end
  218. end
  219. end
  220. 8 register_plugin :proxy, Proxy
  221. end
  222. 8 class ProxySSL < SSL
  223. 8 def initialize(tcp, request_uri, options)
  224. 68 @io = tcp.to_io
  225. 68 super(request_uri, tcp.addresses, options)
  226. 68 @hostname = request_uri.host
  227. 68 @state = :connected
  228. end
  229. end
  230. end

lib/httpx/plugins/proxy/http.rb

100.0% lines covered

99 relevant lines. 99 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. 8 module Proxy
  5. 8 module HTTP
  6. 8 class << self
  7. 8 def extra_options(options)
  8. 253 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[http])
  9. end
  10. end
  11. 8 module InstanceMethods
  12. 8 def with_proxy_basic_auth(opts)
  13. 6 with(proxy: opts.merge(scheme: "basic"))
  14. end
  15. 8 def with_proxy_digest_auth(opts)
  16. 18 with(proxy: opts.merge(scheme: "digest"))
  17. end
  18. 8 def with_proxy_ntlm_auth(opts)
  19. 6 with(proxy: opts.merge(scheme: "ntlm"))
  20. end
  21. 8 def fetch_response(request, connections, options)
  22. 1117 response = super
  23. 1117 if response &&
  24. response.is_a?(Response) &&
  25. response.status == 407 &&
  26. !request.headers.key?("proxy-authorization") &&
  27. response.headers.key?("proxy-authenticate")
  28. 18 connection = find_connection(request, connections, options)
  29. 18 if connection.options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
  30. 6 request.transition(:idle)
  31. 5 request.headers["proxy-authorization"] =
  32. connection.options.proxy.authenticate(request, response.headers["proxy-authenticate"])
  33. 6 send_request(request, connections)
  34. 6 return
  35. end
  36. end
  37. 1111 response
  38. end
  39. end
  40. 8 module ConnectionMethods
  41. 8 def connecting?
  42. 435310 super || @state == :connecting || @state == :connected
  43. end
  44. 8 private
  45. 8 def handle_transition(nextstate)
  46. 2108 return super unless @options.proxy && @options.proxy.uri.scheme == "http"
  47. 972 case nextstate
  48. when :connecting
  49. 256 return unless @state == :idle
  50. 256 @io.connect
  51. 256 return unless @io.connected?
  52. 128 @parser || begin
  53. 122 @parser = self.class.parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
  54. 122 parser = @parser
  55. 122 parser.extend(ProxyParser)
  56. 122 parser.on(:response, &method(:__http_on_connect))
  57. 166 parser.on(:close) { transition(:closing) }
  58. 122 parser.on(:reset) do
  59. 12 if parser.empty?
  60. 6 reset
  61. else
  62. 6 transition(:closing)
  63. 6 transition(:closed)
  64. 6 parser.reset if @parser
  65. 6 transition(:idle)
  66. 6 transition(:connecting)
  67. end
  68. end
  69. 122 __http_proxy_connect(parser)
  70. end
  71. 128 return if @state == :connected
  72. when :connected
  73. 116 return unless @state == :idle || @state == :connecting
  74. 99 case @state
  75. when :connecting
  76. 38 @parser.close
  77. 38 @parser = nil
  78. when :idle
  79. 78 @parser.callbacks.clear
  80. 78 set_parser_callbacks(@parser)
  81. end
  82. end
  83. 930 super
  84. end
  85. 8 def __http_proxy_connect(parser)
  86. 122 req = @pending.first
  87. 122 if req && req.uri.scheme == "https"
  88. # if the first request after CONNECT is to an https address, it is assumed that
  89. # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
  90. # and therefore, will share the connection.
  91. #
  92. 44 connect_request = ConnectRequest.new(req.uri, @options)
  93. 38 @inflight += 1
  94. 44 parser.send(connect_request)
  95. else
  96. 78 handle_transition(:connected)
  97. end
  98. end
  99. 8 def __http_on_connect(request, response)
  100. 43 @inflight -= 1
  101. 50 if response.is_a?(Response) && response.status == 200
  102. 38 req = @pending.first
  103. 38 request_uri = req.uri
  104. 38 @io = ProxySSL.new(@io, request_uri, @options)
  105. 38 transition(:connected)
  106. 38 throw(:called)
  107. 12 elsif response.is_a?(Response) &&
  108. response.status == 407 &&
  109. !request.headers.key?("proxy-authorization") &&
  110. @options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
  111. 6 request.transition(:idle)
  112. 5 request.headers["proxy-authorization"] = @options.proxy.authenticate(request, response.headers["proxy-authenticate"])
  113. 6 @parser.send(request)
  114. 5 @inflight += 1
  115. else
  116. 6 pending = @pending + @parser.pending
  117. 17 while (req = pending.shift)
  118. 6 req.emit(:response, response)
  119. end
  120. 6 reset
  121. end
  122. end
  123. end
  124. 8 module ProxyParser
  125. 8 def join_headline(request)
  126. 110 return super if request.verb == "CONNECT"
  127. 51 "#{request.verb} #{request.uri} HTTP/#{@version.join(".")}"
  128. end
  129. 8 def set_protocol_headers(request)
  130. 116 extra_headers = super
  131. 116 proxy_params = @options.proxy
  132. 116 if proxy_params.scheme == "basic"
  133. # opt for basic auth
  134. 63 extra_headers["proxy-authorization"] = proxy_params.authenticate(extra_headers)
  135. end
  136. 116 extra_headers["proxy-connection"] = extra_headers.delete("connection") if extra_headers.key?("connection")
  137. 116 extra_headers
  138. end
  139. end
  140. 8 class ConnectRequest < Request
  141. 8 def initialize(uri, _options)
  142. 44 super("CONNECT", uri, {})
  143. 44 @headers.delete("accept")
  144. end
  145. 8 def path
  146. 48 "#{@uri.hostname}:#{@uri.port}"
  147. end
  148. end
  149. end
  150. end
  151. 8 register_plugin :"proxy/http", Proxy::HTTP
  152. end
  153. end

lib/httpx/plugins/proxy/socks4.rb

98.72% lines covered

78 relevant lines. 77 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "resolv"
  3. 8 require "ipaddr"
  4. 8 module HTTPX
  5. 8 class Socks4Error < HTTPProxyError; end
  6. 8 module Plugins
  7. 8 module Proxy
  8. 8 module Socks4
  9. 8 VERSION = 4
  10. 8 CONNECT = 1
  11. 8 GRANTED = 0x5A
  12. 8 PROTOCOLS = %w[socks4 socks4a].freeze
  13. 8 Error = Socks4Error
  14. 8 class << self
  15. 8 def extra_options(options)
  16. 253 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + PROTOCOLS)
  17. end
  18. end
  19. 8 module ConnectionMethods
  20. 8 def interests
  21. 434757 if @state == :connecting
  22. return @write_buffer.empty? ? :r : :w
  23. end
  24. 434757 super
  25. end
  26. 8 private
  27. 8 def handle_transition(nextstate)
  28. 2156 return super unless @options.proxy && PROTOCOLS.include?(@options.proxy.uri.scheme)
  29. 305 case nextstate
  30. when :connecting
  31. 96 return unless @state == :idle
  32. 96 @io.connect
  33. 96 return unless @io.connected?
  34. 48 req = @pending.first
  35. 48 return unless req
  36. 48 request_uri = req.uri
  37. 48 @write_buffer << Packet.connect(@options.proxy, request_uri)
  38. 48 __socks4_proxy_connect
  39. when :connected
  40. 36 return unless @state == :connecting
  41. 36 @parser = nil
  42. end
  43. 318 log(level: 1) { "SOCKS4: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
  44. 318 super
  45. end
  46. 8 def __socks4_proxy_connect
  47. 48 @parser = SocksParser.new(@write_buffer, @options)
  48. 48 @parser.once(:packet, &method(:__socks4_on_packet))
  49. end
  50. 8 def __socks4_on_packet(packet)
  51. 48 _version, status, _port, _ip = packet.unpack("CCnN")
  52. 48 if status == GRANTED
  53. 36 req = @pending.first
  54. 36 request_uri = req.uri
  55. 36 @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
  56. 36 transition(:connected)
  57. 36 throw(:called)
  58. else
  59. 12 on_socks4_error("socks error: #{status}")
  60. end
  61. end
  62. 8 def on_socks4_error(message)
  63. 12 ex = Error.new(message)
  64. 12 ex.set_backtrace(caller)
  65. 12 on_error(ex)
  66. 12 throw(:called)
  67. end
  68. end
  69. 8 class SocksParser
  70. 8 include HTTPX::Callbacks
  71. 8 def initialize(buffer, options)
  72. 48 @buffer = buffer
  73. 48 @options = Options.new(options)
  74. end
  75. 8 def close; end
  76. 8 def consume(*); end
  77. 8 def empty?
  78. 12 true
  79. end
  80. 8 def <<(packet)
  81. 48 emit(:packet, packet)
  82. end
  83. end
  84. 8 module Packet
  85. 8 module_function
  86. 8 def connect(parameters, uri)
  87. 48 packet = [VERSION, CONNECT, uri.port].pack("CCn")
  88. 40 case parameters.uri.scheme
  89. when "socks4"
  90. 36 socks_host = uri.host
  91. begin
  92. 72 ip = IPAddr.new(socks_host)
  93. 36 packet << ip.hton
  94. rescue IPAddr::InvalidAddressError
  95. 36 socks_host = Resolv.getaddress(socks_host)
  96. 36 retry
  97. end
  98. 36 packet << [parameters.username].pack("Z*")
  99. when "socks4a"
  100. 12 packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
  101. end
  102. 48 packet
  103. end
  104. end
  105. end
  106. end
  107. 8 register_plugin :"proxy/socks4", Proxy::Socks4
  108. end
  109. end

lib/httpx/plugins/proxy/socks5.rb

100.0% lines covered

112 relevant lines. 112 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 class Socks5Error < HTTPProxyError; end
  4. 8 module Plugins
  5. 8 module Proxy
  6. 8 module Socks5
  7. 8 VERSION = 5
  8. 8 NOAUTH = 0
  9. 8 PASSWD = 2
  10. 8 NONE = 0xff
  11. 8 CONNECT = 1
  12. 8 IPV4 = 1
  13. 8 DOMAIN = 3
  14. 8 IPV6 = 4
  15. 8 SUCCESS = 0
  16. 8 Error = Socks5Error
  17. 8 class << self
  18. 8 def load_dependencies(*)
  19. 253 require_relative "../auth/socks5"
  20. end
  21. 8 def extra_options(options)
  22. 253 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[socks5])
  23. end
  24. end
  25. 8 module ConnectionMethods
  26. 8 def call
  27. 786 super
  28. 786 return unless @options.proxy && @options.proxy.uri.scheme == "socks5"
  29. 201 case @state
  30. when :connecting,
  31. :negotiating,
  32. :authenticating
  33. 77 consume
  34. end
  35. end
  36. 8 def connecting?
  37. 435310 super || @state == :authenticating || @state == :negotiating
  38. end
  39. 8 def interests
  40. 436445 if @state == :connecting || @state == :authenticating || @state == :negotiating
  41. 1477 return @write_buffer.empty? ? :r : :w
  42. end
  43. 434757 super
  44. end
  45. 8 private
  46. 8 def handle_transition(nextstate)
  47. 2372 return super unless @options.proxy && @options.proxy.uri.scheme == "socks5"
  48. 670 case nextstate
  49. when :connecting
  50. 216 return unless @state == :idle
  51. 216 @io.connect
  52. 216 return unless @io.connected?
  53. 108 @write_buffer << Packet.negotiate(@options.proxy)
  54. 108 __socks5_proxy_connect
  55. when :authenticating
  56. 36 return unless @state == :connecting
  57. 36 @write_buffer << Packet.authenticate(@options.proxy)
  58. when :negotiating
  59. 144 return unless @state == :connecting || @state == :authenticating
  60. 36 req = @pending.first
  61. 36 request_uri = req.uri
  62. 36 @write_buffer << Packet.connect(request_uri)
  63. when :connected
  64. 24 return unless @state == :negotiating
  65. 24 @parser = nil
  66. end
  67. 588 log(level: 1) { "SOCKS5: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
  68. 588 super
  69. end
  70. 8 def __socks5_proxy_connect
  71. 108 @parser = SocksParser.new(@write_buffer, @options)
  72. 108 @parser.on(:packet, &method(:__socks5_on_packet))
  73. 108 transition(:negotiating)
  74. end
  75. 8 def __socks5_on_packet(packet)
  76. 150 case @state
  77. when :connecting
  78. 108 version, method = packet.unpack("CC")
  79. 108 __socks5_check_version(version)
  80. 90 case method
  81. when PASSWD
  82. 36 transition(:authenticating)
  83. 6 nil
  84. when NONE
  85. 60 __on_socks5_error("no supported authorization methods")
  86. else
  87. 12 transition(:negotiating)
  88. end
  89. when :authenticating
  90. 36 _, status = packet.unpack("CC")
  91. 36 return transition(:negotiating) if status == SUCCESS
  92. 12 __on_socks5_error("socks authentication error: #{status}")
  93. when :negotiating
  94. 36 version, reply, = packet.unpack("CC")
  95. 36 __socks5_check_version(version)
  96. 36 __on_socks5_error("socks5 negotiation error: #{reply}") unless reply == SUCCESS
  97. 24 req = @pending.first
  98. 24 request_uri = req.uri
  99. 24 @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
  100. 24 transition(:connected)
  101. 24 throw(:called)
  102. end
  103. end
  104. 8 def __socks5_check_version(version)
  105. 144 __on_socks5_error("invalid SOCKS version (#{version})") if version != 5
  106. end
  107. 8 def __on_socks5_error(message)
  108. 84 ex = Error.new(message)
  109. 84 ex.set_backtrace(caller)
  110. 84 on_error(ex)
  111. 84 throw(:called)
  112. end
  113. end
  114. 8 class SocksParser
  115. 8 include HTTPX::Callbacks
  116. 8 def initialize(buffer, options)
  117. 108 @buffer = buffer
  118. 108 @options = Options.new(options)
  119. end
  120. 8 def close; end
  121. 8 def consume(*); end
  122. 8 def empty?
  123. 84 true
  124. end
  125. 8 def <<(packet)
  126. 180 emit(:packet, packet)
  127. end
  128. end
  129. 8 module Packet
  130. 8 module_function
  131. 8 def negotiate(parameters)
  132. 108 methods = [NOAUTH]
  133. 108 methods << PASSWD if parameters.can_authenticate?
  134. 108 methods.unshift(methods.size)
  135. 108 methods.unshift(VERSION)
  136. 108 methods.pack("C*")
  137. end
  138. 8 def authenticate(parameters)
  139. 36 parameters.authenticate
  140. end
  141. 8 def connect(uri)
  142. 36 packet = [VERSION, CONNECT, 0].pack("C*")
  143. begin
  144. 36 ip = IPAddr.new(uri.host)
  145. 12 ipcode = ip.ipv6? ? IPV6 : IPV4
  146. 12 packet << [ipcode].pack("C") << ip.hton
  147. rescue IPAddr::InvalidAddressError
  148. 24 packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
  149. end
  150. 36 packet << [uri.port].pack("n")
  151. 36 packet
  152. end
  153. end
  154. end
  155. end
  156. 8 register_plugin :"proxy/socks5", Proxy::Socks5
  157. end
  158. end

lib/httpx/plugins/proxy/ssh.rb

92.31% lines covered

52 relevant lines. 48 lines covered and 4 lines missed.
    
  1. # frozen_string_literal: true
  2. 5 require "httpx/plugins/proxy"
  3. 5 module HTTPX
  4. 5 module Plugins
  5. 5 module Proxy
  6. 5 module SSH
  7. 5 class << self
  8. 5 def load_dependencies(*)
  9. 10 require "net/ssh/gateway"
  10. end
  11. end
  12. 5 module OptionsMethods
  13. 5 def option_proxy(value)
  14. 20 Hash[value]
  15. end
  16. end
  17. 5 module InstanceMethods
  18. 5 def request(*args, **options)
  19. 10 raise ArgumentError, "must perform at least one request" if args.empty?
  20. 10 requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
  21. 10 request = requests.first or return super
  22. 10 request_options = request.options
  23. 10 return super unless request_options.proxy
  24. 10 ssh_options = request_options.proxy
  25. 10 ssh_uris = ssh_options.delete(:uri)
  26. 10 ssh_uri = URI.parse(ssh_uris.shift)
  27. 10 return super unless ssh_uri.scheme == "ssh"
  28. 10 ssh_username = ssh_options.delete(:username)
  29. 10 ssh_options[:port] ||= ssh_uri.port || 22
  30. 10 if request_options.debug
  31. ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
  32. end
  33. 10 request_uri = URI(requests.first.uri)
  34. 10 @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
  35. begin
  36. 10 @_gateway.open(request_uri.host, request_uri.port) do |local_port|
  37. 10 io = build_gateway_socket(local_port, request_uri, request_options)
  38. 10 super(*args, **options.merge(io: io))
  39. end
  40. ensure
  41. 10 @_gateway.shutdown!
  42. end
  43. end
  44. 5 private
  45. 5 def build_gateway_socket(port, request_uri, options)
  46. 10 case request_uri.scheme
  47. when "https"
  48. 5 ctx = OpenSSL::SSL::SSLContext.new
  49. 5 ctx_options = SSL::TLS_OPTIONS.merge(options.ssl)
  50. 5 ctx.set_params(ctx_options) unless ctx_options.empty?
  51. 5 sock = TCPSocket.open("localhost", port)
  52. 5 io = OpenSSL::SSL::SSLSocket.new(sock, ctx)
  53. 5 io.hostname = request_uri.host
  54. 5 io.sync_close = true
  55. 5 io.connect
  56. 5 io.post_connection_check(request_uri.host) if ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
  57. 5 io
  58. when "http"
  59. 5 TCPSocket.open("localhost", port)
  60. else
  61. raise TypeError, "unexpected scheme: #{request_uri.scheme}"
  62. end
  63. end
  64. end
  65. 5 module ConnectionMethods
  66. # should not coalesce connections here, as the IP is the IP of the proxy
  67. 5 def coalescable?(*)
  68. return super unless @options.proxy
  69. false
  70. end
  71. end
  72. end
  73. end
  74. 5 register_plugin :"proxy/ssh", Proxy::SSH
  75. end
  76. end

lib/httpx/plugins/push_promise.rb

100.0% lines covered

41 relevant lines. 41 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin adds support for HTTP/2 Push responses.
  6. #
  7. # In order to benefit from this, requests are sent one at a time, so that
  8. # no push responses are received after corresponding request has been sent.
  9. #
  10. # https://gitlab.com/os85/httpx/wikis/Server-Push
  11. #
  12. 6 module PushPromise
  13. 6 def self.extra_options(options)
  14. 12 options.merge(http2_settings: { settings_enable_push: 1 },
  15. max_concurrent_requests: 1)
  16. end
  17. 6 module ResponseMethods
  18. 6 def pushed?
  19. 12 @__pushed
  20. end
  21. 6 def mark_as_pushed!
  22. 6 @__pushed = true
  23. end
  24. end
  25. 6 module InstanceMethods
  26. 6 private
  27. 6 def promise_headers
  28. 12 @promise_headers ||= {}
  29. end
  30. 6 def on_promise(parser, stream)
  31. 12 stream.on(:promise_headers) do |h|
  32. 12 __on_promise_request(parser, stream, h)
  33. end
  34. 12 stream.on(:headers) do |h|
  35. 6 __on_promise_response(parser, stream, h)
  36. end
  37. end
  38. 6 def __on_promise_request(parser, stream, h)
  39. 12 log(level: 1, color: :yellow) do
  40. skipped # :nocov:
  41. skipped h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
  42. skipped # :nocov:
  43. end
  44. 12 headers = @options.headers_class.new(h)
  45. 12 path = headers[":path"]
  46. 12 authority = headers[":authority"]
  47. 18 request = parser.pending.find { |r| r.authority == authority && r.path == path }
  48. 12 if request
  49. 6 request.merge_headers(headers)
  50. 5 promise_headers[stream] = request
  51. 6 parser.pending.delete(request)
  52. 5 parser.streams[request] = stream
  53. 6 request.transition(:done)
  54. else
  55. 6 stream.refuse
  56. end
  57. end
  58. 6 def __on_promise_response(parser, stream, h)
  59. 6 request = promise_headers.delete(stream)
  60. 6 return unless request
  61. 6 parser.__send__(:on_stream_headers, stream, request, h)
  62. 6 response = request.response
  63. 6 response.mark_as_pushed!
  64. 6 stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
  65. 6 stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
  66. end
  67. end
  68. end
  69. 6 register_plugin(:push_promise, PushPromise)
  70. end
  71. end

lib/httpx/plugins/rate_limiter.rb

100.0% lines covered

16 relevant lines. 16 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when the request:
  6. #
  7. # * is rate limited;
  8. # * when the server is unavailable (503);
  9. # * when a 3xx request comes with a "retry-after" value
  10. #
  11. # https://gitlab.com/os85/httpx/wikis/Rate-Limiter
  12. #
  13. 6 module RateLimiter
  14. 6 class << self
  15. 6 RATE_LIMIT_CODES = [429, 503].freeze
  16. 6 def configure(klass)
  17. 48 klass.plugin(:retries,
  18. retry_change_requests: true,
  19. retry_on: method(:retry_on_rate_limited_response),
  20. retry_after: method(:retry_after_rate_limit))
  21. end
  22. 6 def retry_on_rate_limited_response(response)
  23. 96 return false unless response.is_a?(Response)
  24. 96 status = response.status
  25. 96 RATE_LIMIT_CODES.include?(status)
  26. end
  27. # Servers send the "Retry-After" header field to indicate how long the
  28. # user agent ought to wait before making a follow-up request. When
  29. # sent with a 503 (Service Unavailable) response, Retry-After indicates
  30. # how long the service is expected to be unavailable to the client.
  31. # When sent with any 3xx (Redirection) response, Retry-After indicates
  32. # the minimum time that the user agent is asked to wait before issuing
  33. # the redirected request.
  34. #
  35. 6 def retry_after_rate_limit(_, response)
  36. 48 retry_after = response.headers["retry-after"]
  37. 48 return unless retry_after
  38. 24 Utils.parse_retry_after(retry_after)
  39. end
  40. end
  41. end
  42. 6 register_plugin :rate_limiter, RateLimiter
  43. end
  44. end

lib/httpx/plugins/response_cache.rb

100.0% lines covered

80 relevant lines. 80 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when certain errors happen.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Response-Cache
  8. #
  9. 6 module ResponseCache
  10. 6 CACHEABLE_VERBS = %w[GET HEAD].freeze
  11. 6 CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
  12. 6 private_constant :CACHEABLE_VERBS
  13. 6 private_constant :CACHEABLE_STATUS_CODES
  14. 6 class << self
  15. 6 def load_dependencies(*)
  16. 132 require_relative "response_cache/store"
  17. end
  18. 6 def cacheable_request?(request)
  19. 186 CACHEABLE_VERBS.include?(request.verb) &&
  20. (
  21. 186 !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
  22. )
  23. end
  24. 6 def cacheable_response?(response)
  25. 126 response.is_a?(Response) &&
  26. (
  27. 126 response.cache_control.nil? ||
  28. # TODO: !response.cache_control.include?("private") && is shared cache
  29. !response.cache_control.include?("no-store")
  30. ) &&
  31. CACHEABLE_STATUS_CODES.include?(response.status) &&
  32. # RFC 2616 13.4 - A response received with a status code of 200, 203, 206, 300, 301 or
  33. # 410 MAY be stored by a cache and used in reply to a subsequent
  34. # request, subject to the expiration mechanism, unless a cache-control
  35. # directive prohibits caching. However, a cache that does not support
  36. # the Range and Content-Range headers MUST NOT cache 206 (Partial
  37. # Content) responses.
  38. response.status != 206 && (
  39. 114 response.headers.key?("etag") || response.headers.key?("last-modified-at") || response.fresh?
  40. )
  41. end
  42. 6 def cached_response?(response)
  43. 60 response.is_a?(Response) && response.status == 304
  44. end
  45. 6 def extra_options(options)
  46. 132 options.merge(response_cache_store: Store.new)
  47. end
  48. end
  49. 6 module OptionsMethods
  50. 6 def option_response_cache_store(value)
  51. 132 raise TypeError, "must be an instance of #{Store}" unless value.is_a?(Store)
  52. 132 value
  53. end
  54. end
  55. 6 module InstanceMethods
  56. 6 def clear_response_cache
  57. 12 @options.response_cache_store.clear
  58. end
  59. 6 def build_request(*)
  60. 60 request = super
  61. 60 return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request)
  62. 24 @options.response_cache_store.prepare(request)
  63. 24 request
  64. end
  65. 6 def fetch_response(request, *)
  66. 195 response = super
  67. 195 return unless response
  68. 60 if ResponseCache.cached_response?(response)
  69. 24 log { "returning cached response for #{request.uri}" }
  70. 24 cached_response = @options.response_cache_store.lookup(request)
  71. 24 response.copy_from_cached(cached_response)
  72. else
  73. 36 @options.response_cache_store.cache(request, response)
  74. end
  75. 60 response
  76. end
  77. end
  78. 6 module RequestMethods
  79. 6 def response_cache_key
  80. 384 @response_cache_key ||= Digest::SHA1.hexdigest("httpx-response-cache-#{@verb}-#{@uri}")
  81. end
  82. end
  83. 6 module ResponseMethods
  84. 6 def copy_from_cached(other)
  85. 24 @body = other.body.dup
  86. 24 @body.rewind
  87. end
  88. # A response is fresh if its age has not yet exceeded its freshness lifetime.
  89. 6 def fresh?
  90. 192 if cache_control
  91. 30 return false if cache_control.include?("no-cache")
  92. # check age: max-age
  93. 36 max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
  94. 36 max_age ||= cache_control.find { |directive| directive.start_with?("max-age") }
  95. 18 max_age = max_age[/age=(\d+)/, 1] if max_age
  96. 18 max_age = max_age.to_i if max_age
  97. 18 return max_age > age if max_age
  98. end
  99. # check age: expires
  100. 162 if @headers.key?("expires")
  101. begin
  102. 18 expires = Time.httpdate(@headers["expires"])
  103. rescue ArgumentError
  104. 6 return true
  105. end
  106. 10 return (expires - Time.now).to_i.positive?
  107. end
  108. 144 true
  109. end
  110. 6 def cache_control
  111. 414 return @cache_control if defined?(@cache_control)
  112. 48 @cache_control = begin
  113. 288 return unless @headers.key?("cache-control")
  114. 30 @headers["cache-control"].split(/ *, */)
  115. end
  116. end
  117. 6 def vary
  118. 210 return @vary if defined?(@vary)
  119. 28 @vary = begin
  120. 168 return unless @headers.key?("vary")
  121. 12 @headers["vary"].split(/ *, */)
  122. end
  123. end
  124. 6 private
  125. 6 def age
  126. 18 return @headers["age"].to_i if @headers.key?("age")
  127. 18 (Time.now - date).to_i
  128. end
  129. 6 def date
  130. 18 @date ||= Time.httpdate(@headers["date"])
  131. rescue NoMethodError, ArgumentError
  132. 6 Time.now
  133. end
  134. end
  135. end
  136. 6 register_plugin :response_cache, ResponseCache
  137. end
  138. end

lib/httpx/plugins/response_cache/store.rb

100.0% lines covered

47 relevant lines. 47 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX::Plugins
  3. 6 module ResponseCache
  4. 6 class Store
  5. 6 def initialize
  6. 186 @store = {}
  7. 186 @store_mutex = Thread::Mutex.new
  8. end
  9. 6 def clear
  10. 24 @store_mutex.synchronize { @store.clear }
  11. end
  12. 6 def lookup(request)
  13. 234 responses = _get(request)
  14. 234 return unless responses
  15. 180 responses.find(&method(:match_by_vary?).curry(2)[request])
  16. end
  17. 6 def cached?(request)
  18. 84 lookup(request)
  19. end
  20. 6 def cache(request, response)
  21. 126 return unless ResponseCache.cacheable_request?(request) && ResponseCache.cacheable_response?(response)
  22. 114 _set(request, response)
  23. end
  24. 6 def prepare(request)
  25. 60 cached_response = lookup(request)
  26. 60 return unless cached_response
  27. 42 return unless match_by_vary?(request, cached_response)
  28. 42 if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
  29. 24 request.headers.add("if-modified-since", last_modified)
  30. end
  31. 42 if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
  32. 42 request.headers.add("if-none-match", etag)
  33. end
  34. end
  35. 6 private
  36. 6 def match_by_vary?(request, response)
  37. 210 vary = response.vary
  38. 210 return true unless vary
  39. 54 original_request = response.instance_variable_get(:@request)
  40. 54 return request.headers.same_headers?(original_request.headers) if vary == %w[*]
  41. 30 vary.all? do |cache_field|
  42. 30 cache_field.downcase!
  43. 30 !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
  44. end
  45. end
  46. 6 def _get(request)
  47. 234 @store_mutex.synchronize do
  48. 234 responses = @store[request.response_cache_key]
  49. 234 return unless responses
  50. 180 responses.select! do |res|
  51. 180 !res.body.closed? && res.fresh?
  52. end
  53. 180 responses
  54. end
  55. end
  56. 6 def _set(request, response)
  57. 114 @store_mutex.synchronize do
  58. 114 responses = (@store[request.response_cache_key] ||= [])
  59. 114 responses.reject! do |res|
  60. 12 res.body.closed? || !res.fresh? || match_by_vary?(request, res)
  61. end
  62. 114 responses << response
  63. end
  64. end
  65. end
  66. end
  67. end

lib/httpx/plugins/retries.rb

96.77% lines covered

93 relevant lines. 90 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 13 module HTTPX
  3. 13 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when certain errors happen.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Retries
  8. #
  9. 13 module Retries
  10. 13 MAX_RETRIES = 3
  11. # TODO: pass max_retries in a configure/load block
  12. 13 IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
  13. 1 RETRYABLE_ERRORS = [
  14. 12 IOError,
  15. EOFError,
  16. Errno::ECONNRESET,
  17. Errno::ECONNABORTED,
  18. Errno::EPIPE,
  19. Errno::EINVAL,
  20. Errno::ETIMEDOUT,
  21. Parser::Error,
  22. TLSError,
  23. TimeoutError,
  24. ConnectionError,
  25. Connection::HTTP2::GoawayError,
  26. ].freeze
  27. 13 DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
  28. 13 if ENV.key?("HTTPX_NO_JITTER")
  29. 13 def self.extra_options(options)
  30. 511 options.merge(max_retries: MAX_RETRIES)
  31. end
  32. else
  33. def self.extra_options(options)
  34. options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
  35. end
  36. end
  37. 13 module OptionsMethods
  38. 13 def option_retry_after(value)
  39. # return early if callable
  40. 156 unless value.respond_to?(:call)
  41. 72 value = Float(value)
  42. 72 raise TypeError, ":retry_after must be positive" unless value.positive?
  43. end
  44. 156 value
  45. end
  46. 13 def option_retry_jitter(value)
  47. # return early if callable
  48. 36 raise TypeError, ":retry_jitter must be callable" unless value.respond_to?(:call)
  49. 36 value
  50. end
  51. 13 def option_max_retries(value)
  52. 1607 num = Integer(value)
  53. 1607 raise TypeError, ":max_retries must be positive" unless num >= 0
  54. 1607 num
  55. end
  56. 13 def option_retry_change_requests(v)
  57. 887 v
  58. end
  59. 13 def option_retry_on(value)
  60. 149 raise TypeError, ":retry_on must be called with the response" unless value.respond_to?(:call)
  61. 149 value
  62. end
  63. end
  64. 13 module InstanceMethods
  65. 13 def max_retries(n)
  66. 72 with(max_retries: n.to_i)
  67. end
  68. 13 private
  69. 13 def fetch_response(request, connections, options)
  70. 3237358 response = super
  71. 3237358 if response &&
  72. request.retries.positive? &&
  73. __repeatable_request?(request, options) &&
  74. (
  75. 94 (
  76. 217 response.is_a?(ErrorResponse) && __retryable_error?(response.error)
  77. ) ||
  78. (
  79. 155 options.retry_on && options.retry_on.call(response)
  80. )
  81. )
  82. 332 __try_partial_retry(request, response)
  83. 332 log { "failed to get response, #{request.retries} tries to go..." }
  84. 332 request.retries -= 1
  85. 332 request.transition(:idle)
  86. 332 retry_after = options.retry_after
  87. 332 retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
  88. 332 if retry_after
  89. # apply jitter
  90. 72 if (jitter = request.options.retry_jitter)
  91. 12 retry_after = jitter.call(retry_after)
  92. end
  93. 72 retry_start = Utils.now
  94. 72 log { "retrying after #{retry_after} secs..." }
  95. 72 pool.after(retry_after) do
  96. 72 log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
  97. 72 send_request(request, connections, options)
  98. end
  99. else
  100. 260 send_request(request, connections, options)
  101. end
  102. 280 return
  103. end
  104. 3237026 response
  105. end
  106. 13 def __repeatable_request?(request, options)
  107. 709 IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
  108. end
  109. 13 def __retryable_error?(ex)
  110. 2513 RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
  111. end
  112. 13 def proxy_error?(request, response)
  113. 48 super && !request.retries.positive?
  114. end
  115. #
  116. # Atttempt to set the request to perform a partial range request.
  117. # This happens if the peer server accepts byte-range requests, and
  118. # the last response contains some body payload.
  119. #
  120. 13 def __try_partial_retry(request, response)
  121. 332 response = response.response if response.is_a?(ErrorResponse)
  122. 332 return unless response
  123. 141 unless response.headers.key?("accept-ranges") &&
  124. response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
  125. 12 (original_body = response.body)
  126. 129 response.close if response.respond_to?(:close)
  127. 109 return
  128. end
  129. 12 request.partial_response = response
  130. 12 size = original_body.bytesize
  131. 10 request.headers["range"] = "bytes=#{size}-"
  132. end
  133. end
  134. 13 module RequestMethods
  135. 13 attr_accessor :retries
  136. 13 attr_writer :partial_response
  137. 13 def initialize(*args)
  138. 526 super
  139. 526 @retries = @options.max_retries
  140. end
  141. 13 def response=(response)
  142. 870 if @partial_response
  143. 12 if response.is_a?(Response) && response.status == 206
  144. 12 response.from_partial_response(@partial_response)
  145. else
  146. @partial_response.close
  147. end
  148. 12 @partial_response = nil
  149. end
  150. 870 super
  151. end
  152. end
  153. 13 module ResponseMethods
  154. 13 def from_partial_response(response)
  155. 12 @status = response.status
  156. 12 @headers = response.headers
  157. 12 @body = response.body
  158. end
  159. end
  160. end
  161. 13 register_plugin :retries, Retries
  162. end
  163. end

lib/httpx/plugins/ssrf_filter.rb

100.0% lines covered

61 relevant lines. 61 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 class ServerSideRequestForgeryError < Error; end
  4. 6 module Plugins
  5. #
  6. # This plugin adds support for preventing Server-Side Request Forgery attacks.
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Server-Side-Request-Forgery-Filter
  9. #
  10. 6 module SsrfFilter
  11. 6 module IPAddrExtensions
  12. 6 refine IPAddr do
  13. 6 def prefixlen
  14. 96 mask_addr = @mask_addr
  15. 96 raise "Invalid mask" if mask_addr.zero?
  16. 355 mask_addr >>= 1 while (mask_addr & 0x1).zero?
  17. 96 length = 0
  18. 349 while mask_addr & 0x1 == 0x1
  19. 1265 length += 1
  20. 1265 mask_addr >>= 1
  21. end
  22. 96 length
  23. end
  24. end
  25. end
  26. 6 using IPAddrExtensions
  27. # https://en.wikipedia.org/wiki/Reserved_IP_addresses
  28. 1 IPV4_BLACKLIST = [
  29. 5 IPAddr.new("0.0.0.0/8"), # Current network (only valid as source address)
  30. IPAddr.new("10.0.0.0/8"), # Private network
  31. IPAddr.new("100.64.0.0/10"), # Shared Address Space
  32. IPAddr.new("127.0.0.0/8"), # Loopback
  33. IPAddr.new("169.254.0.0/16"), # Link-local
  34. IPAddr.new("172.16.0.0/12"), # Private network
  35. IPAddr.new("192.0.0.0/24"), # IETF Protocol Assignments
  36. IPAddr.new("192.0.2.0/24"), # TEST-NET-1, documentation and examples
  37. IPAddr.new("192.88.99.0/24"), # IPv6 to IPv4 relay (includes 2002::/16)
  38. IPAddr.new("192.168.0.0/16"), # Private network
  39. IPAddr.new("198.18.0.0/15"), # Network benchmark tests
  40. IPAddr.new("198.51.100.0/24"), # TEST-NET-2, documentation and examples
  41. IPAddr.new("203.0.113.0/24"), # TEST-NET-3, documentation and examples
  42. IPAddr.new("224.0.0.0/4"), # IP multicast (former Class D network)
  43. IPAddr.new("240.0.0.0/4"), # Reserved (former Class E network)
  44. IPAddr.new("255.255.255.255"), # Broadcast
  45. ].freeze
  46. 1 IPV6_BLACKLIST = ([
  47. 5 IPAddr.new("::1/128"), # Loopback
  48. IPAddr.new("64:ff9b::/96"), # IPv4/IPv6 translation (RFC 6052)
  49. IPAddr.new("100::/64"), # Discard prefix (RFC 6666)
  50. IPAddr.new("2001::/32"), # Teredo tunneling
  51. IPAddr.new("2001:10::/28"), # Deprecated (previously ORCHID)
  52. IPAddr.new("2001:20::/28"), # ORCHIDv2
  53. IPAddr.new("2001:db8::/32"), # Addresses used in documentation and example source code
  54. IPAddr.new("2002::/16"), # 6to4
  55. IPAddr.new("fc00::/7"), # Unique local address
  56. IPAddr.new("fe80::/10"), # Link-local address
  57. IPAddr.new("ff00::/8"), # Multicast
  58. ] + IPV4_BLACKLIST.flat_map do |ipaddr|
  59. 96 prefixlen = ipaddr.prefixlen
  60. 96 ipv4_compatible = ipaddr.ipv4_compat.mask(96 + prefixlen)
  61. 96 ipv4_mapped = ipaddr.ipv4_mapped.mask(80 + prefixlen)
  62. 96 [ipv4_compatible, ipv4_mapped]
  63. end).freeze
  64. 6 class << self
  65. 6 def extra_options(options)
  66. 54 options.merge(allowed_schemes: %w[https http])
  67. end
  68. 6 def unsafe_ip_address?(ipaddr)
  69. 96 range = ipaddr.to_range
  70. 96 return true if range.first != range.last
  71. 108 return IPV6_BLACKLIST.any? { |r| r.include?(ipaddr) } if ipaddr.ipv6?
  72. 1140 IPV4_BLACKLIST.any? { |r| r.include?(ipaddr) } # then it's IPv4
  73. end
  74. end
  75. 6 module OptionsMethods
  76. 6 def option_allowed_schemes(value)
  77. 60 Array(value)
  78. end
  79. end
  80. 6 module InstanceMethods
  81. 6 def send_requests(*requests)
  82. 66 responses = requests.map do |request|
  83. 66 next if @options.allowed_schemes.include?(request.uri.scheme)
  84. 6 error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
  85. 6 error.set_backtrace(caller)
  86. 6 response = ErrorResponse.new(request, error, request.options)
  87. 6 request.emit(:response, response)
  88. 6 response
  89. end
  90. 132 allowed_requests = requests.select { |req| responses[requests.index(req)].nil? }
  91. 66 allowed_responses = super(*allowed_requests)
  92. 66 allowed_responses.each_with_index do |res, idx|
  93. 60 req = allowed_requests[idx]
  94. 50 responses[requests.index(req)] = res
  95. end
  96. 66 responses
  97. end
  98. end
  99. 6 module ConnectionMethods
  100. 6 def initialize(*)
  101. begin
  102. 60 super
  103. 8 rescue ServerSideRequestForgeryError => e
  104. # may raise when IPs are passed as options via :addresses
  105. 12 throw(:resolve_error, e)
  106. end
  107. end
  108. 6 def addresses=(addrs)
  109. 156 addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
  110. 60 addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
  111. 60 raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
  112. 12 super
  113. end
  114. end
  115. end
  116. 6 register_plugin :ssrf_filter, SsrfFilter
  117. end
  118. end

lib/httpx/plugins/stream.rb

100.0% lines covered

70 relevant lines. 70 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 11 module HTTPX
  3. 11 class StreamResponse
  4. 11 def initialize(request, session)
  5. 119 @request = request
  6. 119 @session = session
  7. 119 @response = nil
  8. end
  9. 11 def each(&block)
  10. 143 return enum_for(__method__) unless block
  11. 107 @request.stream = self
  12. begin
  13. 107 @on_chunk = block
  14. 107 response.raise_for_status
  15. ensure
  16. 107 response.close if @response
  17. 107 @on_chunk = nil
  18. end
  19. end
  20. 11 def each_line
  21. 82 return enum_for(__method__) unless block_given?
  22. 41 line = "".b
  23. 41 each do |chunk|
  24. 34 line << chunk
  25. 104 while (idx = line.index("\n"))
  26. 41 yield line.byteslice(0..idx - 1)
  27. 41 line = line.byteslice(idx + 1..-1)
  28. end
  29. end
  30. 17 yield line unless line.empty?
  31. end
  32. # This is a ghost method. It's to be used ONLY internally, when processing streams
  33. 11 def on_chunk(chunk)
  34. 161 raise NoMethodError unless @on_chunk
  35. 161 @on_chunk.call(chunk)
  36. end
  37. skipped # :nocov:
  38. skipped def inspect
  39. skipped "#<StreamResponse:#{object_id}>"
  40. skipped end
  41. skipped # :nocov:
  42. 11 def to_s
  43. 12 response.to_s
  44. end
  45. 11 private
  46. 11 def response
  47. 476 return @response if @response
  48. 149 @request.response || begin
  49. 119 @response = @session.request(@request)
  50. end
  51. end
  52. 11 def respond_to_missing?(meth, *args)
  53. 12 response.respond_to?(meth, *args) || super
  54. end
  55. 11 def method_missing(meth, *args, &block)
  56. 119 return super unless response.respond_to?(meth)
  57. 119 response.__send__(meth, *args, &block)
  58. end
  59. end
  60. 11 module Plugins
  61. #
  62. # This plugin adds support for stream response (text/event-stream).
  63. #
  64. # https://gitlab.com/os85/httpx/wikis/Stream
  65. #
  66. 11 module Stream
  67. 11 def self.extra_options(options)
  68. 226 options.merge(timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 })
  69. end
  70. 11 module InstanceMethods
  71. 11 def request(*args, stream: false, **options)
  72. 347 return super(*args, **options) unless stream
  73. 131 requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
  74. 131 raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
  75. 119 request = requests.first
  76. 119 StreamResponse.new(request, self)
  77. end
  78. end
  79. 11 module RequestMethods
  80. 11 attr_accessor :stream
  81. end
  82. 11 module ResponseMethods
  83. 11 def stream
  84. 211 request = @request.root_request if @request.respond_to?(:root_request)
  85. 211 request ||= @request
  86. 211 request.stream
  87. end
  88. end
  89. 11 module ResponseBodyMethods
  90. 11 def initialize(*)
  91. 211 super
  92. 211 @stream = @response.stream
  93. end
  94. 11 def write(chunk)
  95. 266 return super unless @stream
  96. 183 return 0 if chunk.empty?
  97. 161 chunk = decode_chunk(chunk)
  98. 161 @stream.on_chunk(chunk.dup)
  99. 161 chunk.size
  100. end
  101. 11 private
  102. 11 def transition(*)
  103. 198 return if @stream
  104. 103 super
  105. end
  106. end
  107. end
  108. 11 register_plugin :stream, Stream
  109. end
  110. end

lib/httpx/plugins/upgrade.rb

100.0% lines covered

37 relevant lines. 37 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin helps negotiating a new protocol from an HTTP/1.1 connection, via the
  6. # Upgrade header.
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Upgrade
  9. #
  10. 6 module Upgrade
  11. 6 class << self
  12. 6 def configure(klass)
  13. 24 klass.plugin(:"upgrade/h2")
  14. end
  15. 6 def extra_options(options)
  16. 24 options.merge(upgrade_handlers: {})
  17. end
  18. end
  19. 6 module OptionsMethods
  20. 6 def option_upgrade_handlers(value)
  21. 66 raise TypeError, ":upgrade_handlers must be a Hash" unless value.is_a?(Hash)
  22. 66 value
  23. end
  24. end
  25. 6 module InstanceMethods
  26. 6 def fetch_response(request, connections, options)
  27. 203 response = super
  28. 203 if response
  29. 66 return response unless response.is_a?(Response)
  30. 66 return response unless response.headers.key?("upgrade")
  31. 24 upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
  32. 24 return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
  33. 24 protocol_handler = options.upgrade_handlers[upgrade_protocol]
  34. 24 return response unless protocol_handler
  35. 24 log { "upgrading to #{upgrade_protocol}..." }
  36. 24 connection = find_connection(request, connections, options)
  37. # do not upgrade already upgraded connections
  38. 24 return if connection.upgrade_protocol == upgrade_protocol
  39. 24 protocol_handler.call(connection, request, response)
  40. # keep in the loop if the server is switching, unless
  41. # the connection has been hijacked, in which case you want
  42. # to terminante immediately
  43. 24 return if response.status == 101 && !connection.hijacked
  44. end
  45. 149 response
  46. end
  47. 6 def close(*args)
  48. 30 return super if args.empty?
  49. 18 connections, = args
  50. 18 pool.close(connections.reject(&:hijacked))
  51. end
  52. end
  53. 6 module ConnectionMethods
  54. 6 attr_reader :upgrade_protocol, :hijacked
  55. 6 def hijack_io
  56. 6 @hijacked = true
  57. end
  58. end
  59. end
  60. 6 register_plugin(:upgrade, Upgrade)
  61. end
  62. end

lib/httpx/plugins/upgrade/h2.rb

91.67% lines covered

24 relevant lines. 22 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin adds support for upgrading an HTTP/1.1 connection to HTTP/2
  6. # via an Upgrade: h2 response declaration
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2
  9. #
  10. 6 module H2
  11. 6 class << self
  12. 6 def extra_options(options)
  13. 24 options.merge(upgrade_handlers: options.upgrade_handlers.merge("h2" => self))
  14. end
  15. 6 def call(connection, _request, _response)
  16. 6 connection.upgrade_to_h2
  17. end
  18. end
  19. 6 module ConnectionMethods
  20. 6 using URIExtensions
  21. 6 def upgrade_to_h2
  22. 6 prev_parser = @parser
  23. 6 if prev_parser
  24. 6 prev_parser.reset
  25. 5 @inflight -= prev_parser.requests.size
  26. end
  27. 6 @parser = Connection::HTTP2.new(@write_buffer, @options)
  28. 6 set_parser_callbacks(@parser)
  29. 6 @upgrade_protocol = "h2"
  30. # what's happening here:
  31. # a deviation from the state machine is done to perform the actions when a
  32. # connection is closed, without transitioning, so the connection is kept in the pool.
  33. # the state is reset to initial, so that the socket reconnect works out of the box,
  34. # while the parser is already here.
  35. 6 purge_after_closed
  36. 6 transition(:idle)
  37. 6 prev_parser.requests.each do |req|
  38. req.transition(:idle)
  39. send(req)
  40. end
  41. end
  42. end
  43. end
  44. 6 register_plugin(:"upgrade/h2", H2)
  45. end
  46. end

lib/httpx/plugins/webdav.rb

100.0% lines covered

35 relevant lines. 35 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. #
  5. # This plugin implements convenience methods for performing WEBDAV requests.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/WebDav
  8. #
  9. 6 module WebDav
  10. 6 module InstanceMethods
  11. 6 def copy(src, dest)
  12. 12 request("COPY", src, headers: { "destination" => @options.origin.merge(dest) })
  13. end
  14. 6 def move(src, dest)
  15. 12 request("MOVE", src, headers: { "destination" => @options.origin.merge(dest) })
  16. end
  17. 6 def lock(path, timeout: nil, &blk)
  18. 36 headers = {}
  19. 30 headers["timeout"] = if timeout && timeout.positive?
  20. 12 "Second-#{timeout}"
  21. else
  22. 24 "Infinite, Second-4100000000"
  23. end
  24. 36 xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" \
  25. "<D:lockinfo xmlns:D=\"DAV:\">" \
  26. "<D:lockscope><D:exclusive/></D:lockscope>" \
  27. "<D:locktype><D:write/></D:locktype>" \
  28. "<D:owner>null</D:owner>" \
  29. "</D:lockinfo>"
  30. 36 response = request("LOCK", path, headers: headers, xml: xml)
  31. 36 return response unless response.is_a?(Response)
  32. 36 return response unless blk && response.status == 200
  33. 12 lock_token = response.headers["lock-token"]
  34. begin
  35. 12 blk.call(response)
  36. ensure
  37. 12 unlock(path, lock_token)
  38. end
  39. end
  40. 6 def unlock(path, lock_token)
  41. 24 request("UNLOCK", path, headers: { "lock-token" => lock_token })
  42. end
  43. 6 def mkcol(dir)
  44. 12 request("MKCOL", dir)
  45. end
  46. 6 def propfind(path, xml = nil)
  47. 48 body = case xml
  48. when :acl
  49. 10 '<?xml version="1.0" encoding="utf-8" ?><D:propfind xmlns:D="DAV:"><D:prop><D:owner/>' \
  50. "<D:supported-privilege-set/><D:current-user-privilege-set/><D:acl/></D:prop></D:propfind>"
  51. when nil
  52. 24 '<?xml version="1.0" encoding="utf-8"?><DAV:propfind xmlns:DAV="DAV:"><DAV:allprop/></DAV:propfind>'
  53. else
  54. 12 xml
  55. end
  56. 48 request("PROPFIND", path, headers: { "depth" => "1" }, xml: body)
  57. end
  58. 6 def proppatch(path, xml)
  59. 2 body = "<?xml version=\"1.0\"?>" \
  60. 12 "<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"
  61. 12 request("PROPPATCH", path, xml: body)
  62. end
  63. # %i[ orderpatch acl report search]
  64. end
  65. end
  66. 6 register_plugin(:webdav, WebDav)
  67. end
  68. end

lib/httpx/pmatch_extensions.rb

100.0% lines covered

17 relevant lines. 17 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 module ResponsePatternMatchExtensions
  4. 23 def deconstruct
  5. 31 [@status, @headers, @body]
  6. end
  7. 23 def deconstruct_keys(_keys)
  8. 50 { status: @status, headers: @headers, body: @body }
  9. end
  10. end
  11. 23 module ErrorResponsePatternMatchExtensions
  12. 23 def deconstruct
  13. 9 [@error]
  14. end
  15. 23 def deconstruct_keys(_keys)
  16. 25 { error: @error }
  17. end
  18. end
  19. 23 module HeadersPatternMatchExtensions
  20. 23 def deconstruct
  21. 5 to_a
  22. end
  23. end
  24. 23 Headers.include HeadersPatternMatchExtensions
  25. 23 Response.include ResponsePatternMatchExtensions
  26. 23 ErrorResponse.include ErrorResponsePatternMatchExtensions
  27. end

lib/httpx/pool.rb

82.14% lines covered

168 relevant lines. 138 lines covered and 30 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "forwardable"
  3. 23 require "httpx/selector"
  4. 23 require "httpx/connection"
  5. 23 require "httpx/resolver"
  6. 23 module HTTPX
  7. 23 class Pool
  8. 23 using ArrayExtensions::FilterMap
  9. 23 extend Forwardable
  10. 23 def_delegator :@timers, :after
  11. 23 def initialize
  12. 500 @resolvers = {}
  13. 500 @timers = Timers.new
  14. 500 @selector = Selector.new
  15. 500 @connections = []
  16. end
  17. 23 def wrap
  18. 377 connections = @connections
  19. 377 @connections = []
  20. begin
  21. 377 yield self
  22. ensure
  23. 377 @connections.unshift(*connections)
  24. end
  25. end
  26. 23 def empty?
  27. 763 @connections.empty?
  28. end
  29. 23 def next_tick
  30. 3691913 catch(:jump_tick) do
  31. 3691913 timeout = next_timeout
  32. 3691913 if timeout && timeout.negative?
  33. @timers.fire
  34. throw(:jump_tick)
  35. end
  36. begin
  37. 3691913 @selector.select(timeout, &:call)
  38. 3691809 @timers.fire
  39. rescue TimeoutError => e
  40. 5 @timers.fire(e)
  41. end
  42. end
  43. rescue StandardError => e
  44. 21 @connections.each do |connection|
  45. 38 connection.emit(:error, e)
  46. end
  47. rescue Exception # rubocop:disable Lint/RescueException
  48. 78 @connections.each(&:force_reset)
  49. 72 raise
  50. end
  51. 23 def close(connections = @connections)
  52. 4814 return if connections.empty?
  53. 4684 connections = connections.reject(&:inflight?)
  54. 4684 connections.each(&:terminate)
  55. 9552 next_tick until connections.none? { |c| c.state != :idle && @connections.include?(c) }
  56. # close resolvers
  57. 4678 outstanding_connections = @connections
  58. 4678 resolver_connections = @resolvers.each_value.flat_map(&:connections).compact
  59. 3969 outstanding_connections -= resolver_connections
  60. 4678 return unless outstanding_connections.empty?
  61. 1983 @resolvers.each_value do |resolver|
  62. 1849 resolver.close unless resolver.closed?
  63. end
  64. # for https resolver
  65. 1983 resolver_connections.each(&:terminate)
  66. 2018 next_tick until resolver_connections.none? { |c| c.state != :idle && @connections.include?(c) }
  67. end
  68. 23 def init_connection(connection, _options)
  69. 5105 connection.timers = @timers
  70. 5105 connection.on(:activate) do
  71. 460 select_connection(connection)
  72. end
  73. 5105 connection.on(:exhausted) do
  74. 5 case connection.state
  75. when :closed
  76. 6 connection.idling
  77. 6 @connections << connection
  78. 6 select_connection(connection)
  79. when :closing
  80. connection.once(:close) do
  81. connection.idling
  82. @connections << connection
  83. select_connection(connection)
  84. end
  85. end
  86. end
  87. 5105 connection.on(:close) do
  88. 5718 unregister_connection(connection)
  89. end
  90. 5105 connection.on(:terminate) do
  91. 4556 unregister_connection(connection, true)
  92. end
  93. 5105 resolve_connection(connection) unless connection.family
  94. end
  95. 23 def deactivate(connections)
  96. 779 connections.each do |connection|
  97. 809 connection.deactivate
  98. 809 deselect_connection(connection) if connection.state == :inactive
  99. end
  100. end
  101. # opens a connection to the IP reachable through +uri+.
  102. # Many hostnames are reachable through the same IP, so we try to
  103. # maximize pipelining by opening as few connections as possible.
  104. #
  105. 23 def find_connection(uri, options)
  106. 6487 conn = @connections.find do |connection|
  107. 12159 connection.match?(uri, options)
  108. end
  109. 6487 return unless conn
  110. 1245 case conn.state
  111. when :closed
  112. 336 conn.idling
  113. 336 select_connection(conn)
  114. when :closing
  115. conn.once(:close) do
  116. conn.idling
  117. select_connection(conn)
  118. end
  119. end
  120. 1364 conn
  121. end
  122. 23 private
  123. 23 def resolve_connection(connection)
  124. 5110 @connections << connection unless @connections.include?(connection)
  125. 5110 if connection.addresses || connection.open?
  126. #
  127. # there are two cases in which we want to activate initialization of
  128. # connection immediately:
  129. #
  130. # 1. when the connection already has addresses, i.e. it doesn't need to
  131. # resolve a name (not the same as name being an IP, yet)
  132. # 2. when the connection is initialized with an external already open IO.
  133. #
  134. 170 connection.once(:connect_error, &connection.method(:handle_error))
  135. 170 on_resolver_connection(connection)
  136. 161 return
  137. end
  138. 4940 find_resolver_for(connection) do |resolver|
  139. 370 resolver << try_clone_connection(connection, resolver.family)
  140. 360 next if resolver.empty?
  141. 290 select_connection(resolver)
  142. end
  143. end
  144. 23 def try_clone_connection(connection, family)
  145. 370 connection.family ||= family
  146. 370 return connection if connection.family == family
  147. new_connection = connection.class.new(connection.origin, connection.options)
  148. new_connection.family = family
  149. connection.once(:tcp_open) { new_connection.force_reset }
  150. connection.once(:connect_error) do |err|
  151. if new_connection.connecting?
  152. new_connection.merge(connection)
  153. connection.emit(:cloned, new_connection)
  154. connection.force_reset
  155. else
  156. connection.__send__(:handle_error, err)
  157. end
  158. end
  159. new_connection.once(:tcp_open) do |new_conn|
  160. if new_conn != connection
  161. new_conn.merge(connection)
  162. connection.force_reset
  163. end
  164. end
  165. new_connection.once(:connect_error) do |err|
  166. if connection.connecting?
  167. # main connection has the requests
  168. connection.merge(new_connection)
  169. new_connection.emit(:cloned, connection)
  170. new_connection.force_reset
  171. else
  172. new_connection.__send__(:handle_error, err)
  173. end
  174. end
  175. init_connection(new_connection, connection.options)
  176. new_connection
  177. end
  178. 23 def on_resolver_connection(connection)
  179. 5102 @connections << connection unless @connections.include?(connection)
  180. 5102 found_connection = @connections.find do |ch|
  181. 14412 ch != connection && ch.mergeable?(connection)
  182. end
  183. 5102 return register_connection(connection) unless found_connection
  184. 23 if found_connection.open?
  185. 22 coalesce_connections(found_connection, connection)
  186. 22 throw(:coalesced, found_connection) unless @connections.include?(connection)
  187. else
  188. 1 found_connection.once(:open) do
  189. 1 coalesce_connections(found_connection, connection)
  190. end
  191. end
  192. end
  193. 23 def on_resolver_error(connection, error)
  194. 151 return connection.emit(:connect_error, error) if connection.connecting? && connection.callbacks_for?(:connect_error)
  195. 151 connection.emit(:error, error)
  196. end
  197. 23 def on_resolver_close(resolver)
  198. 179 resolver_type = resolver.class
  199. 179 return if resolver.closed?
  200. 179 @resolvers.delete(resolver_type)
  201. 179 deselect_connection(resolver)
  202. 179 resolver.close unless resolver.closed?
  203. end
  204. 23 def register_connection(connection)
  205. 5091 select_connection(connection)
  206. end
  207. 23 def unregister_connection(connection, cleanup = !connection.used?)
  208. 10279 @connections.delete(connection) if cleanup
  209. 10279 deselect_connection(connection)
  210. end
  211. 23 def select_connection(connection)
  212. 6183 @selector.register(connection)
  213. end
  214. 23 def deselect_connection(connection)
  215. 10955 @selector.deregister(connection)
  216. end
  217. 23 def coalesce_connections(conn1, conn2)
  218. 23 return register_connection(conn2) unless conn1.coalescable?(conn2)
  219. 11 conn2.emit(:tcp_open, conn1)
  220. 11 conn1.merge(conn2)
  221. 11 @connections.delete(conn2)
  222. end
  223. 23 def next_timeout
  224. 450619 [
  225. 3241294 @timers.wait_interval,
  226. *@resolvers.values.reject(&:closed?).filter_map(&:timeout),
  227. *@connections.filter_map(&:timeout),
  228. ].compact.min
  229. end
  230. 23 def find_resolver_for(connection)
  231. 4940 connection_options = connection.options
  232. 4940 resolver_type = connection_options.resolver_class
  233. 4940 resolver_type = Resolver.resolver_for(resolver_type)
  234. 4940 @resolvers[resolver_type] ||= begin
  235. 600 resolver_manager = if resolver_type.multi?
  236. 579 Resolver::Multi.new(resolver_type, connection_options)
  237. else
  238. 21 resolver_type.new(connection_options)
  239. end
  240. 600 resolver_manager.on(:resolve, &method(:on_resolver_connection))
  241. 600 resolver_manager.on(:error, &method(:on_resolver_error))
  242. 600 resolver_manager.on(:close, &method(:on_resolver_close))
  243. 600 resolver_manager
  244. end
  245. 4940 manager = @resolvers[resolver_type]
  246. 4940 (manager.is_a?(Resolver::Multi) && manager.early_resolve(connection)) || manager.resolvers.each do |resolver|
  247. 370 resolver.pool = self
  248. 370 yield resolver
  249. end
  250. 4895 manager
  251. end
  252. end
  253. end

lib/httpx/punycode.rb

100.0% lines covered

9 relevant lines. 9 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 module Punycode
  4. 23 module_function
  5. begin
  6. 23 require "idnx"
  7. 22 def encode_hostname(hostname)
  8. 24 Idnx.to_punycode(hostname)
  9. end
  10. rescue LoadError
  11. 1 def encode_hostname(hostname)
  12. 1 warn "#{hostname} cannot be converted to punycode. Install the " \
  13. "\"idnx\" gem: https://github.com/HoneyryderChuck/idnx"
  14. 1 hostname
  15. end
  16. end
  17. end
  18. end

lib/httpx/request.rb

100.0% lines covered

109 relevant lines. 109 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "delegate"
  3. 23 require "forwardable"
  4. 23 module HTTPX
  5. # Defines how an HTTP request is handled internally, both in terms of making attributes accessible,
  6. # as well as maintaining the state machine which manages streaming the request onto the wire.
  7. 23 class Request
  8. 23 extend Forwardable
  9. 23 include Callbacks
  10. 23 using URIExtensions
  11. # default value used for "user-agent" header, when not overridden.
  12. 23 USER_AGENT = "httpx.rb/#{VERSION}"
  13. # the upcased string HTTP verb for this request.
  14. 23 attr_reader :verb
  15. # the absolute URI object for this request.
  16. 23 attr_reader :uri
  17. # an HTTPX::Headers object containing the request HTTP headers.
  18. 23 attr_reader :headers
  19. # an HTTPX::Request::Body object containing the request body payload (or +nil+, whenn there is none).
  20. 23 attr_reader :body
  21. # a symbol describing which frame is currently being flushed.
  22. 23 attr_reader :state
  23. # an HTTPX::Options object containing request options.
  24. 23 attr_reader :options
  25. # the corresponding HTTPX::Response object, when there is one.
  26. 23 attr_reader :response
  27. # Exception raised during enumerable body writes.
  28. 23 attr_reader :drain_error
  29. # The IP address from the peer server.
  30. 23 attr_accessor :peer_address
  31. 23 attr_writer :persistent
  32. # will be +true+ when request body has been completely flushed.
  33. 23 def_delegator :@body, :empty?
  34. # initializes the instance with the given +verb+, an absolute or relative +uri+, and the
  35. # request options.
  36. 23 def initialize(verb, uri, options = {})
  37. 6916 @verb = verb.to_s.upcase
  38. 6916 @options = Options.new(options)
  39. 6916 @uri = Utils.to_uri(uri)
  40. 6915 if @uri.relative?
  41. 432 origin = @options.origin
  42. 432 raise(Error, "invalid URI: #{@uri}") unless origin
  43. 432 base_path = @options.base_path
  44. 432 @uri = origin.merge("#{base_path}#{@uri}")
  45. end
  46. 6915 @headers = @options.headers.dup
  47. 6915 @headers["user-agent"] ||= USER_AGENT
  48. 6915 @headers["accept"] ||= "*/*"
  49. 6915 @body = @options.request_body_class.new(@headers, @options)
  50. 6909 @state = :idle
  51. 6909 @response = nil
  52. 6909 @peer_address = nil
  53. 6909 @persistent = @options.persistent
  54. end
  55. # the read timeout defied for this requet.
  56. 23 def read_timeout
  57. 16506 @options.timeout[:read_timeout]
  58. end
  59. # the write timeout defied for this requet.
  60. 23 def write_timeout
  61. 16506 @options.timeout[:write_timeout]
  62. end
  63. # the request timeout defied for this requet.
  64. 23 def request_timeout
  65. 16506 @options.timeout[:request_timeout]
  66. end
  67. 23 def persistent?
  68. 3453 @persistent
  69. end
  70. 23 def trailers?
  71. 2110 defined?(@trailers)
  72. end
  73. 23 def trailers
  74. 66 @trailers ||= @options.headers_class.new
  75. end
  76. # returns +:r+ or +:w+, depending on whether the request is waiting for a response or flushing.
  77. 23 def interests
  78. 18733 return :r if @state == :done || @state == :expect
  79. 2885 :w
  80. end
  81. 23 def merge_headers(h)
  82. 6 @headers = @headers.merge(h)
  83. end
  84. # the URI scheme of the request +uri+.
  85. 23 def scheme
  86. 2530 @uri.scheme
  87. end
  88. # sets the +response+ on this request.
  89. 23 def response=(response)
  90. 6553 return unless response
  91. 6553 if response.is_a?(Response) && response.status < 200
  92. # deal with informational responses
  93. 114 if response.status == 100 && @headers.key?("expect")
  94. 96 @informational_status = response.status
  95. 96 return
  96. end
  97. # 103 Early Hints advertises resources in document to browsers.
  98. # not very relevant for an HTTP client, discard.
  99. 18 return if response.status >= 103
  100. end
  101. 6457 @response = response
  102. 6457 emit(:response_started, response)
  103. end
  104. # returnns the URI path of the request +uri+.
  105. 23 def path
  106. 6099 path = uri.path.dup
  107. 6099 path = +"" if path.nil?
  108. 6099 path << "/" if path.empty?
  109. 6099 path << "?#{query}" unless query.empty?
  110. 6099 path
  111. end
  112. # returs the URI authority of the request.
  113. #
  114. # session.build_request("GET", "https://google.com/query").authority #=> "google.com"
  115. # session.build_request("GET", "http://internal:3182/a").authority #=> "internal:3182"
  116. 23 def authority
  117. 6145 @uri.authority
  118. end
  119. # returs the URI origin of the request.
  120. #
  121. # session.build_request("GET", "https://google.com/query").authority #=> "https://google.com"
  122. # session.build_request("GET", "http://internal:3182/a").authority #=> "http://internal:3182"
  123. 23 def origin
  124. 2730 @uri.origin
  125. end
  126. # returs the URI query string of the request (when available).
  127. #
  128. # session.build_request("GET", "https://search.com").query #=> ""
  129. # session.build_request("GET", "https://search.com?q=a").query #=> "q=a"
  130. # session.build_request("GET", "https://search.com", params: { q: "a"}).query #=> "q=a"
  131. # session.build_request("GET", "https://search.com?q=a", params: { foo: "bar"}).query #=> "q=a&foo&bar"
  132. 23 def query
  133. 6737 return @query if defined?(@query)
  134. 5673 query = []
  135. 5673 if (q = @options.params)
  136. 116 query << Transcoder::Form.encode(q)
  137. end
  138. 5673 query << @uri.query if @uri.query
  139. 5673 @query = query.join("&")
  140. end
  141. # consumes and returns the next available chunk of request body that can be sent
  142. 23 def drain_body
  143. 6628 return nil if @body.nil?
  144. 6628 @drainer ||= @body.each
  145. 6628 chunk = @drainer.next.dup
  146. 4527 emit(:body_chunk, chunk)
  147. 4527 chunk
  148. rescue StopIteration
  149. 2091 nil
  150. rescue StandardError => e
  151. 10 @drain_error = e
  152. 10 nil
  153. end
  154. skipped # :nocov:
  155. skipped def inspect
  156. skipped "#<HTTPX::Request:#{object_id} " \
  157. skipped "#{@verb} " \
  158. skipped "#{uri} " \
  159. skipped "@headers=#{@headers} " \
  160. skipped "@body=#{@body}>"
  161. skipped end
  162. skipped # :nocov:
  163. # moves on to the +nextstate+ of the request state machine (when all preconditions are met)
  164. 23 def transition(nextstate)
  165. 24903 case nextstate
  166. when :idle
  167. 513 @body.rewind
  168. 513 @response = nil
  169. 513 @drainer = nil
  170. when :headers
  171. 8020 return unless @state == :idle
  172. when :body
  173. 8008 return unless @state == :headers ||
  174. @state == :expect
  175. 6236 if @headers.key?("expect")
  176. 365 if @informational_status && @informational_status == 100
  177. # check for 100 Continue response, and deallocate the var
  178. # if @informational_status == 100
  179. # @response = nil
  180. # end
  181. else
  182. 277 return if @state == :expect # do not re-set it
  183. 108 nextstate = :expect
  184. end
  185. end
  186. when :trailers
  187. 6226 return unless @state == :body
  188. when :done
  189. 6232 return if @state == :expect
  190. end
  191. 24447 @state = nextstate
  192. 24447 emit(@state, self)
  193. 3357 nil
  194. end
  195. # whether the request supports the 100-continue handshake and already processed the 100 response.
  196. 23 def expects?
  197. 5739 @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
  198. end
  199. end
  200. end
  201. 23 require_relative "request/body"

lib/httpx/request/body.rb

100.0% lines covered

71 relevant lines. 71 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. # Implementation of the HTTP Request body as a delegator which iterates (responds to +each+) payload chunks.
  4. 23 class Request::Body < SimpleDelegator
  5. 23 class << self
  6. 23 def new(_, options)
  7. 6921 return options.body if options.body.is_a?(self)
  8. 6714 super
  9. end
  10. end
  11. # inits the instance with the request +headers+ and +options+, which contain the payload definition.
  12. 23 def initialize(headers, options)
  13. 6714 @headers = headers
  14. # forego compression in the Range request case
  15. 6714 if @headers.key?("range")
  16. 6 @headers.delete("accept-encoding")
  17. else
  18. 6708 @headers["accept-encoding"] ||= options.supported_compression_formats
  19. end
  20. 6714 initialize_body(options)
  21. 6714 return if @body.nil?
  22. 2142 @headers["content-type"] ||= @body.content_type
  23. 2142 @headers["content-length"] = @body.bytesize unless unbounded_body?
  24. 2136 super(@body)
  25. end
  26. # consumes and yields the request payload in chunks.
  27. 23 def each(&block)
  28. 4658 return enum_for(__method__) unless block
  29. 2329 return if @body.nil?
  30. 2154 body = stream(@body)
  31. 2154 if body.respond_to?(:read)
  32. 922 ::IO.copy_stream(body, ProcIO.new(block))
  33. 1232 elsif body.respond_to?(:each)
  34. 303 body.each(&block)
  35. else
  36. 929 block[body.to_s]
  37. end
  38. end
  39. # if the +@body+ is rewindable, it rewinnds it.
  40. 23 def rewind
  41. 597 return if empty?
  42. 96 @body.rewind if @body.respond_to?(:rewind)
  43. end
  44. # return +true+ if the +body+ has been fully drained (or does nnot exist).
  45. 23 def empty?
  46. 13508 return true if @body.nil?
  47. 6205 return false if chunked?
  48. 6133 @body.bytesize.zero?
  49. end
  50. # returns the +@body+ payload size in bytes.
  51. 23 def bytesize
  52. 2472 return 0 if @body.nil?
  53. 96 @body.bytesize
  54. end
  55. # sets the body to yield using chunked trannsfer encoding format.
  56. 23 def stream(body)
  57. 2154 return body unless chunked?
  58. 72 Transcoder::Chunker.encode(body.enum_for(:each))
  59. end
  60. # returns whether the body yields infinitely.
  61. 23 def unbounded_body?
  62. 2448 return @unbounded_body if defined?(@unbounded_body)
  63. 2142 @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
  64. end
  65. # returns whether the chunked transfer encoding header is set.
  66. 23 def chunked?
  67. 14112 @headers["transfer-encoding"] == "chunked"
  68. end
  69. # sets the chunked transfer encoding header.
  70. 23 def chunk!
  71. 24 @headers.add("transfer-encoding", "chunked")
  72. end
  73. skipped # :nocov:
  74. skipped def inspect
  75. skipped "#<HTTPX::Request::Body:#{object_id} " \
  76. skipped "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
  77. skipped end
  78. skipped # :nocov:
  79. 23 private
  80. # wraps the given body with the appropriate encoder.
  81. #
  82. # ..., json: { foo: "bar" }) #=> json encoder
  83. # ..., form: { foo: "bar" }) #=> form urlencoded encoder
  84. # ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
  85. # ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
  86. # ..., form: { body: "bla") }) #=> raw data encoder
  87. 23 def initialize_body(options)
  88. 6714 @body = if options.body
  89. 902 Transcoder::Body.encode(options.body)
  90. 5812 elsif options.form
  91. 1084 Transcoder::Form.encode(options.form)
  92. 4728 elsif options.json
  93. 54 Transcoder::JSON.encode(options.json)
  94. 4674 elsif options.xml
  95. 102 Transcoder::Xml.encode(options.xml)
  96. end
  97. 6714 return unless @body && options.compress_request_body && @headers.key?("content-encoding")
  98. 62 @headers.get("content-encoding").each do |encoding|
  99. 62 @body = self.class.initialize_deflater_body(@body, encoding)
  100. end
  101. end
  102. 23 class << self
  103. # returns the +body+ wrapped with the correct deflater accordinng to the given +encodisng+.
  104. 23 def initialize_deflater_body(body, encoding)
  105. 55 case encoding
  106. when "gzip"
  107. 32 Transcoder::GZIP.encode(body)
  108. when "deflate"
  109. 12 Transcoder::Deflate.encode(body)
  110. when "identity"
  111. 12 body
  112. else
  113. 6 body
  114. end
  115. end
  116. end
  117. end
  118. # Wrapper yielder which can be used with functions which expect an IO writer.
  119. 23 class ProcIO
  120. 23 def initialize(block)
  121. 922 @block = block
  122. end
  123. # Implementation the IO write protocol, which yield the given chunk to +@block+.
  124. 23 def write(data)
  125. 2622 @block.call(data.dup)
  126. 2616 data.bytesize
  127. end
  128. end
  129. end

lib/httpx/resolver.rb

100.0% lines covered

76 relevant lines. 76 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "resolv"
  3. 23 require "ipaddr"
  4. 23 module HTTPX
  5. 23 module Resolver
  6. 23 RESOLVE_TIMEOUT = [2, 3].freeze
  7. 23 require "httpx/resolver/resolver"
  8. 23 require "httpx/resolver/system"
  9. 23 require "httpx/resolver/native"
  10. 23 require "httpx/resolver/https"
  11. 23 require "httpx/resolver/multi"
  12. 23 @lookup_mutex = Thread::Mutex.new
  13. 152 @lookups = Hash.new { |h, k| h[k] = [] }
  14. 23 @identifier_mutex = Thread::Mutex.new
  15. 23 @identifier = 1
  16. 23 @system_resolver = Resolv::Hosts.new
  17. 23 module_function
  18. 23 def resolver_for(resolver_type)
  19. 4227 case resolver_type
  20. 4820 when :native then Native
  21. 27 when :system then System
  22. 61 when :https then HTTPS
  23. else
  24. 62 return resolver_type if resolver_type.is_a?(Class) && resolver_type < Resolver
  25. 6 raise Error, "unsupported resolver type (#{resolver_type})"
  26. end
  27. end
  28. 23 def nolookup_resolve(hostname)
  29. 4815 ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
  30. end
  31. 23 def ip_resolve(hostname)
  32. 4815 [IPAddr.new(hostname)]
  33. rescue ArgumentError
  34. end
  35. 23 def system_resolve(hostname)
  36. 429 ips = @system_resolver.getaddresses(hostname)
  37. 429 return if ips.empty?
  38. 555 ips.map { |ip| IPAddr.new(ip) }
  39. rescue IOError
  40. end
  41. 23 def cached_lookup(hostname)
  42. 4578 now = Utils.now
  43. 4578 @lookup_mutex.synchronize do
  44. 4578 lookup(hostname, now)
  45. end
  46. end
  47. 23 def cached_lookup_set(hostname, family, entries)
  48. 176 now = Utils.now
  49. 176 entries.each do |entry|
  50. 233 entry["TTL"] += now
  51. end
  52. 176 @lookup_mutex.synchronize do
  53. 148 case family
  54. when Socket::AF_INET6
  55. 30 @lookups[hostname].concat(entries)
  56. when Socket::AF_INET
  57. 146 @lookups[hostname].unshift(*entries)
  58. end
  59. 176 entries.each do |entry|
  60. 233 next unless entry["name"] != hostname
  61. 22 case family
  62. when Socket::AF_INET6
  63. 6 @lookups[entry["name"]] << entry
  64. when Socket::AF_INET
  65. 18 @lookups[entry["name"]].unshift(entry)
  66. end
  67. end
  68. end
  69. end
  70. # do not use directly!
  71. 23 def lookup(hostname, ttl)
  72. 4584 return unless @lookups.key?(hostname)
  73. 4150 entries = @lookups[hostname] = @lookups[hostname].select do |address|
  74. 8657 address["TTL"] > ttl
  75. end
  76. 4150 ips = entries.flat_map do |address|
  77. 8650 if address.key?("alias")
  78. 6 lookup(address["alias"], ttl)
  79. else
  80. 8644 IPAddr.new(address["data"])
  81. end
  82. end.compact
  83. 4150 ips unless ips.empty?
  84. end
  85. 23 def generate_id
  86. 1216 @identifier_mutex.synchronize { @identifier = (@identifier + 1) & 0xFFFF }
  87. end
  88. 23 def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
  89. 608 Resolv::DNS::Message.new.tap do |query|
  90. 608 query.id = message_id
  91. 608 query.rd = 1
  92. 608 query.add_question(hostname, type)
  93. end.encode
  94. end
  95. 23 def decode_dns_answer(payload)
  96. begin
  97. 551 message = Resolv::DNS::Message.decode(payload)
  98. rescue Resolv::DNS::DecodeError => e
  99. 5 return :decode_error, e
  100. end
  101. # no domain was found
  102. 546 return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
  103. 220 return :message_truncated if message.tc == 1
  104. 210 return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
  105. 200 addresses = []
  106. 200 message.each_answer do |question, _, value|
  107. 849 case value
  108. when Resolv::DNS::Resource::IN::CNAME
  109. 18 addresses << {
  110. "name" => question.to_s,
  111. "TTL" => value.ttl,
  112. "alias" => value.name.to_s,
  113. }
  114. when Resolv::DNS::Resource::IN::A,
  115. Resolv::DNS::Resource::IN::AAAA
  116. 860 addresses << {
  117. "name" => question.to_s,
  118. "TTL" => value.ttl,
  119. "data" => value.address.to_s,
  120. }
  121. end
  122. end
  123. 200 [:ok, addresses]
  124. end
  125. end
  126. end

lib/httpx/resolver/https.rb

87.92% lines covered

149 relevant lines. 131 lines covered and 18 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "resolv"
  3. 23 require "uri"
  4. 23 require "cgi"
  5. 23 require "forwardable"
  6. 23 require "httpx/base64"
  7. 23 module HTTPX
  8. 23 class Resolver::HTTPS < Resolver::Resolver
  9. 23 extend Forwardable
  10. 23 using URIExtensions
  11. 23 module DNSExtensions
  12. 23 refine Resolv::DNS do
  13. 23 def generate_candidates(name)
  14. 35 @config.generate_candidates(name)
  15. end
  16. end
  17. end
  18. 23 using DNSExtensions
  19. 23 NAMESERVER = "https://1.1.1.1/dns-query"
  20. 1 DEFAULTS = {
  21. 22 uri: NAMESERVER,
  22. use_get: false,
  23. }.freeze
  24. 23 def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate
  25. 23 def initialize(_, options)
  26. 40 super
  27. 40 @resolver_options = DEFAULTS.merge(@options.resolver_options)
  28. 40 @queries = {}
  29. 40 @requests = {}
  30. 40 @connections = []
  31. 40 @uri = URI(@resolver_options[:uri])
  32. 40 @uri_addresses = nil
  33. 40 @resolver = Resolv::DNS.new
  34. 40 @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
  35. 40 @resolver.lazy_initialize
  36. end
  37. 23 def <<(connection)
  38. 75 return if @uri.origin == connection.origin.to_s
  39. 40 @uri_addresses ||= HTTPX::Resolver.nolookup_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
  40. 40 if @uri_addresses.empty?
  41. 5 ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
  42. 5 ex.set_backtrace(caller)
  43. 5 connection.force_reset
  44. 5 throw(:resolve_error, ex)
  45. end
  46. 35 resolve(connection)
  47. end
  48. 23 def closed?
  49. 200 true
  50. end
  51. 23 def empty?
  52. 70 true
  53. end
  54. 23 def resolver_connection
  55. 90 @resolver_connection ||= @pool.find_connection(@uri, @options) || begin
  56. 35 @building_connection = true
  57. 35 connection = @options.connection_class.new(@uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
  58. 35 @pool.init_connection(connection, @options)
  59. # only explicity emit addresses if connection didn't pre-resolve, i.e. it's not an IP.
  60. 35 emit_addresses(connection, @family, @uri_addresses) unless connection.addresses
  61. 35 @building_connection = false
  62. 35 connection
  63. end
  64. end
  65. 23 private
  66. 23 def resolve(connection = @connections.first, hostname = nil)
  67. 55 return if @building_connection
  68. 55 return unless connection
  69. 55 hostname ||= @queries.key(connection)
  70. 55 if hostname.nil?
  71. 35 hostname = connection.origin.host
  72. 35 log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
  73. 35 hostname = @resolver.generate_candidates(hostname).each do |name|
  74. 105 @queries[name.to_s] = connection
  75. end.first.to_s
  76. else
  77. 20 @queries[hostname] = connection
  78. end
  79. 55 log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
  80. begin
  81. 55 request = build_request(hostname)
  82. 55 request.on(:response, &method(:on_response).curry(2)[request])
  83. 55 request.on(:promise, &method(:on_promise))
  84. 55 @requests[request] = hostname
  85. 55 resolver_connection.send(request)
  86. 55 @connections << connection
  87. rescue ResolveError, Resolv::DNS::EncodeError => e
  88. reset_hostname(hostname)
  89. emit_resolve_error(connection, connection.origin.host, e)
  90. end
  91. end
  92. 23 def on_response(request, response)
  93. 55 response.raise_for_status
  94. rescue StandardError => e
  95. 5 hostname = @requests.delete(request)
  96. 5 connection = reset_hostname(hostname)
  97. 5 emit_resolve_error(connection, connection.origin.host, e)
  98. else
  99. # @type var response: HTTPX::Response
  100. 50 parse(request, response)
  101. ensure
  102. 55 @requests.delete(request)
  103. end
  104. 23 def on_promise(_, stream)
  105. log(level: 2) { "#{stream.id}: refusing stream!" }
  106. stream.refuse
  107. end
  108. 23 def parse(request, response)
  109. 50 code, result = decode_response_body(response)
  110. 50 case code
  111. when :ok
  112. 15 parse_addresses(result, request)
  113. when :no_domain_found
  114. # Indicates no such domain was found.
  115. 30 host = @requests.delete(request)
  116. 30 connection = reset_hostname(host, reset_candidates: false)
  117. 30 unless @queries.value?(connection)
  118. 10 emit_resolve_error(connection)
  119. 10 return
  120. end
  121. 20 resolve
  122. when :dns_error
  123. host = @requests.delete(request)
  124. connection = reset_hostname(host)
  125. emit_resolve_error(connection)
  126. when :decode_error
  127. 5 host = @requests.delete(request)
  128. 5 connection = reset_hostname(host)
  129. 5 emit_resolve_error(connection, connection.origin.host, result)
  130. end
  131. end
  132. 23 def parse_addresses(answers, request)
  133. 15 if answers.empty?
  134. # no address found, eliminate candidates
  135. host = @requests.delete(request)
  136. connection = reset_hostname(host)
  137. emit_resolve_error(connection)
  138. return
  139. else
  140. 35 answers = answers.group_by { |answer| answer["name"] }
  141. 15 answers.each do |hostname, addresses|
  142. 20 addresses = addresses.flat_map do |address|
  143. 20 if address.key?("alias")
  144. 5 alias_address = answers[address["alias"]]
  145. 5 if alias_address.nil?
  146. reset_hostname(address["name"])
  147. if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
  148. @connections.delete(connection)
  149. else
  150. resolve(connection, address["alias"])
  151. return # rubocop:disable Lint/NonLocalExitFromIterator
  152. end
  153. else
  154. 5 alias_address
  155. end
  156. else
  157. 15 address
  158. end
  159. end.compact
  160. 20 next if addresses.empty?
  161. 20 hostname.delete_suffix!(".") if hostname.end_with?(".")
  162. 20 connection = reset_hostname(hostname, reset_candidates: false)
  163. 20 next unless connection # probably a retried query for which there's an answer
  164. 15 @connections.delete(connection)
  165. # eliminate other candidates
  166. 45 @queries.delete_if { |_, conn| connection == conn }
  167. 15 Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
  168. 30 emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
  169. end
  170. end
  171. 15 return if @connections.empty?
  172. resolve
  173. end
  174. 23 def build_request(hostname)
  175. 50 uri = @uri.dup
  176. 50 rklass = @options.request_class
  177. 50 payload = Resolver.encode_dns_query(hostname, type: @record_type)
  178. 50 if @resolver_options[:use_get]
  179. 5 params = URI.decode_www_form(uri.query.to_s)
  180. 5 params << ["type", FAMILY_TYPES[@record_type]]
  181. 5 params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
  182. 5 uri.query = URI.encode_www_form(params)
  183. 5 request = rklass.new("GET", uri, @options)
  184. else
  185. 45 request = rklass.new("POST", uri, @options.merge(body: [payload]))
  186. 45 request.headers["content-type"] = "application/dns-message"
  187. end
  188. 50 request.headers["accept"] = "application/dns-message"
  189. 50 request
  190. end
  191. 23 def decode_response_body(response)
  192. 45 case response.headers["content-type"]
  193. when "application/dns-udpwireformat",
  194. "application/dns-message"
  195. 45 Resolver.decode_dns_answer(response.to_s)
  196. else
  197. raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
  198. end
  199. end
  200. 23 def reset_hostname(hostname, reset_candidates: true)
  201. 60 connection = @queries.delete(hostname)
  202. 60 return connection unless connection && reset_candidates
  203. # eliminate other candidates
  204. 30 candidates = @queries.select { |_, conn| connection == conn }.keys
  205. 30 @queries.delete_if { |h, _| candidates.include?(h) }
  206. 10 connection
  207. end
  208. end
  209. end

lib/httpx/resolver/multi.rb

100.0% lines covered

40 relevant lines. 40 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "forwardable"
  3. 23 require "resolv"
  4. 23 module HTTPX
  5. 23 class Resolver::Multi
  6. 23 include Callbacks
  7. 23 using ArrayExtensions::FilterMap
  8. 23 attr_reader :resolvers
  9. 23 def initialize(resolver_type, options)
  10. 579 @options = options
  11. 579 @resolver_options = @options.resolver_options
  12. 579 @resolvers = options.ip_families.map do |ip_family|
  13. 579 resolver = resolver_type.new(ip_family, options)
  14. 579 resolver.on(:resolve, &method(:on_resolver_connection))
  15. 579 resolver.on(:error, &method(:on_resolver_error))
  16. 737 resolver.on(:close) { on_resolver_close(resolver) }
  17. 579 resolver
  18. end
  19. 579 @errors = Hash.new { |hs, k| hs[k] = [] }
  20. end
  21. 23 def closed?
  22. 3692973 @resolvers.all?(&:closed?)
  23. end
  24. 23 def timeout
  25. 3690700 @resolvers.filter_map(&:timeout).min
  26. end
  27. 23 def close
  28. 1813 @resolvers.each(&:close)
  29. end
  30. 23 def connections
  31. 8988 @resolvers.filter_map { |r| r.resolver_connection if r.respond_to?(:resolver_connection) }
  32. end
  33. 23 def early_resolve(connection)
  34. 4919 hostname = connection.origin.host
  35. 4919 addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
  36. 4919 return unless addresses
  37. 4755 addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
  38. # try to match the resolver by family. However, there are cases where that's not possible, as when
  39. # the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
  40. 9486 resolver = @resolvers.find { |r| r.family == family } || @resolvers.first
  41. 4743 next unless resolver # this should ever happen
  42. # it does not matter which resolver it is, as early-resolve code is shared.
  43. 4743 resolver.emit_addresses(connection, family, addrs, true)
  44. end
  45. end
  46. 23 private
  47. 23 def on_resolver_connection(connection)
  48. 4922 emit(:resolve, connection)
  49. end
  50. 23 def on_resolver_error(connection, error)
  51. 140 emit(:error, connection, error)
  52. end
  53. 23 def on_resolver_close(resolver)
  54. 158 emit(:close, resolver)
  55. end
  56. end
  57. end

lib/httpx/resolver/native.rb

94.19% lines covered

258 relevant lines. 243 lines covered and 15 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "forwardable"
  3. 23 require "resolv"
  4. 23 module HTTPX
  5. 23 class Resolver::Native < Resolver::Resolver
  6. 23 extend Forwardable
  7. 23 using URIExtensions
  8. 1 DEFAULTS = {
  9. 22 nameserver: nil,
  10. **Resolv::DNS::Config.default_config_hash,
  11. packet_size: 512,
  12. timeouts: Resolver::RESOLVE_TIMEOUT,
  13. }.freeze
  14. 23 DNS_PORT = 53
  15. 23 def_delegator :@connections, :empty?
  16. 23 attr_reader :state
  17. 23 def initialize(family, options)
  18. 539 super
  19. 539 @ns_index = 0
  20. 539 @resolver_options = DEFAULTS.merge(@options.resolver_options)
  21. 539 @socket_type = @resolver_options.fetch(:socket_type, :udp)
  22. 539 @nameserver = if (nameserver = @resolver_options[:nameserver])
  23. 534 nameserver = nameserver[family] if nameserver.is_a?(Hash)
  24. 534 Array(nameserver)
  25. end
  26. 539 @ndots = @resolver_options.fetch(:ndots, 1)
  27. 1617 @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
  28. 539 @_timeouts = Array(@resolver_options[:timeouts])
  29. 1698 @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
  30. 539 @connections = []
  31. 539 @queries = {}
  32. 539 @read_buffer = "".b
  33. 539 @write_buffer = Buffer.new(@resolver_options[:packet_size])
  34. 539 @state = :idle
  35. end
  36. 23 def close
  37. 1971 transition(:closed)
  38. end
  39. 23 def closed?
  40. 3693089 @state == :closed
  41. end
  42. 23 def to_io
  43. 1019 @io.to_io
  44. end
  45. 23 def call
  46. 753 case @state
  47. when :open
  48. 835 consume
  49. end
  50. 78 nil
  51. rescue Errno::EHOSTUNREACH => e
  52. 15 @ns_index += 1
  53. 15 nameserver = @nameserver
  54. 15 if nameserver && @ns_index < nameserver.size
  55. 10 log { "resolver: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
  56. 10 transition(:idle)
  57. else
  58. 5 handle_error(e)
  59. end
  60. rescue NativeResolveError => e
  61. 94 handle_error(e)
  62. end
  63. 23 def interests
  64. 1772281 case @state
  65. when :idle
  66. 3052 transition(:open)
  67. when :closed
  68. 259 transition(:idle)
  69. 259 transition(:open)
  70. end
  71. 1772525 calculate_interests
  72. end
  73. 23 def <<(connection)
  74. 274 if @nameserver.nil?
  75. 5 ex = ResolveError.new("No available nameserver")
  76. 5 ex.set_backtrace(caller)
  77. 5 connection.force_reset
  78. 5 throw(:resolve_error, ex)
  79. else
  80. 269 @connections << connection
  81. 269 resolve
  82. end
  83. end
  84. 23 def timeout
  85. 3690700 return if @connections.empty?
  86. 3702 @start_timeout = Utils.now
  87. 3702 hosts = @queries.keys
  88. 3702 @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
  89. end
  90. 23 def handle_socket_timeout(interval)
  91. 82 do_retry(interval)
  92. end
  93. 23 private
  94. 23 def calculate_interests
  95. 1774066 return :w unless @write_buffer.empty?
  96. 1772865 return :r unless @queries.empty?
  97. 178 nil
  98. end
  99. 23 def consume
  100. 820 dread if calculate_interests == :r
  101. 721 do_retry
  102. 721 dwrite if calculate_interests == :w
  103. end
  104. 23 def do_retry(loop_time = nil)
  105. 803 return if @queries.empty? || !@start_timeout
  106. 619 loop_time ||= Utils.elapsed_time(@start_timeout)
  107. 619 query = @queries.first
  108. 619 return unless query
  109. 619 h, connection = query
  110. 619 host = connection.origin.host
  111. 619 timeout = (@timeouts[host][0] -= loop_time)
  112. 619 return unless timeout <= 0
  113. 62 @timeouts[host].shift
  114. 62 if !@timeouts[host].empty?
  115. 37 log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
  116. 37 resolve(connection)
  117. 25 elsif @ns_index + 1 < @nameserver.size
  118. # try on the next nameserver
  119. 5 @ns_index += 1
  120. 5 log { "resolver: failed resolving #{host} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)" }
  121. 5 transition(:idle)
  122. 5 resolve(connection)
  123. else
  124. 20 @timeouts.delete(host)
  125. 20 reset_hostname(h, reset_candidates: false)
  126. 20 return unless @queries.empty?
  127. 5 @connections.delete(connection)
  128. # This loop_time passed to the exception is bogus. Ideally we would pass the total
  129. # resolve timeout, including from the previous retries.
  130. 5 raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
  131. end
  132. end
  133. 23 def dread(wsize = @resolver_options[:packet_size])
  134. 506 loop do
  135. 750 wsize = @large_packet.capacity if @large_packet
  136. 750 siz = @io.read(wsize, @read_buffer)
  137. 750 unless siz
  138. ex = EOFError.new("descriptor closed")
  139. ex.set_backtrace(caller)
  140. raise ex
  141. end
  142. 750 return unless siz.positive?
  143. 516 if @socket_type == :tcp
  144. # packet may be incomplete, need to keep draining from the socket
  145. 25 if @large_packet
  146. # large packet buffer already exists, continue pumping
  147. 10 @large_packet << @read_buffer
  148. 10 next unless @large_packet.full?
  149. 10 parse(@large_packet.to_s)
  150. 10 @socket_type = @resolver_options.fetch(:socket_type, :udp)
  151. 10 @large_packet = nil
  152. 10 transition(:idle)
  153. 10 transition(:open)
  154. 10 return
  155. else
  156. 15 size = @read_buffer[0, 2].unpack1("n")
  157. 15 buffer = @read_buffer.byteslice(2..-1)
  158. 15 if size > @read_buffer.bytesize
  159. # only do buffer logic if it's worth it, and the whole packet isn't here already
  160. 10 @large_packet = Buffer.new(size)
  161. 10 @large_packet << buffer
  162. 10 next
  163. else
  164. 5 parse(buffer)
  165. end
  166. end
  167. else # udp
  168. 491 parse(@read_buffer)
  169. end
  170. 397 return if @state == :closed
  171. end
  172. end
  173. 23 def dwrite
  174. 553 loop do
  175. 1106 return if @write_buffer.empty?
  176. 553 siz = @io.write(@write_buffer)
  177. 553 unless siz
  178. ex = EOFError.new("descriptor closed")
  179. ex.set_backtrace(caller)
  180. raise ex
  181. end
  182. 553 return unless siz.positive?
  183. 553 return if @state == :closed
  184. end
  185. end
  186. 23 def parse(buffer)
  187. 506 code, result = Resolver.decode_dns_answer(buffer)
  188. 449 case code
  189. when :ok
  190. 185 parse_addresses(result)
  191. when :no_domain_found
  192. # Indicates no such domain was found.
  193. 296 hostname, connection = @queries.first
  194. 296 reset_hostname(hostname, reset_candidates: false)
  195. 296 unless @queries.value?(connection)
  196. 74 @connections.delete(connection)
  197. 74 raise NativeResolveError.new(connection, connection.origin.host, "name or service not known")
  198. end
  199. 222 resolve
  200. when :message_truncated
  201. # TODO: what to do if it's already tcp??
  202. 10 return if @socket_type == :tcp
  203. 10 @socket_type = :tcp
  204. 10 hostname, _ = @queries.first
  205. 10 reset_hostname(hostname)
  206. 10 transition(:closed)
  207. when :dns_error
  208. 10 hostname, connection = @queries.first
  209. 10 reset_hostname(hostname)
  210. 10 @connections.delete(connection)
  211. 10 ex = NativeResolveError.new(connection, connection.origin.host, "unknown DNS error (error code #{result})")
  212. 10 raise ex
  213. when :decode_error
  214. 5 hostname, connection = @queries.first
  215. 5 reset_hostname(hostname)
  216. 5 @connections.delete(connection)
  217. 5 ex = NativeResolveError.new(connection, connection.origin.host, result.message)
  218. 5 ex.set_backtrace(result.backtrace)
  219. 5 raise ex
  220. end
  221. end
  222. 23 def parse_addresses(addresses)
  223. 185 if addresses.empty?
  224. # no address found, eliminate candidates
  225. 5 hostname, connection = @queries.first
  226. 5 reset_hostname(hostname)
  227. 5 @connections.delete(connection)
  228. 5 raise NativeResolveError.new(connection, connection.origin.host)
  229. else
  230. 180 address = addresses.first
  231. 180 name = address["name"]
  232. 180 connection = @queries.delete(name)
  233. 180 unless connection
  234. orig_name = name
  235. # absolute name
  236. name_labels = Resolv::DNS::Name.create(name).to_a
  237. name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
  238. # probably a retried query for which there's an answer
  239. unless name
  240. @timeouts.delete(orig_name)
  241. return
  242. end
  243. address["name"] = name
  244. connection = @queries.delete(name)
  245. end
  246. 180 if address.key?("alias") # CNAME
  247. # clean up intermediate queries
  248. 16 @timeouts.delete(name) unless connection.origin.host == name
  249. 32 if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
  250. 1 @connections.delete(connection)
  251. else
  252. 15 resolve(connection, address["alias"])
  253. 15 return
  254. end
  255. else
  256. 164 reset_hostname(name, connection: connection)
  257. 164 @timeouts.delete(connection.origin.host)
  258. 164 @connections.delete(connection)
  259. 164 Resolver.cached_lookup_set(connection.origin.host, @family, addresses) if @resolver_options[:cache]
  260. 680 emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
  261. end
  262. end
  263. 160 return emit(:close) if @connections.empty?
  264. 2 resolve
  265. end
  266. 23 def resolve(connection = @connections.first, hostname = nil)
  267. 560 raise Error, "no URI to resolve" unless connection
  268. 560 return unless @write_buffer.empty?
  269. 558 hostname ||= @queries.key(connection)
  270. 558 if hostname.nil?
  271. 279 hostname = connection.origin.host
  272. 279 log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
  273. 279 hostname = generate_candidates(hostname).each do |name|
  274. 996 @queries[name] = connection
  275. end.first
  276. else
  277. 252 @queries[hostname] = connection
  278. end
  279. 558 log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
  280. begin
  281. 558 @write_buffer << encode_dns_query(hostname)
  282. rescue Resolv::DNS::EncodeError => e
  283. emit_resolve_error(connection, hostname, e)
  284. end
  285. end
  286. 23 def encode_dns_query(hostname)
  287. 558 message_id = Resolver.generate_id
  288. 558 msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
  289. 558 msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
  290. 558 msg
  291. end
  292. 23 def generate_candidates(name)
  293. 279 return [name] if name.end_with?(".")
  294. 279 candidates = []
  295. 279 name_parts = name.scan(/[^.]+/)
  296. 279 candidates = [name] if @ndots <= name_parts.size - 1
  297. 837 candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
  298. 279 fname = "#{name}."
  299. 279 candidates << fname unless candidates.include?(fname)
  300. 279 candidates
  301. end
  302. 23 def build_socket
  303. 514 ip, port = @nameserver[@ns_index]
  304. 514 port ||= DNS_PORT
  305. 455 case @socket_type
  306. when :udp
  307. 499 log { "resolver: server: udp://#{ip}:#{port}..." }
  308. 499 UDP.new(ip, port, @options)
  309. when :tcp
  310. 15 log { "resolver: server: tcp://#{ip}:#{port}..." }
  311. 15 origin = URI("tcp://#{ip}:#{port}")
  312. 15 TCP.new(origin, [ip], @options)
  313. end
  314. end
  315. 23 def transition(nextstate)
  316. 5052 case nextstate
  317. when :idle
  318. 284 if @io
  319. 284 @io.close
  320. 284 @io = nil
  321. end
  322. 284 @timeouts.clear
  323. when :open
  324. 3321 return unless @state == :idle
  325. 3321 @io ||= build_socket
  326. 3321 @io.connect
  327. 3321 return unless @io.connected?
  328. 514 resolve if @queries.empty? && !@connections.empty?
  329. when :closed
  330. 1981 return unless @state == :open
  331. 465 @io.close if @io
  332. 465 @start_timeout = nil
  333. 465 @write_buffer.clear
  334. 465 @read_buffer.clear
  335. end
  336. 1263 @state = nextstate
  337. end
  338. 23 def handle_error(error)
  339. 99 if error.respond_to?(:connection) &&
  340. error.respond_to?(:host)
  341. 94 emit_resolve_error(error.connection, error.host, error)
  342. else
  343. 5 @queries.each do |host, connection|
  344. 20 emit_resolve_error(connection, host, error)
  345. end
  346. end
  347. end
  348. 23 def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
  349. 510 @timeouts.delete(hostname)
  350. 510 @timeouts.delete(hostname)
  351. 510 return unless connection && reset_candidates
  352. # eliminate other candidates
  353. 776 candidates = @queries.select { |_, conn| connection == conn }.keys
  354. 776 @queries.delete_if { |h, _| candidates.include?(h) }
  355. # reset timeouts
  356. 789 @timeouts.delete_if { |h, _| candidates.include?(h) }
  357. end
  358. end
  359. end

lib/httpx/resolver/resolver.rb

90.0% lines covered

60 relevant lines. 54 lines covered and 6 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "resolv"
  3. 23 require "ipaddr"
  4. 23 module HTTPX
  5. 23 class Resolver::Resolver
  6. 23 include Callbacks
  7. 23 include Loggable
  8. 23 using ArrayExtensions::Intersect
  9. 1 RECORD_TYPES = {
  10. 22 Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
  11. Socket::AF_INET => Resolv::DNS::Resource::IN::A,
  12. }.freeze
  13. 1 FAMILY_TYPES = {
  14. 22 Resolv::DNS::Resource::IN::AAAA => "AAAA",
  15. Resolv::DNS::Resource::IN::A => "A",
  16. }.freeze
  17. 23 class << self
  18. 23 def multi?
  19. 579 true
  20. end
  21. end
  22. 23 attr_reader :family
  23. 23 attr_writer :pool
  24. 23 def initialize(family, options)
  25. 600 @family = family
  26. 600 @record_type = RECORD_TYPES[family]
  27. 600 @options = Options.new(options)
  28. end
  29. 23 def close; end
  30. 23 alias_method :terminate, :close
  31. 23 def closed?
  32. true
  33. end
  34. 23 def empty?
  35. true
  36. end
  37. 23 def emit_addresses(connection, family, addresses, early_resolve = false)
  38. 4968 addresses.map! do |address|
  39. 9797 address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
  40. end
  41. # double emission check, but allow early resolution to work
  42. 4968 return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
  43. 5027 log { "resolver: answer #{FAMILY_TYPES[RECORD_TYPES[family]]} #{connection.origin.host}: #{addresses.inspect}" }
  44. 4968 if @pool && # if triggered by early resolve, pool may not be here yet
  45. !connection.io &&
  46. connection.options.ip_families.size > 1 &&
  47. family == Socket::AF_INET &&
  48. addresses.first.to_s != connection.origin.host.to_s
  49. log { "resolver: A response, applying resolution delay..." }
  50. @pool.after(0.05) do
  51. unless connection.state == :closed ||
  52. # double emission check
  53. (connection.addresses && addresses.intersect?(connection.addresses))
  54. emit_resolved_connection(connection, addresses, early_resolve)
  55. end
  56. end
  57. else
  58. 4968 emit_resolved_connection(connection, addresses, early_resolve)
  59. end
  60. end
  61. 23 private
  62. 23 def emit_resolved_connection(connection, addresses, early_resolve)
  63. begin
  64. 4968 connection.addresses = addresses
  65. 4932 emit(:resolve, connection)
  66. 24 rescue StandardError => e
  67. 36 if early_resolve
  68. 30 connection.force_reset
  69. 30 throw(:resolve_error, e)
  70. else
  71. 6 emit(:error, connection, e)
  72. end
  73. end
  74. end
  75. 23 def early_resolve(connection, hostname: connection.origin.host)
  76. 16 addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
  77. 16 return unless addresses
  78. 5 addresses = addresses.select { |addr| addr.family == @family }
  79. 1 return if addresses.empty?
  80. 1 emit_addresses(connection, @family, addresses, true)
  81. end
  82. 23 def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
  83. 145 emit(:error, connection, resolve_error(hostname, ex))
  84. end
  85. 23 def resolve_error(hostname, ex = nil)
  86. 145 return ex if ex.is_a?(ResolveError) || ex.is_a?(ResolveTimeoutError)
  87. 50 message = ex ? ex.message : "Can't resolve #{hostname}"
  88. 50 error = ResolveError.new(message)
  89. 50 error.set_backtrace(ex ? ex.backtrace : caller)
  90. 50 error
  91. end
  92. end
  93. end

lib/httpx/resolver/system.rb

92.68% lines covered

123 relevant lines. 114 lines covered and 9 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "forwardable"
  3. 23 require "resolv"
  4. 23 module HTTPX
  5. 23 class Resolver::System < Resolver::Resolver
  6. 23 using URIExtensions
  7. 23 extend Forwardable
  8. 23 RESOLV_ERRORS = [Resolv::ResolvError,
  9. Resolv::DNS::Requester::RequestError,
  10. Resolv::DNS::EncodeError,
  11. Resolv::DNS::DecodeError].freeze
  12. 23 DONE = 1
  13. 23 ERROR = 2
  14. 23 class << self
  15. 23 def multi?
  16. 21 false
  17. end
  18. end
  19. 23 attr_reader :state
  20. 23 def_delegator :@connections, :empty?
  21. 23 def initialize(options)
  22. 21 super(nil, options)
  23. 21 @resolver_options = @options.resolver_options
  24. 21 resolv_options = @resolver_options.dup
  25. 21 timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
  26. 21 @_timeouts = Array(timeouts)
  27. 42 @timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
  28. 21 resolv_options.delete(:cache)
  29. 21 @connections = []
  30. 21 @queries = []
  31. 21 @ips = []
  32. 21 @pipe_mutex = Thread::Mutex.new
  33. 21 @state = :idle
  34. end
  35. 23 def resolvers
  36. 42 return enum_for(__method__) unless block_given?
  37. 21 yield self
  38. end
  39. 23 def connections
  40. EMPTY
  41. end
  42. 23 def close
  43. 21 transition(:closed)
  44. end
  45. 23 def closed?
  46. 63 @state == :closed
  47. end
  48. 23 def to_io
  49. 21 @pipe_read.to_io
  50. end
  51. 23 def call
  52. 21 case @state
  53. when :open
  54. 21 consume
  55. end
  56. nil
  57. end
  58. 23 def interests
  59. 21 return if @queries.empty?
  60. 21 :r
  61. end
  62. 23 def timeout
  63. 21 return unless @queries.empty?
  64. _, connection = @queries.first
  65. return unless connection
  66. @timeouts[connection.origin.host].first
  67. end
  68. 23 def <<(connection)
  69. 21 @connections << connection
  70. 21 resolve
  71. end
  72. 23 def handle_socket_timeout(interval)
  73. error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
  74. error.set_backtrace(caller)
  75. on_error(error)
  76. end
  77. 23 private
  78. 23 def transition(nextstate)
  79. 42 case nextstate
  80. when :idle
  81. @timeouts.clear
  82. when :open
  83. 21 return unless @state == :idle
  84. 21 @pipe_read, @pipe_write = ::IO.pipe
  85. when :closed
  86. 21 return unless @state == :open
  87. 21 @pipe_write.close
  88. 21 @pipe_read.close
  89. end
  90. 42 @state = nextstate
  91. end
  92. 23 def consume
  93. 42 return if @connections.empty?
  94. 63 while @pipe_read.ready? && (event = @pipe_read.getbyte)
  95. 21 case event
  96. when DONE
  97. 20 *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
  98. 10 @queries.delete(pair)
  99. 10 family, connection = pair
  100. 10 emit_addresses(connection, family, addrs)
  101. when ERROR
  102. 22 *pair, error = @pipe_mutex.synchronize { @ips.pop }
  103. 11 @queries.delete(pair)
  104. 11 family, connection = pair
  105. 11 emit_resolve_error(connection, connection.origin.host, error)
  106. end
  107. 21 @connections.delete(connection) if @queries.empty?
  108. end
  109. 42 return emit(:close, self) if @connections.empty?
  110. 21 resolve
  111. end
  112. 23 def resolve(connection = @connections.first)
  113. 42 raise Error, "no URI to resolve" unless connection
  114. 42 return unless @queries.empty?
  115. 21 hostname = connection.origin.host
  116. 21 scheme = connection.origin.scheme
  117. 21 log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
  118. 21 transition(:open)
  119. 21 connection.options.ip_families.each do |family|
  120. 21 @queries << [family, connection]
  121. end
  122. 21 async_resolve(connection, hostname, scheme)
  123. 21 consume
  124. end
  125. 23 def async_resolve(connection, hostname, scheme)
  126. 21 families = connection.options.ip_families
  127. 21 log { "resolver: query for #{hostname}" }
  128. 21 timeouts = @timeouts[connection.origin.host]
  129. 21 resolve_timeout = timeouts.first
  130. 21 Thread.start do
  131. 21 Thread.current.report_on_exception = false
  132. begin
  133. 21 addrs = if resolve_timeout
  134. 21 Timeout.timeout(resolve_timeout) do
  135. 21 __addrinfo_resolve(hostname, scheme)
  136. end
  137. else
  138. __addrinfo_resolve(hostname, scheme)
  139. end
  140. 10 addrs = addrs.sort_by(&:afamily).group_by(&:afamily)
  141. 10 families.each do |family|
  142. 10 addresses = addrs[family]
  143. 10 next unless addresses
  144. 10 addresses.map!(&:ip_address)
  145. 10 addresses.uniq!
  146. 10 @pipe_mutex.synchronize do
  147. 10 @ips.unshift([family, connection, addresses])
  148. 10 @pipe_write.putc(DONE) unless @pipe_write.closed?
  149. end
  150. end
  151. rescue StandardError => e
  152. 11 if e.is_a?(Timeout::Error)
  153. 1 timeouts.shift
  154. 1 retry unless timeouts.empty?
  155. 1 e = ResolveTimeoutError.new(resolve_timeout, e.message)
  156. 1 e.set_backtrace(e.backtrace)
  157. end
  158. 11 @pipe_mutex.synchronize do
  159. 11 families.each do |family|
  160. 11 @ips.unshift([family, connection, e])
  161. 11 @pipe_write.putc(ERROR) unless @pipe_write.closed?
  162. end
  163. end
  164. end
  165. end
  166. end
  167. 23 def __addrinfo_resolve(host, scheme)
  168. 21 Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
  169. end
  170. end
  171. end

lib/httpx/response.rb

100.0% lines covered

105 relevant lines. 105 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "objspace"
  3. 23 require "stringio"
  4. 23 require "tempfile"
  5. 23 require "fileutils"
  6. 23 require "forwardable"
  7. 23 module HTTPX
  8. # Defines a HTTP response is handled internally, with a few properties exposed as attributes.
  9. #
  10. # It delegates the following methods to the corresponding HTTPX::Request:
  11. #
  12. # * HTTPX::Request#uri
  13. # * HTTPX::Request#peer_address
  14. #
  15. # It implements (indirectly, via the +body+) the IO write protocol to internally buffer payloads.
  16. #
  17. # It implements the IO reader protocol in order for users to buffer/stream it, acts as an enumerable
  18. # (of payload chunks).
  19. #
  20. 23 class Response
  21. 23 extend Forwardable
  22. 23 include Callbacks
  23. # the HTTP response status code
  24. 23 attr_reader :status
  25. # an HTTPX::Headers object containing the response HTTP headers.
  26. 23 attr_reader :headers
  27. # a HTTPX::Response::Body object wrapping the response body. The following methods are delegated to it:
  28. #
  29. # * HTTPX::Response::Body#to_s
  30. # * HTTPX::Response::Body#to_str
  31. # * HTTPX::Response::Body#read
  32. # * HTTPX::Response::Body#copy_to
  33. # * HTTPX::Response::Body#close
  34. 23 attr_reader :body
  35. # The HTTP protocol version used to fetch the response.
  36. 23 attr_reader :version
  37. # returns the response body buffered in a string.
  38. 23 def_delegator :@body, :to_s
  39. 23 def_delegator :@body, :to_str
  40. # implements the IO reader +#read+ interface.
  41. 23 def_delegator :@body, :read
  42. # copies the response body to a different location.
  43. 23 def_delegator :@body, :copy_to
  44. # closes the body.
  45. 23 def_delegator :@body, :close
  46. # the corresponding request uri.
  47. 23 def_delegator :@request, :uri
  48. # the IP address of the peer server.
  49. 23 def_delegator :@request, :peer_address
  50. # inits the instance with the corresponding +request+ to this response, an the
  51. # response HTTP +status+, +version+ and HTTPX::Headers instance of +headers+.
  52. 23 def initialize(request, status, version, headers)
  53. 6343 @request = request
  54. 6343 @options = request.options
  55. 6343 @version = version
  56. 6343 @status = Integer(status)
  57. 6343 @headers = @options.headers_class.new(headers)
  58. 6343 @body = @options.response_body_class.new(self, @options)
  59. 6343 @finished = complete?
  60. 6343 @content_type = nil
  61. end
  62. # merges headers defined in +h+ into the response headers.
  63. 23 def merge_headers(h)
  64. 167 @headers = @headers.merge(h)
  65. end
  66. # writes +data+ chunk into the response body.
  67. 23 def <<(data)
  68. 8239 @body.write(data)
  69. end
  70. # returns the HTTPX::ContentType for the response, as per what's declared in the content-type header.
  71. #
  72. # response.content_type #=> #<HTTPX::ContentType:xxx @header_value="text/plain">
  73. # response.content_type.mime_type #=> "text/plain"
  74. 23 def content_type
  75. 6568 @content_type ||= ContentType.new(@headers["content-type"])
  76. end
  77. # returns whether the response has been fully fetched.
  78. 23 def finished?
  79. 3361 @finished
  80. end
  81. # marks the response as finished, freezes the headers.
  82. 23 def finish!
  83. 3279 @finished = true
  84. 3279 @headers.freeze
  85. end
  86. # returns whether the response contains body payload.
  87. 23 def bodyless?
  88. 6343 @request.verb == "HEAD" ||
  89. @status < 200 || # informational response
  90. @status == 204 ||
  91. @status == 205 ||
  92. @status == 304 || begin
  93. 6010 content_length = @headers["content-length"]
  94. 6010 return false if content_length.nil?
  95. 5202 content_length == "0"
  96. end
  97. end
  98. 23 def complete?
  99. 6343 bodyless? || (@request.verb == "CONNECT" && @status == 200)
  100. end
  101. skipped # :nocov:
  102. skipped def inspect
  103. skipped "#<Response:#{object_id} " \
  104. skipped "HTTP/#{version} " \
  105. skipped "@status=#{@status} " \
  106. skipped "@headers=#{@headers} " \
  107. skipped "@body=#{@body.bytesize}>"
  108. skipped end
  109. skipped # :nocov:
  110. # returns an instance of HTTPX::HTTPError if the response has a 4xx or 5xx
  111. # status code, or nothing.
  112. #
  113. # ok_response.error #=> nil
  114. # not_found_response.error #=> HTTPX::HTTPError instance, status 404
  115. 23 def error
  116. 412 return if @status < 400
  117. 41 HTTPError.new(self)
  118. end
  119. # it raises the exception returned by +error+, or itself otherwise.
  120. #
  121. # ok_response.raise_for_status #=> ok_response
  122. # not_found_response.raise_for_status #=> raises HTTPX::HTTPError exception
  123. 23 def raise_for_status
  124. 382 return self unless (err = error)
  125. 29 raise err
  126. end
  127. # decodes the response payload into a ruby object **if** the payload is valid json.
  128. #
  129. # response.json #≈> { "foo" => "bar" } for "{\"foo\":\"bar\"}" payload
  130. # response.json(symbolize_names: true) #≈> { foo: "bar" } for "{\"foo\":\"bar\"}" payload
  131. 23 def json(*args)
  132. 57 decode(Transcoder::JSON, *args)
  133. end
  134. # decodes the response payload into a ruby object **if** the payload is valid
  135. # "application/x-www-urlencoded" or "multipart/form-data".
  136. 23 def form
  137. 48 decode(Transcoder::Form)
  138. end
  139. # decodes the response payload into a Nokogiri::XML::Node object **if** the payload is valid
  140. # "application/xml" (requires the "nokogiri" gem).
  141. 23 def xml
  142. 6 decode(Transcoder::Xml)
  143. end
  144. 23 private
  145. # decodes the response payload using the given +transcoder+, which implements the decoding logic.
  146. #
  147. # +transcoder+ must implement the internal transcoder API, i.e. respond to <tt>decode(HTTPX::Response response)</tt>,
  148. # which returns a decoder which responds to <tt>call(HTTPX::Response response, **kwargs)</tt>
  149. 23 def decode(transcoder, *args)
  150. # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
  151. 111 decoder = transcoder.decode(self)
  152. 99 raise Error, "no decoder available for \"#{transcoder}\"" unless decoder
  153. 99 @body.rewind
  154. 99 decoder.call(self, *args)
  155. end
  156. end
  157. # Helper class which decodes the HTTP "content-type" header.
  158. 23 class ContentType
  159. 23 MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
  160. 23 CHARSET_RE = /;\s*charset=([^;]+)/i.freeze
  161. 23 def initialize(header_value)
  162. 6538 @header_value = header_value
  163. end
  164. # returns the mime type declared in the header.
  165. #
  166. # ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
  167. 23 def mime_type
  168. 111 return @mime_type if defined?(@mime_type)
  169. 81 m = @header_value.to_s[MIME_TYPE_RE, 1]
  170. 81 m && @mime_type = m.strip.downcase
  171. end
  172. # returns the charset declared in the header.
  173. #
  174. # ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
  175. # ContentType.new("text/plain").charset #=> nil
  176. 23 def charset
  177. 6457 return @charset if defined?(@charset)
  178. 6457 m = @header_value.to_s[CHARSET_RE, 1]
  179. 6457 m && @charset = m.strip.delete('"')
  180. end
  181. end
  182. # Wraps an error which has happened while processing an HTTP Request. It has partial
  183. # public API parity with HTTPX::Response, so users should rely on it to infer whether
  184. # the returned response is one or the other.
  185. #
  186. # response = HTTPX.get("https://some-domain/path") #=> response is HTTPX::Response or HTTPX::ErrorResponse
  187. # response.raise_for_status #=> raises if it wraps an error
  188. 23 class ErrorResponse
  189. 23 include Loggable
  190. 23 extend Forwardable
  191. # the corresponding HTTPX::Request instance.
  192. 23 attr_reader :request
  193. # the HTTPX::Response instance, when there is one (i.e. error happens fetching the response).
  194. 23 attr_reader :response
  195. # the wrapped exception.
  196. 23 attr_reader :error
  197. # the request uri
  198. 23 def_delegator :@request, :uri
  199. # the IP address of the peer server.
  200. 23 def_delegator :@request, :peer_address
  201. 23 def initialize(request, error, options)
  202. 818 @request = request
  203. 818 @response = request.response if request.response.is_a?(Response)
  204. 818 @error = error
  205. 818 @options = Options.new(options)
  206. 818 log_exception(@error)
  207. end
  208. # returns the exception full message.
  209. 23 def to_s
  210. 7 @error.full_message(highlight: false)
  211. end
  212. # closes the error resources.
  213. 23 def close
  214. 18 @response.close if @response && @response.respond_to?(:close)
  215. end
  216. # always true for error responses.
  217. 23 def finished?
  218. 6 true
  219. end
  220. # raises the wrapped exception.
  221. 23 def raise_for_status
  222. 58 raise @error
  223. end
  224. # buffers lost chunks to error response
  225. 23 def <<(data)
  226. 6 @response << data
  227. end
  228. end
  229. end
  230. 23 require_relative "response/body"
  231. 23 require_relative "response/buffer"
  232. 23 require_relative "pmatch_extensions" if RUBY_VERSION >= "2.7.0"

lib/httpx/response/body.rb

100.0% lines covered

109 relevant lines. 109 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. # Implementation of the HTTP Response body as a buffer which implements the IO writer protocol
  4. # (for buffering the response payload), the IO reader protocol (for consuming the response payload),
  5. # and can be iterated over (via #each, which yields the payload in chunks).
  6. 23 class Response::Body
  7. # the payload encoding (i.e. "utf-8", "ASCII-8BIT")
  8. 23 attr_reader :encoding
  9. # Array of encodings contained in the response "content-encoding" header.
  10. 23 attr_reader :encodings
  11. # initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
  12. 23 def initialize(response, options)
  13. 6457 @response = response
  14. 6457 @headers = response.headers
  15. 6457 @options = options
  16. 6457 @window_size = options.window_size
  17. 6457 @encoding = response.content_type.charset || Encoding::BINARY
  18. 6457 @encodings = []
  19. 6457 @length = 0
  20. 6457 @buffer = nil
  21. 6457 @reader = nil
  22. 6457 @state = :idle
  23. 6457 initialize_inflaters
  24. end
  25. 23 def initialize_dup(other)
  26. 24 super
  27. 24 @buffer = other.instance_variable_get(:@buffer).dup
  28. end
  29. 23 def closed?
  30. 222 @state == :closed
  31. end
  32. # write the response payload +chunk+ into the buffer. Inflates the chunk when required
  33. # and supported.
  34. 23 def write(chunk)
  35. 8110 return if @state == :closed
  36. 8110 return 0 if chunk.empty?
  37. 7820 chunk = decode_chunk(chunk)
  38. 7820 size = chunk.bytesize
  39. 6684 @length += size
  40. 7820 transition(:open)
  41. 7820 @buffer.write(chunk)
  42. 7820 @response.emit(:chunk_received, chunk)
  43. 7808 size
  44. end
  45. # reads a chunk from the payload (implementation of the IO reader protocol).
  46. 23 def read(*args)
  47. 123 return unless @buffer
  48. 123 unless @reader
  49. 81 rewind
  50. 81 @reader = @buffer
  51. end
  52. 123 @reader.read(*args)
  53. end
  54. # size of the decoded response payload. May differ from "content-length" header if
  55. # response was encoded over-the-wire.
  56. 23 def bytesize
  57. 133 @length
  58. end
  59. # yields the payload in chunks.
  60. 23 def each
  61. 36 return enum_for(__method__) unless block_given?
  62. begin
  63. 24 if @buffer
  64. 24 rewind
  65. 68 while (chunk = @buffer.read(@window_size))
  66. 24 yield(chunk.force_encoding(@encoding))
  67. end
  68. end
  69. ensure
  70. 24 close
  71. end
  72. end
  73. # returns the declared filename in the "contennt-disposition" header, when present.
  74. 23 def filename
  75. 36 return unless @headers.key?("content-disposition")
  76. 30 Utils.get_filename(@headers["content-disposition"])
  77. end
  78. # returns the full response payload as a string.
  79. 23 def to_s
  80. 3227 return "".b unless @buffer
  81. 2991 @buffer.to_s
  82. end
  83. 23 alias_method :to_str, :to_s
  84. # whether the payload is empty.
  85. 23 def empty?
  86. 24 @length.zero?
  87. end
  88. # copies the payload to +dest+.
  89. #
  90. # body.copy_to("path/to/file")
  91. # body.copy_to(Pathname.new("path/to/file"))
  92. # body.copy_to(File.new("path/to/file"))
  93. 23 def copy_to(dest)
  94. 36 return unless @buffer
  95. 36 rewind
  96. 36 if dest.respond_to?(:path) && @buffer.respond_to?(:path)
  97. 6 FileUtils.mv(@buffer.path, dest.path)
  98. else
  99. 30 ::IO.copy_stream(@buffer, dest)
  100. end
  101. end
  102. # closes/cleans the buffer, resets everything
  103. 23 def close
  104. 501 if @buffer
  105. 275 @buffer.close
  106. 275 @buffer = nil
  107. end
  108. 501 @length = 0
  109. 501 transition(:closed)
  110. end
  111. 23 def ==(other)
  112. 82 object_id == other.object_id || begin
  113. 82 if other.respond_to?(:read)
  114. 60 _with_same_buffer_pos { FileUtils.compare_stream(@buffer, other) }
  115. else
  116. 52 to_s == other.to_s
  117. end
  118. end
  119. end
  120. skipped # :nocov:
  121. skipped def inspect
  122. skipped "#<HTTPX::Response::Body:#{object_id} " \
  123. skipped "@state=#{@state} " \
  124. skipped "@length=#{@length}>"
  125. skipped end
  126. skipped # :nocov:
  127. # rewinds the response payload buffer.
  128. 23 def rewind
  129. 270 return unless @buffer
  130. # in case there's some reading going on
  131. 270 @reader = nil
  132. 270 @buffer.rewind
  133. end
  134. 23 private
  135. # prepares inflaters for the advertised encodings in "content-encoding" header.
  136. 23 def initialize_inflaters
  137. 6457 @inflaters = nil
  138. 6457 return unless @headers.key?("content-encoding")
  139. 132 return unless @options.decompress_response_body
  140. 120 @inflaters = @headers.get("content-encoding").filter_map do |encoding|
  141. 120 next if encoding == "identity"
  142. 120 inflater = self.class.initialize_inflater_by_encoding(encoding, @response)
  143. # do not uncompress if there is no decoder available. In fact, we can't reliably
  144. # continue decompressing beyond that, so ignore.
  145. 120 break unless inflater
  146. 120 @encodings << encoding
  147. 120 inflater
  148. end
  149. end
  150. # passes the +chunk+ through all inflaters to decode it.
  151. 23 def decode_chunk(chunk)
  152. 40 @inflaters.reverse_each do |inflater|
  153. 339 chunk = inflater.call(chunk)
  154. 7936 end if @inflaters
  155. 7936 chunk
  156. end
  157. # tries transitioning the body STM to the +nextstate+.
  158. 23 def transition(nextstate)
  159. 7035 case nextstate
  160. when :open
  161. 7820 return unless @state == :idle
  162. 4958 @buffer = Response::Buffer.new(
  163. threshold_size: @options.body_threshold_size,
  164. bytesize: @length,
  165. encoding: @encoding
  166. )
  167. when :closed
  168. 406 return if @state == :closed
  169. end
  170. 5364 @state = nextstate
  171. end
  172. 23 def _with_same_buffer_pos # :nodoc:
  173. 30 return yield unless @buffer && @buffer.respond_to?(:pos)
  174. # @type ivar @buffer: StringIO | Tempfile
  175. 30 current_pos = @buffer.pos
  176. 30 @buffer.rewind
  177. begin
  178. 30 yield
  179. ensure
  180. 30 @buffer.pos = current_pos
  181. end
  182. end
  183. 23 class << self
  184. 23 def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
  185. 105 case encoding
  186. when "gzip"
  187. 108 Transcoder::GZIP.decode(response, **kwargs)
  188. when "deflate"
  189. 12 Transcoder::Deflate.decode(response, **kwargs)
  190. end
  191. end
  192. end
  193. end
  194. end

lib/httpx/response/buffer.rb

100.0% lines covered

49 relevant lines. 49 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "delegate"
  3. 23 require "stringio"
  4. 23 require "tempfile"
  5. 23 module HTTPX
  6. # wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
  7. 23 class Response::Buffer < SimpleDelegator
  8. # initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
  9. # the initial +bytesize+, and the +encoding+.
  10. 23 def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
  11. 5012 @threshold_size = threshold_size
  12. 5012 @bytesize = bytesize
  13. 5012 @encoding = encoding
  14. 5012 @buffer = StringIO.new("".b)
  15. 5012 super(@buffer)
  16. end
  17. 23 def initialize_dup(other)
  18. 24 super
  19. 24 @buffer = other.instance_variable_get(:@buffer).dup
  20. end
  21. # size in bytes of the buffered content.
  22. 23 def size
  23. 206 @bytesize
  24. end
  25. # writes the +chunk+ into the buffer.
  26. 23 def write(chunk)
  27. 6774 @bytesize += chunk.bytesize
  28. 7918 try_upgrade_buffer
  29. 7918 @buffer.write(chunk)
  30. end
  31. # returns the buffered content as a string.
  32. 23 def to_s
  33. 2525 case @buffer
  34. when StringIO
  35. begin
  36. 2931 @buffer.string.force_encoding(@encoding)
  37. 4 rescue ArgumentError
  38. 6 @buffer.string
  39. end
  40. when Tempfile
  41. 60 rewind
  42. 120 content = _with_same_buffer_pos { @buffer.read }
  43. begin
  44. 60 content.force_encoding(@encoding)
  45. 4 rescue ArgumentError # ex: unknown encoding name - utf
  46. 6 content
  47. end
  48. end
  49. end
  50. # closes the buffer.
  51. 23 def close
  52. 275 @buffer.close
  53. 275 @buffer.unlink if @buffer.respond_to?(:unlink)
  54. end
  55. 23 private
  56. # initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
  57. # has been reached.
  58. 23 def try_upgrade_buffer
  59. 7918 return unless @bytesize > @threshold_size
  60. 242 return if @buffer.is_a?(Tempfile)
  61. 99 aux = @buffer
  62. 99 @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  63. 99 if aux
  64. 99 aux.rewind
  65. 99 ::IO.copy_stream(aux, @buffer)
  66. 99 aux.close
  67. end
  68. 99 __setobj__(@buffer)
  69. end
  70. 23 def _with_same_buffer_pos # :nodoc:
  71. 60 current_pos = @buffer.pos
  72. 60 @buffer.rewind
  73. begin
  74. 60 yield
  75. ensure
  76. 60 @buffer.pos = current_pos
  77. end
  78. end
  79. end
  80. end

lib/httpx/selector.rb

86.36% lines covered

66 relevant lines. 57 lines covered and 9 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "io/wait"
  3. 23 class HTTPX::Selector
  4. 23 READABLE = %i[rw r].freeze
  5. 23 WRITABLE = %i[rw w].freeze
  6. 23 private_constant :READABLE
  7. 23 private_constant :WRITABLE
  8. 23 def initialize
  9. 500 @selectables = []
  10. end
  11. # deregisters +io+ from selectables.
  12. 23 def deregister(io)
  13. 10955 @selectables.delete(io)
  14. end
  15. # register +io+.
  16. 23 def register(io)
  17. 6183 return if @selectables.include?(io)
  18. 5971 @selectables << io
  19. end
  20. 23 private
  21. 23 def select_many(interval, &block)
  22. 7678 selectables, r, w = nil
  23. # first, we group IOs based on interest type. On call to #interests however,
  24. # things might already happen, and new IOs might be registered, so we might
  25. # have to start all over again. We do this until we group all selectables
  26. begin
  27. 7678 loop do
  28. begin
  29. 7678 r = nil
  30. 7678 w = nil
  31. 7678 selectables = @selectables
  32. 7678 @selectables = []
  33. 7678 selectables.delete_if do |io|
  34. 20079 interests = io.interests
  35. 20079 (r ||= []) << io if READABLE.include?(interests)
  36. 20079 (w ||= []) << io if WRITABLE.include?(interests)
  37. 20079 io.state == :closed
  38. end
  39. 7678 if @selectables.empty?
  40. 7678 @selectables = selectables
  41. # do not run event loop if there's nothing to wait on.
  42. # this might happen if connect failed and connection was unregistered.
  43. 7678 return if (!r || r.empty?) && (!w || w.empty?) && !selectables.empty?
  44. 3248 break
  45. else
  46. @selectables.concat(selectables)
  47. end
  48. rescue StandardError
  49. @selectables = selectables if selectables
  50. raise
  51. end
  52. end
  53. # TODO: what to do if there are no selectables?
  54. 3248 readers, writers = IO.select(r, w, nil, interval)
  55. 3248 if readers.nil? && writers.nil? && interval
  56. 278 [*r, *w].each { |io| io.handle_socket_timeout(interval) }
  57. 164 return
  58. end
  59. rescue IOError, SystemCallError
  60. @selectables.reject!(&:closed?)
  61. retry
  62. end
  63. 3084 if writers
  64. 407 readers.each do |io|
  65. 1484 yield io
  66. # so that we don't yield 2 times
  67. 1483 writers.delete(io)
  68. 3084 end if readers
  69. 3083 writers.each(&block)
  70. else
  71. readers.each(&block) if readers
  72. end
  73. end
  74. 23 def select_one(interval)
  75. 3684235 io = @selectables.first
  76. 3684235 return unless io
  77. 3684235 interests = io.interests
  78. 3684235 result = case interests
  79. 6633 when :r then io.to_io.wait_readable(interval)
  80. 5725 when :w then io.to_io.wait_writable(interval)
  81. when :rw then io.to_io.wait(interval, :read_write)
  82. 3671877 when nil then return
  83. end
  84. 12358 unless result || interval.nil?
  85. 308 io.handle_socket_timeout(interval)
  86. 303 return
  87. end
  88. # raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
  89. 12050 yield io
  90. rescue IOError, SystemCallError
  91. @selectables.reject!(&:closed?)
  92. raise unless @selectables.empty?
  93. end
  94. 23 def select(interval, &block)
  95. # do not cause an infinite loop here.
  96. #
  97. # this may happen if timeout calculation actually triggered an error which causes
  98. # the connections to be reaped (such as the total timeout error) before #select
  99. # gets called.
  100. 3691913 return if interval.nil? && @selectables.empty?
  101. 3691913 return select_one(interval, &block) if @selectables.size == 1
  102. 7678 select_many(interval, &block)
  103. end
  104. 23 public :select
  105. end

lib/httpx/session.rb

95.81% lines covered

167 relevant lines. 160 lines covered and 7 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. # Class implementing the APIs being used publicly.
  4. #
  5. # HTTPX.get(..) #=> delegating to an internal HTTPX::Session object.
  6. # HTTPX.plugin(..).get(..) #=> creating an intermediate HTTPX::Session with plugin, then sending the GET request
  7. 23 class Session
  8. 23 include Loggable
  9. 23 include Chainable
  10. 23 EMPTY_HASH = {}.freeze
  11. # initializes the session with a set of +options+, which will be shared by all
  12. # requests sent from it.
  13. #
  14. # When pass a block, it'll yield itself to it, then closes after the block is evaluated.
  15. 23 def initialize(options = EMPTY_HASH, &blk)
  16. 7403 @options = self.class.default_options.merge(options)
  17. 7403 @responses = {}
  18. 7403 @persistent = @options.persistent
  19. 7403 wrap(&blk) if blk
  20. end
  21. # Yields itself the block, then closes it after the block is evaluated.
  22. #
  23. # session.wrap do |http|
  24. # http.get("https://wikipedia.com")
  25. # end # wikipedia connection closes here
  26. 23 def wrap
  27. 377 prev_persistent = @persistent
  28. 377 @persistent = true
  29. 377 pool.wrap do
  30. begin
  31. 377 yield self
  32. ensure
  33. 377 @persistent = prev_persistent
  34. 377 close unless @persistent
  35. end
  36. end
  37. end
  38. # closes all the active connections from the session
  39. 23 def close(*args)
  40. 4796 pool.close(*args)
  41. end
  42. # performs one, or multple requests; it accepts:
  43. #
  44. # 1. one or multiple HTTPX::Request objects;
  45. # 2. an HTTP verb, then a sequence of URIs or URI/options tuples;
  46. # 3. one or multiple HTTP verb / uri / (optional) options tuples;
  47. #
  48. # when present, the set of +options+ kwargs is applied to all of the
  49. # sent requests.
  50. #
  51. # respectively returns a single HTTPX::Response response, or all of them in an Array, in the same order.
  52. #
  53. # resp1 = session.request(req1)
  54. # resp1, resp2 = session.request(req1, req2)
  55. # resp1 = session.request("GET", "https://server.org/a")
  56. # resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"])
  57. # resp1, resp2 = session.request(["GET", "https://server.org/a"], ["GET", "https://server.org/b"])
  58. # resp1 = session.request("POST", "https://server.org/a", form: { "foo" => "bar" })
  59. # resp1, resp2 = session.request(["POST", "https://server.org/a", form: { "foo" => "bar" }], ["GET", "https://server.org/b"])
  60. # resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"], headers: { "x-api-token" => "TOKEN" })
  61. #
  62. 23 def request(*args, **options)
  63. 5005 raise ArgumentError, "must perform at least one request" if args.empty?
  64. 5005 requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
  65. 5004 responses = send_requests(*requests)
  66. 4897 return responses.first if responses.size == 1
  67. 147 responses
  68. end
  69. # returns a HTTP::Request instance built from the HTTP +verb+, the request +uri+, and
  70. # the optional set of request-specific +options+. This request **must** be sent through
  71. # the same session it was built from.
  72. #
  73. # req = session.build_request("GET", "https://server.com")
  74. # resp = session.request(req)
  75. 23 def build_request(verb, uri, options = EMPTY_HASH)
  76. 6117 rklass = @options.request_class
  77. 6117 options = @options.merge(options) unless options.is_a?(Options)
  78. 6117 request = rklass.new(verb, uri, options)
  79. 6116 request.persistent = @persistent
  80. 6116 set_request_callbacks(request)
  81. 6116 request
  82. end
  83. 23 private
  84. # returns the HTTPX::Pool object which manages the networking required to
  85. # perform requests.
  86. 23 def pool
  87. 3700932 Thread.current[:httpx_connection_pool] ||= Pool.new
  88. end
  89. # callback executed when a response for a given request has been received.
  90. 23 def on_response(request, response)
  91. 5490 @responses[request] = response
  92. end
  93. # callback executed when an HTTP/2 promise frame has been received.
  94. 23 def on_promise(_, stream)
  95. 6 log(level: 2) { "#{stream.id}: refusing stream!" }
  96. 6 stream.refuse
  97. end
  98. # returns the corresponding HTTP::Response to the given +request+ if it has been received.
  99. 23 def fetch_response(request, _, _)
  100. 3697331 @responses.delete(request)
  101. end
  102. # returns the HTTPX::Connection through which the +request+ should be sent through.
  103. 23 def find_connection(request, connections, options)
  104. 6126 uri = request.uri
  105. 6126 connection = pool.find_connection(uri, options) || init_connection(uri, options)
  106. 6074 unless connections.nil? || connections.include?(connection)
  107. 4832 connections << connection
  108. 4832 set_connection_callbacks(connection, connections, options)
  109. end
  110. 6074 connection
  111. end
  112. 23 def send_request(request, connections, options = request.options)
  113. 6404 error = catch(:resolve_error) do
  114. 6404 connection = find_connection(request, connections, options)
  115. 6334 connection.send(request)
  116. end
  117. 6381 return unless error.is_a?(Error)
  118. 52 request.emit(:response, ErrorResponse.new(request, error, options))
  119. end
  120. # sets the callbacks on the +connection+ required to process certain specific
  121. # connection lifecycle events which deal with request rerouting.
  122. 23 def set_connection_callbacks(connection, connections, options, cloned: false)
  123. 5127 connection.only(:misdirected) do |misdirected_request|
  124. 6 other_connection = connection.create_idle(ssl: { alpn_protocols: %w[http/1.1] })
  125. 6 other_connection.merge(connection)
  126. 6 catch(:coalesced) do
  127. 6 pool.init_connection(other_connection, options)
  128. end
  129. 6 set_connection_callbacks(other_connection, connections, options)
  130. 6 connections << other_connection
  131. 6 misdirected_request.transition(:idle)
  132. 6 other_connection.send(misdirected_request)
  133. end
  134. 5127 connection.only(:altsvc) do |alt_origin, origin, alt_params|
  135. 6 other_connection = build_altsvc_connection(connection, connections, alt_origin, origin, alt_params, options)
  136. 6 connections << other_connection if other_connection
  137. end
  138. 5127 connection.only(:cloned) do |cloned_conn|
  139. set_connection_callbacks(cloned_conn, connections, options, cloned: true)
  140. connections << cloned_conn
  141. 5127 end unless cloned
  142. end
  143. # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
  144. 23 def build_altsvc_connection(existing_connection, connections, alt_origin, origin, alt_params, options)
  145. # do not allow security downgrades on altsvc negotiation
  146. 6 return if existing_connection.origin.scheme == "https" && alt_origin.scheme != "https"
  147. 6 altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
  148. # altsvc already exists, somehow it wasn't advertised, probably noop
  149. 6 return unless altsvc
  150. 6 alt_options = options.merge(ssl: options.ssl.merge(hostname: URI(origin).host))
  151. 6 connection = pool.find_connection(alt_origin, alt_options) || init_connection(alt_origin, alt_options)
  152. # advertised altsvc is the same origin being used, ignore
  153. 6 return if connection == existing_connection
  154. 6 connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
  155. 6 set_connection_callbacks(connection, connections, alt_options)
  156. 6 log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
  157. 6 connection.merge(existing_connection)
  158. 6 existing_connection.terminate
  159. 6 connection
  160. rescue UnsupportedSchemeError
  161. altsvc["noop"] = true
  162. nil
  163. end
  164. # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
  165. 23 def build_requests(*args, options)
  166. 4620 request_options = @options.merge(options)
  167. 4620 requests = if args.size == 1
  168. 59 reqs = args.first
  169. 59 reqs.map do |verb, uri, opts = EMPTY_HASH|
  170. 118 build_request(verb, uri, request_options.merge(opts))
  171. end
  172. else
  173. 4561 verb, uris = args
  174. 4561 if uris.respond_to?(:each)
  175. 4381 uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
  176. 5090 build_request(verb, uri, request_options.merge(opts))
  177. end
  178. else
  179. 180 [build_request(verb, uris, request_options)]
  180. end
  181. end
  182. 4619 raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
  183. 4619 requests
  184. end
  185. 23 def set_request_callbacks(request)
  186. 6116 request.on(:response, &method(:on_response).curry(2)[request])
  187. 6116 request.on(:promise, &method(:on_promise))
  188. end
  189. 23 def init_connection(uri, options)
  190. 5076 connection = options.connection_class.new(uri, options)
  191. 5064 catch(:coalesced) do
  192. 5064 pool.init_connection(connection, options)
  193. 5019 connection
  194. end
  195. end
  196. # sends an array of HTTPX::Request +requests+, returns the respective array of HTTPX::Response objects.
  197. 23 def send_requests(*requests)
  198. 5078 connections = _send_requests(requests)
  199. 5055 receive_requests(requests, connections)
  200. end
  201. # sends an array of HTTPX::Request objects
  202. 23 def _send_requests(requests)
  203. 5078 connections = []
  204. 5078 requests.each do |request|
  205. 5835 send_request(request, connections)
  206. end
  207. 5055 connections
  208. end
  209. # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
  210. 23 def receive_requests(requests, connections)
  211. # @type var responses: Array[response]
  212. 5055 responses = []
  213. begin
  214. # guarantee ordered responses
  215. 5055 loop do
  216. 5818 request = requests.first
  217. 5818 return responses unless request
  218. 4147445 catch(:coalesced) { pool.next_tick } until (response = fetch_response(request, connections, request.options))
  219. 5734 responses << response
  220. 5734 requests.shift
  221. 5734 break if requests.empty?
  222. 763 next unless pool.empty?
  223. # in some cases, the pool of connections might have been drained because there was some
  224. # handshake error, and the error responses have already been emitted, but there was no
  225. # opportunity to traverse the requests, hence we're returning only a fraction of the errors
  226. # we were supposed to. This effectively fetches the existing responses and return them.
  227. while (request = requests.shift)
  228. responses << fetch_response(request, connections, request.options)
  229. end
  230. break
  231. end
  232. 4971 responses
  233. ensure
  234. 5055 if @persistent
  235. 779 pool.deactivate(connections)
  236. else
  237. 4276 close(connections)
  238. end
  239. end
  240. end
  241. 23 @default_options = Options.new
  242. 23 @default_options.freeze
  243. 23 @plugins = []
  244. 23 class << self
  245. 23 attr_reader :default_options
  246. 23 def inherited(klass)
  247. 3642 super
  248. 3642 klass.instance_variable_set(:@default_options, @default_options)
  249. 3642 klass.instance_variable_set(:@plugins, @plugins.dup)
  250. 3642 klass.instance_variable_set(:@callbacks, @callbacks.dup)
  251. end
  252. # returns a new HTTPX::Session instance, with the plugin pointed by +pl+ loaded.
  253. #
  254. # session_with_retries = session.plugin(:retries)
  255. # session_with_custom = session.plugin(CustomPlugin)
  256. #
  257. 23 def plugin(pl, options = nil, &block)
  258. # raise Error, "Cannot add a plugin to a frozen config" if frozen?
  259. 4974 pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
  260. 4974 if !@plugins.include?(pl)
  261. 4815 @plugins << pl
  262. 4815 pl.load_dependencies(self, &block) if pl.respond_to?(:load_dependencies)
  263. 4815 @default_options = @default_options.dup
  264. 4815 include(pl::InstanceMethods) if defined?(pl::InstanceMethods)
  265. 4815 extend(pl::ClassMethods) if defined?(pl::ClassMethods)
  266. 4815 opts = @default_options
  267. 4815 opts.extend_with_plugin_classes(pl)
  268. 4815 if defined?(pl::OptionsMethods)
  269. 1815 (pl::OptionsMethods.instance_methods - Object.instance_methods).each do |meth|
  270. 5358 opts.options_class.method_added(meth)
  271. end
  272. 1815 @default_options = opts.options_class.new(opts)
  273. end
  274. 4815 @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
  275. 4815 @default_options = @default_options.merge(options) if options
  276. 4815 pl.configure(self, &block) if pl.respond_to?(:configure)
  277. 4815 @default_options.freeze
  278. 159 elsif options
  279. # this can happen when two plugins are loaded, an one of them calls the other under the hood,
  280. # albeit changing some default.
  281. 12 @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
  282. 12 @default_options = @default_options.merge(options) if options
  283. 12 @default_options.freeze
  284. end
  285. 4974 self
  286. end
  287. end
  288. end
  289. # session may be overridden by certain adapters.
  290. 23 S = Session
  291. end

lib/httpx/session_extensions.rb

100.0% lines covered

14 relevant lines. 14 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 unless ENV.keys.grep(/\Ahttps?_proxy\z/i).empty?
  4. 1 proxy_session = plugin(:proxy)
  5. 1 remove_const(:Session)
  6. 1 const_set(:Session, proxy_session.class)
  7. # redefine the default options static var, which needs to
  8. # refresh options_class
  9. 1 options = proxy_session.class.default_options.to_hash
  10. 1 original_verbosity = $VERBOSE
  11. 1 $VERBOSE = nil
  12. 1 const_set(:Options, proxy_session.class.default_options.options_class)
  13. 1 options[:options_class] = Class.new(options[:options_class])
  14. 1 options.freeze
  15. 1 Options.send(:const_set, :DEFAULT_OPTIONS, options)
  16. 1 Session.instance_variable_set(:@default_options, Options.new(options))
  17. 1 $VERBOSE = original_verbosity
  18. end
  19. skipped # :nocov:
  20. skipped if Session.default_options.debug_level > 2
  21. skipped proxy_session = plugin(:internal_telemetry)
  22. skipped remove_const(:Session)
  23. skipped const_set(:Session, proxy_session.class)
  24. skipped end
  25. skipped # :nocov:
  26. end

lib/httpx/timers.rb

100.0% lines covered

57 relevant lines. 57 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 class Timers
  4. 23 def initialize
  5. 500 @intervals = []
  6. end
  7. 23 def after(interval_in_secs, cb = nil, &blk)
  8. 32769 return unless interval_in_secs
  9. 32769 callback = cb || blk
  10. # I'm assuming here that most requests will have the same
  11. # request timeout, as in most cases they share common set of
  12. # options. A user setting different request timeouts for 100s of
  13. # requests will already have a hard time dealing with that.
  14. 60167 unless (interval = @intervals.find { |t| t.interval == interval_in_secs })
  15. 6167 interval = Interval.new(interval_in_secs)
  16. 12202 interval.on_empty { @intervals.delete(interval) }
  17. 6167 @intervals << interval
  18. 6167 @intervals.sort!
  19. end
  20. 32769 interval << callback
  21. 32769 @next_interval_at = nil
  22. 32769 interval
  23. end
  24. 23 def wait_interval
  25. 3691913 return if @intervals.empty?
  26. 3679068 @next_interval_at = Utils.now
  27. 3679068 @intervals.first.interval
  28. end
  29. 23 def fire(error = nil)
  30. 3691814 raise error if error && error.timeout != @intervals.first
  31. 3691809 return if @intervals.empty? || !@next_interval_at
  32. 3674597 elapsed_time = Utils.elapsed_time(@next_interval_at)
  33. 7349266 @intervals.delete_if { |interval| interval.elapse(elapsed_time) <= 0 }
  34. 3674597 @next_interval_at = nil if @intervals.empty?
  35. end
  36. 23 class Interval
  37. 23 include Comparable
  38. 23 attr_reader :interval
  39. 23 def initialize(interval)
  40. 6167 @interval = interval
  41. 6167 @callbacks = []
  42. 6167 @on_empty = nil
  43. end
  44. 23 def on_empty(&blk)
  45. 6167 @on_empty = blk
  46. end
  47. 23 def <=>(other)
  48. 450 @interval <=> other.interval
  49. end
  50. 23 def ==(other)
  51. 1846 return @interval == other if other.is_a?(Numeric)
  52. 1846 @interval == other.to_f # rubocop:disable Lint/FloatComparison
  53. end
  54. 23 def to_f
  55. 1846 Float(@interval)
  56. end
  57. 23 def <<(callback)
  58. 32769 @callbacks << callback
  59. end
  60. 23 def delete(callback)
  61. 48589 @callbacks.delete(callback)
  62. 48589 @on_empty.call if @callbacks.empty?
  63. end
  64. 23 def no_callbacks?
  65. 48589 @callbacks.empty?
  66. end
  67. 23 def elapsed?
  68. 832 @interval <= 0
  69. end
  70. 23 def elapse(elapsed)
  71. 3225584 @interval -= elapsed
  72. 3674669 if @interval <= 0
  73. 400 cb = @callbacks.dup
  74. 400 cb.each(&:call)
  75. end
  76. 3674669 @interval
  77. end
  78. end
  79. 23 private_constant :Interval
  80. end
  81. end

lib/httpx/transcoder.rb

100.0% lines covered

53 relevant lines. 53 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 module Transcoder
  4. 23 module_function
  5. 23 def normalize_keys(key, value, cond = nil, &block)
  6. 2482 if cond && cond.call(value)
  7. 749 block.call(key.to_s, value)
  8. 1733 elsif value.respond_to?(:to_ary)
  9. 341 if value.empty?
  10. 96 block.call("#{key}[]")
  11. else
  12. 245 value.to_ary.each do |element|
  13. 394 normalize_keys("#{key}[]", element, cond, &block)
  14. end
  15. end
  16. 1392 elsif value.respond_to?(:to_hash)
  17. 384 value.to_hash.each do |child_key, child_value|
  18. 384 normalize_keys("#{key}[#{child_key}]", child_value, cond, &block)
  19. end
  20. else
  21. 1008 block.call(key.to_s, value)
  22. end
  23. end
  24. # based on https://github.com/rack/rack/blob/d15dd728440710cfc35ed155d66a98dc2c07ae42/lib/rack/query_parser.rb#L82
  25. 23 def normalize_query(params, name, v, depth)
  26. 138 raise Error, "params depth surpasses what's supported" if depth <= 0
  27. 138 name =~ /\A[\[\]]*([^\[\]]+)\]*/
  28. 138 k = Regexp.last_match(1) || ""
  29. 138 after = Regexp.last_match ? Regexp.last_match.post_match : ""
  30. 138 if k.empty?
  31. 12 return Array(v) if !v.empty? && name == "[]"
  32. 5 return
  33. end
  34. 105 case after
  35. when ""
  36. 35 params[k] = v
  37. when "["
  38. 5 params[name] = v
  39. when "[]"
  40. 12 params[k] ||= []
  41. 12 raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
  42. 12 params[k] << v
  43. when /^\[\]\[([^\[\]]+)\]$/, /^\[\](.+)$/
  44. 24 child_key = Regexp.last_match(1)
  45. 24 params[k] ||= []
  46. 24 raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
  47. 24 if params[k].last.is_a?(Hash) && !params_hash_has_key?(params[k].last, child_key)
  48. 6 normalize_query(params[k].last, child_key, v, depth - 1)
  49. else
  50. 18 params[k] << normalize_query({}, child_key, v, depth - 1)
  51. end
  52. else
  53. 42 params[k] ||= {}
  54. 42 raise Error, "expected Hash (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Hash)
  55. 35 params[k] = normalize_query(params[k], after, v, depth - 1)
  56. end
  57. 126 params
  58. end
  59. 23 def params_hash_has_key?(hash, key)
  60. 12 return false if key.include?("[]")
  61. 12 key.split(/[\[\]]+/).inject(hash) do |h, part|
  62. 12 next h if part == ""
  63. 12 return false unless h.is_a?(Hash) && h.key?(part)
  64. 6 h[part]
  65. end
  66. 6 true
  67. end
  68. end
  69. end
  70. 23 require "httpx/transcoder/body"
  71. 23 require "httpx/transcoder/form"
  72. 23 require "httpx/transcoder/json"
  73. 23 require "httpx/transcoder/xml"
  74. 23 require "httpx/transcoder/chunker"
  75. 23 require "httpx/transcoder/deflate"
  76. 23 require "httpx/transcoder/gzip"

lib/httpx/transcoder/body.rb

100.0% lines covered

32 relevant lines. 32 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "forwardable"
  3. 23 module HTTPX::Transcoder
  4. 23 module Body
  5. 23 class Error < HTTPX::Error; end
  6. 23 module_function
  7. 23 class Encoder
  8. 23 extend Forwardable
  9. 23 def_delegator :@raw, :to_s
  10. 23 def initialize(body)
  11. 902 @raw = body
  12. end
  13. 23 def bytesize
  14. 3746 if @raw.respond_to?(:bytesize)
  15. 1775 @raw.bytesize
  16. 1971 elsif @raw.respond_to?(:to_ary)
  17. 699 @raw.sum(&:bytesize)
  18. 1272 elsif @raw.respond_to?(:size)
  19. 858 @raw.size || Float::INFINITY
  20. 414 elsif @raw.respond_to?(:length)
  21. 192 @raw.length || Float::INFINITY
  22. 222 elsif @raw.respond_to?(:each)
  23. 216 Float::INFINITY
  24. else
  25. 6 raise Error, "cannot determine size of body: #{@raw.inspect}"
  26. end
  27. end
  28. 23 def content_type
  29. 866 "application/octet-stream"
  30. end
  31. 23 private
  32. 23 def respond_to_missing?(meth, *args)
  33. 3678 @raw.respond_to?(meth, *args) || super
  34. end
  35. 23 def method_missing(meth, *args, &block)
  36. 1017 return super unless @raw.respond_to?(meth)
  37. 1017 @raw.__send__(meth, *args, &block)
  38. end
  39. end
  40. 23 def encode(body)
  41. 902 Encoder.new(body)
  42. end
  43. end
  44. end

lib/httpx/transcoder/chunker.rb

100.0% lines covered

66 relevant lines. 66 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "forwardable"
  3. 23 module HTTPX::Transcoder
  4. 23 module Chunker
  5. 23 class Error < HTTPX::Error; end
  6. 23 CRLF = "\r\n".b
  7. 23 class Encoder
  8. 23 extend Forwardable
  9. 23 def initialize(body)
  10. 72 @raw = body
  11. end
  12. 23 def each
  13. 72 return enum_for(__method__) unless block_given?
  14. 72 @raw.each do |chunk|
  15. 336 yield "#{chunk.bytesize.to_s(16)}#{CRLF}#{chunk}#{CRLF}"
  16. end
  17. 72 yield "0#{CRLF}"
  18. end
  19. 23 def respond_to_missing?(meth, *args)
  20. 120 @raw.respond_to?(meth, *args) || super
  21. end
  22. end
  23. 23 class Decoder
  24. 23 extend Forwardable
  25. 23 def_delegator :@buffer, :empty?
  26. 23 def_delegator :@buffer, :<<
  27. 23 def_delegator :@buffer, :clear
  28. 23 def initialize(buffer, trailers = false)
  29. 80 @buffer = buffer
  30. 80 @chunk_buffer = "".b
  31. 80 @finished = false
  32. 80 @state = :length
  33. 80 @trailers = trailers
  34. end
  35. 23 def to_s
  36. 74 @buffer
  37. end
  38. 23 def each
  39. 144 loop do
  40. 679 case @state
  41. when :length
  42. 232 index = @buffer.index(CRLF)
  43. 232 return unless index && index.positive?
  44. # Read hex-length
  45. 232 hexlen = @buffer.byteslice(0, index)
  46. 232 @buffer = @buffer.byteslice(index..-1) || "".b
  47. 232 hexlen[/\h/] || raise(Error, "wrong chunk size line: #{hexlen}")
  48. 232 @chunk_length = hexlen.hex
  49. # check if is last chunk
  50. 232 @finished = @chunk_length.zero?
  51. 232 nextstate(:crlf)
  52. when :crlf
  53. 384 crlf_size = @finished && !@trailers ? 4 : 2
  54. # consume CRLF
  55. 384 return if @buffer.bytesize < crlf_size
  56. 384 raise Error, "wrong chunked encoding format" unless @buffer.start_with?(CRLF * (crlf_size / 2))
  57. 384 @buffer = @buffer.byteslice(crlf_size..-1)
  58. 384 if @chunk_length.nil?
  59. 152 nextstate(:length)
  60. else
  61. 232 return if @finished
  62. 158 nextstate(:data)
  63. end
  64. when :data
  65. 196 chunk = @buffer.byteslice(0, @chunk_length)
  66. 196 @buffer = @buffer.byteslice(@chunk_length..-1) || "".b
  67. 196 @chunk_buffer << chunk
  68. 164 @chunk_length -= chunk.bytesize
  69. 196 if @chunk_length.zero?
  70. 158 yield @chunk_buffer unless @chunk_buffer.empty?
  71. 152 @chunk_buffer.clear
  72. 152 @chunk_length = nil
  73. 152 nextstate(:crlf)
  74. end
  75. end
  76. 732 break if @buffer.empty?
  77. end
  78. end
  79. 23 def finished?
  80. 138 @finished
  81. end
  82. 23 private
  83. 23 def nextstate(state)
  84. 694 @state = state
  85. end
  86. end
  87. 23 module_function
  88. 23 def encode(chunks)
  89. 72 Encoder.new(chunks)
  90. end
  91. end
  92. end

lib/httpx/transcoder/deflate.rb

100.0% lines covered

20 relevant lines. 20 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "zlib"
  3. 23 require_relative "utils/deflater"
  4. 23 module HTTPX
  5. 23 module Transcoder
  6. 23 module Deflate
  7. 23 class Deflater < Transcoder::Deflater
  8. 23 def deflate(chunk)
  9. 36 @deflater ||= Zlib::Deflate.new
  10. 36 if chunk.nil?
  11. 24 unless @deflater.closed?
  12. 12 last = @deflater.finish
  13. 12 @deflater.close
  14. 12 last.empty? ? nil : last
  15. end
  16. else
  17. 12 @deflater.deflate(chunk)
  18. end
  19. end
  20. end
  21. 23 module_function
  22. 23 def encode(body)
  23. 12 Deflater.new(body)
  24. end
  25. 23 def decode(response, bytesize: nil)
  26. 12 bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
  27. 12 GZIP::Inflater.new(bytesize)
  28. end
  29. end
  30. end
  31. end

lib/httpx/transcoder/form.rb

100.0% lines covered

41 relevant lines. 41 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "forwardable"
  3. 23 require "uri"
  4. 23 require_relative "multipart"
  5. 23 module HTTPX
  6. 23 module Transcoder
  7. 23 module Form
  8. 23 module_function
  9. 23 PARAM_DEPTH_LIMIT = 32
  10. 23 class Encoder
  11. 23 extend Forwardable
  12. 23 def_delegator :@raw, :to_s
  13. 23 def_delegator :@raw, :to_str
  14. 23 def_delegator :@raw, :bytesize
  15. 23 def initialize(form)
  16. 511 @raw = form.each_with_object("".b) do |(key, val), buf|
  17. 859 HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
  18. 1008 buf << "&" unless buf.empty?
  19. 1008 buf << URI.encode_www_form_component(k)
  20. 1008 buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
  21. end
  22. end
  23. end
  24. 23 def content_type
  25. 395 "application/x-www-form-urlencoded"
  26. end
  27. end
  28. 23 module Decoder
  29. 23 module_function
  30. 23 def call(response, *)
  31. 30 URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
  32. 72 HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
  33. end
  34. end
  35. end
  36. 23 def encode(form)
  37. 1200 if multipart?(form)
  38. 689 Multipart::Encoder.new(form)
  39. else
  40. 511 Encoder.new(form)
  41. end
  42. end
  43. 23 def decode(response)
  44. 48 content_type = response.content_type.mime_type
  45. 40 case content_type
  46. when "application/x-www-form-urlencoded"
  47. 30 Decoder
  48. when "multipart/form-data"
  49. 12 Multipart::Decoder.new(response)
  50. else
  51. 6 raise Error, "invalid form mime type (#{content_type})"
  52. end
  53. end
  54. 23 def multipart?(data)
  55. 1200 data.any? do |_, v|
  56. 1596 Multipart::MULTIPART_VALUE_COND.call(v) ||
  57. 1243 (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
  58. 1531 (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
  59. end
  60. end
  61. end
  62. end
  63. end

lib/httpx/transcoder/gzip.rb

100.0% lines covered

43 relevant lines. 43 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "forwardable"
  3. 23 require "uri"
  4. 23 require "stringio"
  5. 23 require "zlib"
  6. 23 module HTTPX
  7. 23 module Transcoder
  8. 23 module GZIP
  9. 23 class Deflater < Transcoder::Deflater
  10. 23 def initialize(body)
  11. 32 @compressed_chunk = "".b
  12. 32 super
  13. end
  14. 23 def deflate(chunk)
  15. 64 @deflater ||= Zlib::GzipWriter.new(self)
  16. 64 if chunk.nil?
  17. 32 unless @deflater.closed?
  18. 32 @deflater.flush
  19. 32 @deflater.close
  20. 32 compressed_chunk
  21. end
  22. else
  23. 32 @deflater.write(chunk)
  24. 32 compressed_chunk
  25. end
  26. end
  27. 23 private
  28. 23 def write(chunk)
  29. 96 @compressed_chunk << chunk
  30. end
  31. 23 def compressed_chunk
  32. 64 @compressed_chunk.dup
  33. ensure
  34. 64 @compressed_chunk.clear
  35. end
  36. end
  37. 23 class Inflater
  38. 23 def initialize(bytesize)
  39. 120 @inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
  40. 120 @bytesize = bytesize
  41. end
  42. 23 def call(chunk)
  43. 339 buffer = @inflater.inflate(chunk)
  44. 299 @bytesize -= chunk.bytesize
  45. 339 if @bytesize <= 0
  46. 72 buffer << @inflater.finish
  47. 72 @inflater.close
  48. end
  49. 339 buffer
  50. end
  51. end
  52. 23 module_function
  53. 23 def encode(body)
  54. 32 Deflater.new(body)
  55. end
  56. 23 def decode(response, bytesize: nil)
  57. 108 bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
  58. 108 Inflater.new(bytesize)
  59. end
  60. end
  61. end
  62. end

lib/httpx/transcoder/json.rb

100.0% lines covered

32 relevant lines. 32 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "forwardable"
  3. 23 module HTTPX::Transcoder
  4. 23 module JSON
  5. 23 module_function
  6. 23 JSON_REGEX = %r{\bapplication/(?:vnd\.api\+|hal\+)?json\b}i.freeze
  7. 23 class Encoder
  8. 23 extend Forwardable
  9. 23 def_delegator :@raw, :to_s
  10. 23 def_delegator :@raw, :bytesize
  11. 23 def initialize(json)
  12. 54 @raw = JSON.json_dump(json)
  13. 54 @charset = @raw.encoding.name.downcase
  14. end
  15. 23 def content_type
  16. 54 "application/json; charset=#{@charset}"
  17. end
  18. end
  19. 23 def encode(json)
  20. 54 Encoder.new(json)
  21. end
  22. 23 def decode(response)
  23. 57 content_type = response.content_type.mime_type
  24. 57 raise HTTPX::Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type)
  25. 51 method(:json_load)
  26. end
  27. # rubocop:disable Style/SingleLineMethods
  28. 23 if defined?(MultiJson)
  29. 4 def json_load(*args); MultiJson.load(*args); end
  30. 1 def json_dump(*args); MultiJson.dump(*args); end
  31. 22 elsif defined?(Oj)
  32. 4 def json_load(response, *args); Oj.load(response.to_s, *args); end
  33. 1 def json_dump(*args); Oj.dump(*args); end
  34. 21 elsif defined?(Yajl)
  35. 4 def json_load(response, *args); Yajl::Parser.new(*args).parse(response.to_s); end
  36. 1 def json_dump(*args); Yajl::Encoder.encode(*args); end
  37. else
  38. 20 require "json"
  39. 55 def json_load(*args); ::JSON.parse(*args); end
  40. 65 def json_dump(*args); ::JSON.dump(*args); end
  41. end
  42. # rubocop:enable Style/SingleLineMethods
  43. end
  44. end

lib/httpx/transcoder/multipart.rb

100.0% lines covered

10 relevant lines. 10 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require_relative "multipart/encoder"
  3. 23 require_relative "multipart/decoder"
  4. 23 require_relative "multipart/part"
  5. 23 require_relative "multipart/mime_type_detector"
  6. 23 module HTTPX::Transcoder
  7. 23 module Multipart
  8. 23 MULTIPART_VALUE_COND = lambda do |value|
  9. 3555 value.respond_to?(:read) ||
  10. 2601 (value.respond_to?(:to_hash) &&
  11. value.key?(:body) &&
  12. 484 (value.key?(:filename) || value.key?(:content_type)))
  13. end
  14. end
  15. end

lib/httpx/transcoder/multipart/decoder.rb

93.98% lines covered

83 relevant lines. 78 lines covered and 5 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "tempfile"
  3. 23 require "delegate"
  4. 23 module HTTPX
  5. 23 module Transcoder
  6. 23 module Multipart
  7. 23 class FilePart < SimpleDelegator
  8. 23 attr_reader :original_filename, :content_type
  9. 23 def initialize(filename, content_type)
  10. 24 @original_filename = filename
  11. 24 @content_type = content_type
  12. 24 @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  13. 24 super(@file)
  14. end
  15. end
  16. 23 class Decoder
  17. 23 include HTTPX::Utils
  18. 23 CRLF = "\r\n"
  19. 23 BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
  20. 23 MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
  21. 23 MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
  22. 23 MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
  23. 23 WINDOW_SIZE = 2 << 14
  24. 23 def initialize(response)
  25. 2 @boundary = begin
  26. 12 m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
  27. 12 raise Error, "no boundary declared in content-type header" unless m
  28. 12 m.strip
  29. end
  30. 12 @buffer = "".b
  31. 12 @parts = {}
  32. 12 @intermediate_boundary = "--#{@boundary}"
  33. 12 @state = :idle
  34. end
  35. 23 def call(response, *)
  36. 12 response.body.each do |chunk|
  37. 12 @buffer << chunk
  38. 12 parse
  39. end
  40. 12 raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
  41. 12 @parts
  42. end
  43. 23 private
  44. 23 def parse
  45. 10 case @state
  46. when :idle
  47. 12 raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
  48. 12 @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
  49. 12 @state = :part_header
  50. when :part_header
  51. 36 idx = @buffer.index("#{CRLF}#{CRLF}")
  52. # raise Error, "couldn't parse part headers" unless idx
  53. 36 return unless idx
  54. 36 head = @buffer.byteslice(0..idx + 4 - 1)
  55. 36 @buffer = @buffer.byteslice(head.bytesize..-1)
  56. 36 content_type = head[MULTIPART_CONTENT_TYPE, 1]
  57. 66 if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
  58. 36 name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
  59. 36 name.gsub!(/\\(.)/, "\\1")
  60. 6 name
  61. else
  62. name = head[MULTIPART_CONTENT_ID, 1]
  63. end
  64. 36 filename = HTTPX::Utils.get_filename(head)
  65. 36 name = filename || +"#{content_type || "text/plain"}[]" if name.nil? || name.empty?
  66. 36 @current = name
  67. 30 @parts[name] = if filename
  68. 24 FilePart.new(filename, content_type)
  69. else
  70. 12 "".b
  71. end
  72. 36 @state = :part_body
  73. when :part_body
  74. 36 part = @parts[@current]
  75. 36 body_separator = if part.is_a?(FilePart)
  76. 20 "#{CRLF}#{CRLF}"
  77. else
  78. 12 CRLF
  79. end
  80. 36 idx = @buffer.index(body_separator)
  81. 36 if idx
  82. 36 payload = @buffer.byteslice(0..idx - 1)
  83. 36 @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
  84. 36 part << payload
  85. 36 part.rewind if part.respond_to?(:rewind)
  86. 36 @state = :parse_boundary
  87. else
  88. part << @buffer
  89. @buffer.clear
  90. end
  91. when :parse_boundary
  92. 36 raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
  93. 36 @buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
  94. 36 if @buffer == "--"
  95. 12 @buffer.clear
  96. 12 @state = :done
  97. 12 return
  98. 24 elsif @buffer.start_with?(CRLF)
  99. 24 @buffer = @buffer.byteslice(2..-1)
  100. 24 @state = :part_header
  101. else
  102. return
  103. end
  104. when :done
  105. raise Error, "parsing should have been over by now"
  106. 20 end until @buffer.empty?
  107. end
  108. end
  109. end
  110. end
  111. end

lib/httpx/transcoder/multipart/encoder.rb

100.0% lines covered

65 relevant lines. 65 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 module Transcoder::Multipart
  4. 23 class Encoder
  5. 23 attr_reader :bytesize
  6. 23 def initialize(form)
  7. 689 @boundary = ("-" * 21) << SecureRandom.hex(21)
  8. 689 @part_index = 0
  9. 689 @buffer = "".b
  10. 689 @form = form
  11. 689 @parts = to_parts(form)
  12. end
  13. 23 def content_type
  14. 689 "multipart/form-data; boundary=#{@boundary}"
  15. end
  16. 23 def read(length = nil, outbuf = nil)
  17. 2564 data = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
  18. 2564 data ||= "".b
  19. 2564 read_chunks(data, length)
  20. 2564 data unless length && data.empty?
  21. end
  22. 23 def rewind
  23. 12 form = @form.each_with_object([]) do |(key, val), aux|
  24. 12 val = val.reopen(val.path, File::RDONLY) if val.is_a?(File) && val.closed?
  25. 12 val.rewind if val.respond_to?(:rewind)
  26. 12 aux << [key, val]
  27. end
  28. 12 @form = form
  29. 12 @parts = to_parts(form)
  30. 12 @part_index = 0
  31. end
  32. 23 private
  33. 23 def to_parts(form)
  34. 701 @bytesize = 0
  35. 701 params = form.each_with_object([]) do |(key, val), aux|
  36. 845 Transcoder.normalize_keys(key, val, MULTIPART_VALUE_COND) do |k, v|
  37. 845 next if v.nil?
  38. 845 value, content_type, filename = Part.call(v)
  39. 845 header = header_part(k, content_type, filename)
  40. 705 @bytesize += header.size
  41. 845 aux << header
  42. 705 @bytesize += value.size
  43. 845 aux << value
  44. 845 delimiter = StringIO.new("\r\n")
  45. 705 @bytesize += delimiter.size
  46. 845 aux << delimiter
  47. end
  48. end
  49. 701 final_delimiter = StringIO.new("--#{@boundary}--\r\n")
  50. 585 @bytesize += final_delimiter.size
  51. 701 params << final_delimiter
  52. 701 params
  53. end
  54. 23 def header_part(key, content_type, filename)
  55. 845 header = "--#{@boundary}\r\n".b
  56. 845 header << "Content-Disposition: form-data; name=#{key.inspect}".b
  57. 845 header << "; filename=#{filename.inspect}" if filename
  58. 845 header << "\r\nContent-Type: #{content_type}\r\n\r\n"
  59. 845 StringIO.new(header)
  60. end
  61. 23 def read_chunks(buffer, length = nil)
  62. 3636 while @part_index < @parts.size
  63. 7634 chunk = read_from_part(length)
  64. 7634 next unless chunk
  65. 4398 buffer << chunk.force_encoding(Encoding::BINARY)
  66. 4398 next unless length
  67. 3670 length -= chunk.bytesize
  68. 4398 break if length.zero?
  69. end
  70. end
  71. # if there's a current part to read from, tries to read a chunk.
  72. 23 def read_from_part(max_length = nil)
  73. 7634 part = @parts[@part_index]
  74. 7634 chunk = part.read(max_length, @buffer)
  75. 7634 return chunk if chunk && !chunk.empty?
  76. 3236 part.close if part.respond_to?(:close)
  77. 2700 @part_index += 1
  78. 536 nil
  79. end
  80. end
  81. end
  82. end

lib/httpx/transcoder/multipart/mime_type_detector.rb

91.89% lines covered

37 relevant lines. 34 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 module Transcoder::Multipart
  4. 23 module MimeTypeDetector
  5. 23 module_function
  6. 23 DEFAULT_MIMETYPE = "application/octet-stream"
  7. # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
  8. 23 if defined?(FileMagic)
  9. 1 MAGIC_NUMBER = 256 * 1024
  10. 1 def call(file, _)
  11. 1 return nil if file.eof? # FileMagic returns "application/x-empty" for empty files
  12. 1 mime = FileMagic.open(FileMagic::MAGIC_MIME_TYPE) do |filemagic|
  13. 1 filemagic.buffer(file.read(MAGIC_NUMBER))
  14. end
  15. 1 file.rewind
  16. 1 mime
  17. end
  18. 22 elsif defined?(Marcel)
  19. 1 def call(file, filename)
  20. 1 return nil if file.eof? # marcel returns "application/octet-stream" for empty files
  21. 1 Marcel::MimeType.for(file, name: filename)
  22. end
  23. 21 elsif defined?(MimeMagic)
  24. 1 def call(file, _)
  25. 1 mime = MimeMagic.by_magic(file)
  26. 1 mime.type if mime
  27. end
  28. 20 elsif system("which file", out: File::NULL)
  29. 20 require "open3"
  30. 20 def call(file, _)
  31. 457 return if file.eof? # file command returns "application/x-empty" for empty files
  32. 457 Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
  33. begin
  34. 457 ::IO.copy_stream(file, stdin.binmode)
  35. rescue Errno::EPIPE
  36. end
  37. 457 file.rewind
  38. 457 stdin.close
  39. 457 status = thread.value
  40. # call to file command failed
  41. 457 if status.nil? || !status.success?
  42. $stderr.print(stderr.read)
  43. else
  44. 457 output = stdout.read.strip
  45. 457 if output.include?("cannot open")
  46. $stderr.print(output)
  47. else
  48. 457 output
  49. end
  50. end
  51. end
  52. end
  53. else
  54. def call(_, _); end
  55. end
  56. end
  57. end
  58. end

lib/httpx/transcoder/multipart/part.rb

100.0% lines covered

18 relevant lines. 18 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 module Transcoder::Multipart
  4. 23 module Part
  5. 23 module_function
  6. 23 def call(value)
  7. # take out specialized objects of the way
  8. 845 if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
  9. 80 return value, value.content_type, value.filename
  10. end
  11. 749 content_type = filename = nil
  12. 749 if value.is_a?(Hash)
  13. 242 content_type = value[:content_type]
  14. 242 filename = value[:filename]
  15. 242 value = value[:body]
  16. end
  17. 749 value = value.open(File::RDONLY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
  18. 749 if value.respond_to?(:path) && value.respond_to?(:read)
  19. # either a File, a Tempfile, or something else which has to quack like a file
  20. 461 filename ||= File.basename(value.path)
  21. 461 content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
  22. 461 [value, content_type, filename]
  23. else
  24. 288 [StringIO.new(value.to_s), content_type || "text/plain", filename]
  25. end
  26. end
  27. end
  28. end
  29. end

lib/httpx/transcoder/utils/body_reader.rb

96.0% lines covered

25 relevant lines. 24 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "stringio"
  3. 23 module HTTPX
  4. 23 module Transcoder
  5. 23 class BodyReader
  6. 23 def initialize(body)
  7. 159 @body = if body.respond_to?(:read)
  8. 10 body.rewind if body.respond_to?(:rewind)
  9. 10 body
  10. 149 elsif body.respond_to?(:each)
  11. 30 body.enum_for(:each)
  12. else
  13. 119 StringIO.new(body.to_s)
  14. end
  15. end
  16. 23 def bytesize
  17. 335 return @body.bytesize if @body.respond_to?(:bytesize)
  18. 305 Float::INFINITY
  19. end
  20. 23 def read(length = nil, outbuf = nil)
  21. 340 return @body.read(length, outbuf) if @body.respond_to?(:read)
  22. begin
  23. 70 chunk = @body.next
  24. 40 if outbuf
  25. 40 outbuf.clear.force_encoding(Encoding::BINARY)
  26. 40 outbuf << chunk
  27. else
  28. outbuf = chunk
  29. end
  30. 40 outbuf unless length && outbuf.empty?
  31. 24 rescue StopIteration
  32. end
  33. end
  34. 23 def close
  35. 32 @body.close if @body.respond_to?(:close)
  36. end
  37. end
  38. end
  39. end

lib/httpx/transcoder/utils/deflater.rb

97.22% lines covered

36 relevant lines. 35 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "forwardable"
  3. 23 require_relative "body_reader"
  4. 23 module HTTPX
  5. 23 module Transcoder
  6. 23 class Deflater
  7. 23 extend Forwardable
  8. 23 attr_reader :content_type
  9. 23 def initialize(body)
  10. 54 @content_type = body.content_type
  11. 54 @body = BodyReader.new(body)
  12. 54 @closed = false
  13. end
  14. 23 def bytesize
  15. 206 buffer_deflate!
  16. 206 @buffer.size
  17. end
  18. 23 def read(length = nil, outbuf = nil)
  19. 260 return @buffer.read(length, outbuf) if @buffer
  20. 152 return if @closed
  21. 120 chunk = @body.read(length)
  22. 120 compressed_chunk = deflate(chunk)
  23. 120 return unless compressed_chunk
  24. 98 if outbuf
  25. 98 outbuf.clear.force_encoding(Encoding::BINARY)
  26. 98 outbuf << compressed_chunk
  27. else
  28. compressed_chunk
  29. end
  30. end
  31. 23 def close
  32. 32 return if @closed
  33. 32 @buffer.close if @buffer
  34. 32 @body.close
  35. 32 @closed = true
  36. end
  37. 23 private
  38. # rubocop:disable Naming/MemoizedInstanceVariableName
  39. 23 def buffer_deflate!
  40. 206 return @buffer if defined?(@buffer)
  41. 54 buffer = Response::Buffer.new(
  42. threshold_size: Options::MAX_BODY_THRESHOLD_SIZE
  43. )
  44. 54 ::IO.copy_stream(self, buffer)
  45. 54 buffer.rewind
  46. 54 @buffer = buffer
  47. end
  48. # rubocop:enable Naming/MemoizedInstanceVariableName
  49. end
  50. end
  51. end

lib/httpx/transcoder/xml.rb

92.31% lines covered

26 relevant lines. 24 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 require "delegate"
  3. 23 require "forwardable"
  4. 23 require "uri"
  5. 23 module HTTPX::Transcoder
  6. 23 module Xml
  7. 23 module_function
  8. 23 MIME_TYPES = %r{\b(application|text)/(.+\+)?xml\b}.freeze
  9. 23 class Encoder
  10. 23 def initialize(xml)
  11. 102 @raw = xml
  12. end
  13. 23 def content_type
  14. 102 charset = @raw.respond_to?(:encoding) ? @raw.encoding.to_s.downcase : "utf-8"
  15. 102 "application/xml; charset=#{charset}"
  16. end
  17. 23 def bytesize
  18. 306 @raw.to_s.bytesize
  19. end
  20. 23 def to_s
  21. 96 @raw.to_s
  22. end
  23. end
  24. 23 def encode(xml)
  25. 102 Encoder.new(xml)
  26. end
  27. begin
  28. 23 require "nokogiri"
  29. 23 def decode(response)
  30. 6 content_type = response.content_type.mime_type
  31. 6 raise HTTPX::Error, "invalid form mime type (#{content_type})" unless MIME_TYPES.match?(content_type)
  32. 6 Nokogiri::XML.method(:parse)
  33. end
  34. rescue LoadError
  35. def decode(_response)
  36. raise HTTPX::Error, "\"nokogiri\" is required in order to decode XML"
  37. end
  38. end
  39. end
  40. end

lib/httpx/utils.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 23 module HTTPX
  3. 23 module Utils
  4. 23 using URIExtensions
  5. 23 TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
  6. 23 VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
  7. 23 FILENAME_REGEX = /\s*filename=(#{VALUE})/.freeze
  8. 23 FILENAME_EXTENSION_REGEX = /\s*filename\*=(#{VALUE})/.freeze
  9. 23 module_function
  10. 23 def now
  11. 3698401 Process.clock_gettime(Process::CLOCK_MONOTONIC)
  12. end
  13. 23 def elapsed_time(monotonic_timestamp)
  14. 3675468 Process.clock_gettime(Process::CLOCK_MONOTONIC) - monotonic_timestamp
  15. end
  16. # The value of this field can be either an HTTP-date or a number of
  17. # seconds to delay after the response is received.
  18. 23 def parse_retry_after(retry_after)
  19. # first: bet on it being an integer
  20. 36 Integer(retry_after)
  21. rescue ArgumentError
  22. # Then it's a datetime
  23. 12 time = Time.httpdate(retry_after)
  24. 12 time - Time.now
  25. end
  26. 23 def get_filename(header, _prefix_regex = nil)
  27. 66 filename = nil
  28. 55 case header
  29. when FILENAME_REGEX
  30. 42 filename = Regexp.last_match(1)
  31. 42 filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
  32. when FILENAME_EXTENSION_REGEX
  33. 12 filename = Regexp.last_match(1)
  34. 12 encoding, _, filename = filename.split("'", 3)
  35. end
  36. 66 return unless filename
  37. 102 filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
  38. 54 filename.scrub!
  39. 54 filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
  40. 54 filename.force_encoding ::Encoding.find(encoding) if encoding
  41. 54 filename
  42. end
  43. 23 URIParser = URI::RFC2396_Parser.new
  44. 23 def to_uri(uri)
  45. 12246 return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
  46. 25 uri = URI(URIParser.escape(uri))
  47. 25 non_ascii_hostname = URIParser.unescape(uri.host)
  48. 25 non_ascii_hostname.force_encoding(Encoding::UTF_8)
  49. 25 idna_hostname = Punycode.encode_hostname(non_ascii_hostname)
  50. 25 uri.host = idna_hostname
  51. 24 uri.non_ascii_hostname = non_ascii_hostname
  52. 24 uri
  53. end
  54. end
  55. end