loading
Generated 2025-04-26T10:06:14+00:00

All Files ( 96.12% covered at 100964.31 hits/line )

105 files in total.
7583 relevant lines, 7289 lines covered and 294 lines missed. ( 96.12% )
File % covered Lines Relevant Lines Lines covered Lines missed Avg. Hits / Line
lib/httpx.rb 100.00 % 66 39 39 0 797.95
lib/httpx/adapters/datadog.rb 87.04 % 345 162 141 21 34.36
lib/httpx/adapters/faraday.rb 98.13 % 298 160 157 3 133.94
lib/httpx/adapters/sentry.rb 100.00 % 121 62 62 0 99.03
lib/httpx/adapters/webmock.rb 100.00 % 175 93 93 0 96.04
lib/httpx/altsvc.rb 96.39 % 163 83 80 3 242.93
lib/httpx/buffer.rb 100.00 % 61 27 27 0 432309.67
lib/httpx/callbacks.rb 100.00 % 35 19 19 0 166881.00
lib/httpx/chainable.rb 95.65 % 106 46 44 2 1074.43
lib/httpx/connection.rb 92.52 % 945 481 445 36 198161.52
lib/httpx/connection/http1.rb 89.14 % 400 221 197 24 3875.30
lib/httpx/connection/http2.rb 94.98 % 438 259 246 13 313768.21
lib/httpx/domain_name.rb 95.56 % 145 45 43 2 247.22
lib/httpx/errors.rb 97.62 % 109 42 41 1 86.12
lib/httpx/extensions.rb 67.86 % 59 28 19 9 417.29
lib/httpx/headers.rb 100.00 % 176 71 71 0 18651.25
lib/httpx/io.rb 100.00 % 11 5 5 0 27.00
lib/httpx/io/ssl.rb 96.55 % 165 87 84 3 2377.62
lib/httpx/io/tcp.rb 91.15 % 211 113 103 10 9124.42
lib/httpx/io/udp.rb 100.00 % 62 35 35 0 349.60
lib/httpx/io/unix.rb 94.29 % 70 35 33 2 20.49
lib/httpx/loggable.rb 100.00 % 41 17 17 0 53439.59
lib/httpx/options.rb 98.82 % 378 169 167 2 17385.38
lib/httpx/parser/http1.rb 100.00 % 186 109 109 0 7833.29
lib/httpx/plugins/auth.rb 100.00 % 25 9 9 0 24.00
lib/httpx/plugins/auth/basic.rb 100.00 % 20 10 10 0 86.60
lib/httpx/plugins/auth/digest.rb 100.00 % 102 61 61 0 127.85
lib/httpx/plugins/auth/ntlm.rb 100.00 % 35 19 19 0 2.95
lib/httpx/plugins/auth/socks5.rb 100.00 % 22 11 11 0 23.82
lib/httpx/plugins/aws_sdk_authentication.rb 100.00 % 109 43 43 0 13.21
lib/httpx/plugins/aws_sigv4.rb 100.00 % 236 120 120 0 101.51
lib/httpx/plugins/basic_auth.rb 100.00 % 29 12 12 0 35.33
lib/httpx/plugins/brotli.rb 100.00 % 50 25 25 0 10.80
lib/httpx/plugins/callbacks.rb 100.00 % 102 48 48 0 140.23
lib/httpx/plugins/circuit_breaker.rb 100.00 % 145 66 66 0 74.82
lib/httpx/plugins/circuit_breaker/circuit.rb 100.00 % 100 47 47 0 58.21
lib/httpx/plugins/circuit_breaker/circuit_store.rb 100.00 % 53 23 23 0 98.48
lib/httpx/plugins/content_digest.rb 100.00 % 202 102 102 0 79.67
lib/httpx/plugins/cookies.rb 100.00 % 107 52 52 0 122.63
lib/httpx/plugins/cookies/cookie.rb 100.00 % 174 77 77 0 318.21
lib/httpx/plugins/cookies/jar.rb 100.00 % 97 47 47 0 265.34
lib/httpx/plugins/cookies/set_cookie_parser.rb 100.00 % 140 70 70 0 161.01
lib/httpx/plugins/digest_auth.rb 100.00 % 65 29 29 0 99.24
lib/httpx/plugins/expect.rb 100.00 % 118 56 56 0 87.68
lib/httpx/plugins/follow_redirects.rb 100.00 % 231 108 108 0 198820.85
lib/httpx/plugins/grpc.rb 100.00 % 280 133 133 0 117.79
lib/httpx/plugins/grpc/call.rb 90.91 % 63 33 30 3 40.18
lib/httpx/plugins/grpc/grpc_encoding.rb 97.87 % 90 47 46 1 75.57
lib/httpx/plugins/grpc/message.rb 95.83 % 55 24 23 1 39.50
lib/httpx/plugins/h2c.rb 94.92 % 117 59 56 3 14.34
lib/httpx/plugins/ntlm_auth.rb 100.00 % 60 30 30 0 3.97
lib/httpx/plugins/oauth.rb 100.00 % 175 87 87 0 70.23
lib/httpx/plugins/persistent.rb 100.00 % 78 28 28 0 189.46
lib/httpx/plugins/proxy.rb 98.01 % 308 151 148 3 325.37
lib/httpx/plugins/proxy/http.rb 100.00 % 184 104 104 0 186.97
lib/httpx/plugins/proxy/socks4.rb 97.47 % 135 79 77 2 178.38
lib/httpx/plugins/proxy/socks5.rb 99.12 % 194 113 112 1 280.96
lib/httpx/plugins/proxy/ssh.rb 92.31 % 92 52 48 4 8.19
lib/httpx/plugins/push_promise.rb 100.00 % 81 41 41 0 10.49
lib/httpx/plugins/query.rb 100.00 % 35 15 15 0 10.07
lib/httpx/plugins/rate_limiter.rb 100.00 % 55 18 18 0 41.72
lib/httpx/plugins/response_cache.rb 100.00 % 331 143 143 0 185.28
lib/httpx/plugins/response_cache/file_store.rb 100.00 % 139 71 71 0 157.75
lib/httpx/plugins/response_cache/store.rb 100.00 % 33 16 16 0 152.94
lib/httpx/plugins/retries.rb 96.88 % 223 96 93 3 224753.29
lib/httpx/plugins/ssrf_filter.rb 100.00 % 145 61 61 0 144.25
lib/httpx/plugins/stream.rb 97.78 % 183 90 88 2 110.58
lib/httpx/plugins/stream_bidi.rb 98.36 % 133 61 60 1 31.28
lib/httpx/plugins/upgrade.rb 100.00 % 78 34 34 0 49.35
lib/httpx/plugins/upgrade/h2.rb 91.67 % 54 24 22 2 8.29
lib/httpx/plugins/webdav.rb 100.00 % 86 39 39 0 23.10
lib/httpx/plugins/xml.rb 100.00 % 76 34 34 0 84.35
lib/httpx/pmatch_extensions.rb 100.00 % 33 17 17 0 28.71
lib/httpx/pool.rb 98.90 % 189 91 90 1 4745.37
lib/httpx/punycode.rb 100.00 % 22 9 9 0 18.78
lib/httpx/request.rb 100.00 % 317 132 132 0 6700.17
lib/httpx/request/body.rb 100.00 % 153 66 66 0 3202.15
lib/httpx/resolver.rb 100.00 % 161 82 82 0 1770.23
lib/httpx/resolver/https.rb 86.11 % 248 144 124 20 34.08
lib/httpx/resolver/multi.rb 88.24 % 93 51 45 6 3618.65
lib/httpx/resolver/native.rb 88.96 % 532 308 274 34 1915.16
lib/httpx/resolver/resolver.rb 83.13 % 165 83 69 14 1707.04
lib/httpx/resolver/system.rb 81.06 % 230 132 107 25 17.80
lib/httpx/response.rb 100.00 % 304 115 115 0 1926.72
lib/httpx/response/body.rb 100.00 % 242 109 109 0 2958.89
lib/httpx/response/buffer.rb 92.19 % 120 64 59 5 1568.75
lib/httpx/selector.rb 93.27 % 212 104 97 7 2101959.19
lib/httpx/session.rb 95.93 % 550 270 259 11 134134.27
lib/httpx/session_extensions.rb 100.00 % 29 14 14 0 6.64
lib/httpx/timers.rb 93.94 % 133 66 62 4 3654125.97
lib/httpx/transcoder.rb 100.00 % 91 52 52 0 281.75
lib/httpx/transcoder/body.rb 100.00 % 43 26 26 0 962.23
lib/httpx/transcoder/chunker.rb 100.00 % 115 66 66 0 219.77
lib/httpx/transcoder/deflate.rb 100.00 % 37 20 20 0 30.70
lib/httpx/transcoder/form.rb 100.00 % 80 42 42 0 439.62
lib/httpx/transcoder/gzip.rb 100.00 % 71 40 40 0 98.25
lib/httpx/transcoder/json.rb 100.00 % 71 33 33 0 42.61
lib/httpx/transcoder/multipart.rb 100.00 % 17 10 10 0 921.70
lib/httpx/transcoder/multipart/decoder.rb 94.05 % 141 84 79 5 29.20
lib/httpx/transcoder/multipart/encoder.rb 100.00 % 120 70 70 0 1945.89
lib/httpx/transcoder/multipart/mime_type_detector.rb 92.11 % 78 38 35 3 177.47
lib/httpx/transcoder/multipart/part.rb 100.00 % 35 18 18 0 505.17
lib/httpx/transcoder/utils/body_reader.rb 92.00 % 46 25 23 2 94.76
lib/httpx/transcoder/utils/deflater.rb 100.00 % 75 37 37 0 102.92
lib/httpx/utils.rb 100.00 % 75 39 39 0 590482.59

lib/httpx.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "httpx/version"
  3. # Top-Level Namespace
  4. #
  5. 27 module HTTPX
  6. 27 EMPTY = [].freeze
  7. 27 EMPTY_HASH = {}.freeze
  8. # All plugins should be stored under this module/namespace. Can register and load
  9. # plugins.
  10. #
  11. 27 module Plugins
  12. 27 @plugins = {}
  13. 27 @plugins_mutex = Thread::Mutex.new
  14. # Loads a plugin based on a name. If the plugin hasn't been loaded, tries to load
  15. # it from the load path under "httpx/plugins/" directory.
  16. #
  17. 27 def self.load_plugin(name)
  18. 5640 h = @plugins
  19. 5640 m = @plugins_mutex
  20. 11280 unless (plugin = m.synchronize { h[name] })
  21. 266 require "httpx/plugins/#{name}"
  22. 532 raise "Plugin #{name} hasn't been registered" unless (plugin = m.synchronize { h[name] })
  23. end
  24. 5640 plugin
  25. end
  26. # Registers a plugin (+mod+) in the central store indexed by +name+.
  27. #
  28. 27 def self.register_plugin(name, mod)
  29. 336 h = @plugins
  30. 336 m = @plugins_mutex
  31. 640 m.synchronize { h[name] = mod }
  32. end
  33. end
  34. end
  35. 27 require "httpx/extensions"
  36. 27 require "httpx/errors"
  37. 27 require "httpx/utils"
  38. 27 require "httpx/punycode"
  39. 27 require "httpx/domain_name"
  40. 27 require "httpx/altsvc"
  41. 27 require "httpx/callbacks"
  42. 27 require "httpx/loggable"
  43. 27 require "httpx/transcoder"
  44. 27 require "httpx/timers"
  45. 27 require "httpx/pool"
  46. 27 require "httpx/headers"
  47. 27 require "httpx/request"
  48. 27 require "httpx/response"
  49. 27 require "httpx/options"
  50. 27 require "httpx/chainable"
  51. 27 require "httpx/session"
  52. 27 require "httpx/session_extensions"
  53. # load integrations when possible
  54. 27 require "httpx/adapters/datadog" if defined?(DDTrace) || defined?(Datadog::Tracing)
  55. 27 require "httpx/adapters/sentry" if defined?(Sentry)
  56. 27 require "httpx/adapters/webmock" if defined?(WebMock)

lib/httpx/adapters/datadog.rb

87.04% lines covered

162 relevant lines. 141 lines covered and 21 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "datadog/tracing/contrib/integration"
  3. 6 require "datadog/tracing/contrib/configuration/settings"
  4. 6 require "datadog/tracing/contrib/patcher"
  5. 6 module Datadog::Tracing
  6. 6 module Contrib
  7. 6 module HTTPX
  8. 6 DATADOG_VERSION = defined?(::DDTrace) ? ::DDTrace::VERSION : ::Datadog::VERSION
  9. 6 METADATA_MODULE = Datadog::Tracing::Metadata
  10. 6 TYPE_OUTBOUND = Datadog::Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
  11. 6 TAG_BASE_SERVICE = Datadog::Tracing::Contrib::Ext::Metadata::TAG_BASE_SERVICE
  12. 6 TAG_PEER_HOSTNAME = Datadog::Tracing::Metadata::Ext::TAG_PEER_HOSTNAME
  13. 6 TAG_KIND = Datadog::Tracing::Metadata::Ext::TAG_KIND
  14. 6 TAG_CLIENT = Datadog::Tracing::Metadata::Ext::SpanKind::TAG_CLIENT
  15. 6 TAG_COMPONENT = Datadog::Tracing::Metadata::Ext::TAG_COMPONENT
  16. 6 TAG_OPERATION = Datadog::Tracing::Metadata::Ext::TAG_OPERATION
  17. 6 TAG_URL = Datadog::Tracing::Metadata::Ext::HTTP::TAG_URL
  18. 6 TAG_METHOD = Datadog::Tracing::Metadata::Ext::HTTP::TAG_METHOD
  19. 6 TAG_TARGET_HOST = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_HOST
  20. 6 TAG_TARGET_PORT = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_PORT
  21. 6 TAG_STATUS_CODE = Datadog::Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE
  22. # HTTPX Datadog Plugin
  23. #
  24. # Enables tracing for httpx requests.
  25. #
  26. # A span will be created for each request transaction; the span is created lazily only when
  27. # buffering a request, and it is fed the start time stored inside the tracer object.
  28. #
  29. 6 module Plugin
  30. 6 module RequestTracer
  31. 6 extend Contrib::HttpAnnotationHelper
  32. 6 module_function
  33. 6 SPAN_REQUEST = "httpx.request"
  34. # initializes tracing on the +request+.
  35. 6 def call(request)
  36. 105 return unless configuration(request).enabled
  37. 84 span = nil
  38. # request objects are reused, when already buffered requests get rerouted to a different
  39. # connection due to connection issues, or when they already got a response, but need to
  40. # be retried. In such situations, the original span needs to be extended for the former,
  41. # while a new is required for the latter.
  42. 84 request.on(:idle) do
  43. 18 span = nil
  44. end
  45. # the span is initialized when the request is buffered in the parser, which is the closest
  46. # one gets to actually sending the request.
  47. 84 request.on(:headers) do
  48. 96 next if span
  49. 96 span = initialize_span(request, now)
  50. end
  51. 84 request.on(:response) do |response|
  52. 96 unless span
  53. 6 next unless response.is_a?(::HTTPX::ErrorResponse) && response.error.respond_to?(:connection)
  54. # handles the case when the +error+ happened during name resolution, which means
  55. # that the tracing start point hasn't been triggered yet; in such cases, the approximate
  56. # initial resolving time is collected from the connection, and used as span start time,
  57. # and the tracing object in inserted before the on response callback is called.
  58. 6 span = initialize_span(request, response.error.connection.init_time)
  59. end
  60. 96 finish(response, span)
  61. end
  62. end
  63. 6 def finish(response, span)
  64. 96 if response.is_a?(::HTTPX::ErrorResponse)
  65. 6 span.set_error(response.error)
  66. else
  67. 90 span.set_tag(TAG_STATUS_CODE, response.status.to_s)
  68. 90 span.set_error(::HTTPX::HTTPError.new(response)) if response.status >= 400 && response.status <= 599
  69. 90 span.set_tags(
  70. Datadog.configuration.tracing.header_tags.response_tags(response.headers.to_h)
  71. )
  72. end
  73. 96 span.finish
  74. end
  75. # return a span initialized with the +@request+ state.
  76. 6 def initialize_span(request, start_time)
  77. 102 verb = request.verb
  78. 102 uri = request.uri
  79. 102 config = configuration(request)
  80. 102 span = create_span(request, config, start_time)
  81. 102 span.resource = verb
  82. # Tag original global service name if not used
  83. 102 span.set_tag(TAG_BASE_SERVICE, Datadog.configuration.service) if span.service != Datadog.configuration.service
  84. 102 span.set_tag(TAG_KIND, TAG_CLIENT)
  85. 102 span.set_tag(TAG_COMPONENT, "httpx")
  86. 102 span.set_tag(TAG_OPERATION, "request")
  87. 102 span.set_tag(TAG_URL, request.path)
  88. 102 span.set_tag(TAG_METHOD, verb)
  89. 102 span.set_tag(TAG_TARGET_HOST, uri.host)
  90. 102 span.set_tag(TAG_TARGET_PORT, uri.port)
  91. 102 span.set_tag(TAG_PEER_HOSTNAME, uri.host)
  92. # Tag as an external peer service
  93. # span.set_tag(TAG_PEER_SERVICE, span.service)
  94. 102 if config[:distributed_tracing]
  95. 96 propagate_trace_http(
  96. Datadog::Tracing.active_trace,
  97. request.headers
  98. )
  99. end
  100. # Set analytics sample rate
  101. 102 if Contrib::Analytics.enabled?(config[:analytics_enabled])
  102. 12 Contrib::Analytics.set_sample_rate(span, config[:analytics_sample_rate])
  103. end
  104. 102 span.set_tags(
  105. Datadog.configuration.tracing.header_tags.request_tags(request.headers.to_h)
  106. )
  107. 102 span
  108. rescue StandardError => e
  109. Datadog.logger.error("error preparing span for http request: #{e}")
  110. Datadog.logger.error(e.backtrace)
  111. end
  112. 6 def now
  113. 96 ::Datadog::Core::Utils::Time.now.utc
  114. end
  115. 6 def configuration(request)
  116. 207 Datadog.configuration.tracing[:httpx, request.uri.host]
  117. end
  118. 6 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("2.0.0")
  119. 3 def propagate_trace_http(trace, headers)
  120. 48 Datadog::Tracing::Contrib::HTTP.inject(trace, headers)
  121. end
  122. 3 def create_span(request, configuration, start_time)
  123. 51 Datadog::Tracing.trace(
  124. SPAN_REQUEST,
  125. service: service_name(request.uri.host, configuration),
  126. type: TYPE_OUTBOUND,
  127. start_time: start_time
  128. )
  129. end
  130. else
  131. 3 def propagate_trace_http(trace, headers)
  132. 48 Datadog::Tracing::Propagation::HTTP.inject!(trace.to_digest, headers)
  133. end
  134. 3 def create_span(request, configuration, start_time)
  135. 51 Datadog::Tracing.trace(
  136. SPAN_REQUEST,
  137. service: service_name(request.uri.host, configuration),
  138. span_type: TYPE_OUTBOUND,
  139. start_time: start_time
  140. )
  141. end
  142. end
  143. end
  144. 6 module RequestMethods
  145. # intercepts request initialization to inject the tracing logic.
  146. 6 def initialize(*)
  147. 105 super
  148. 105 return unless Datadog::Tracing.enabled?
  149. 105 RequestTracer.call(self)
  150. end
  151. end
  152. 6 module ConnectionMethods
  153. 6 attr_reader :init_time
  154. 6 def initialize(*)
  155. 97 super
  156. 97 @init_time = ::Datadog::Core::Utils::Time.now.utc
  157. end
  158. end
  159. end
  160. 6 module Configuration
  161. # Default settings for httpx
  162. #
  163. 6 class Settings < Datadog::Tracing::Contrib::Configuration::Settings
  164. 6 DEFAULT_ERROR_HANDLER = lambda do |response|
  165. Datadog::Ext::HTTP::ERROR_RANGE.cover?(response.status)
  166. end
  167. 6 option :service_name, default: "httpx"
  168. 6 option :distributed_tracing, default: true
  169. 6 option :split_by_domain, default: false
  170. 6 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  171. 6 option :enabled do |o|
  172. 6 o.type :bool
  173. 6 o.env "DD_TRACE_HTTPX_ENABLED"
  174. 6 o.default true
  175. end
  176. 6 option :analytics_enabled do |o|
  177. 6 o.type :bool
  178. 6 o.env "DD_TRACE_HTTPX_ANALYTICS_ENABLED"
  179. 6 o.default false
  180. end
  181. 6 option :analytics_sample_rate do |o|
  182. 6 o.type :float
  183. 6 o.env "DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE"
  184. 6 o.default 1.0
  185. end
  186. else
  187. option :enabled do |o|
  188. o.default { env_to_bool("DD_TRACE_HTTPX_ENABLED", true) }
  189. o.lazy
  190. end
  191. option :analytics_enabled do |o|
  192. o.default { env_to_bool(%w[DD_TRACE_HTTPX_ANALYTICS_ENABLED DD_HTTPX_ANALYTICS_ENABLED], false) }
  193. o.lazy
  194. end
  195. option :analytics_sample_rate do |o|
  196. o.default { env_to_float(%w[DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE DD_HTTPX_ANALYTICS_SAMPLE_RATE], 1.0) }
  197. o.lazy
  198. end
  199. end
  200. 6 if defined?(Datadog::Tracing::Contrib::SpanAttributeSchema)
  201. 6 option :service_name do |o|
  202. 6 o.default do
  203. 66 Datadog::Tracing::Contrib::SpanAttributeSchema.fetch_service_name(
  204. "DD_TRACE_HTTPX_SERVICE_NAME",
  205. "httpx"
  206. )
  207. end
  208. 6 o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  209. end
  210. else
  211. option :service_name do |o|
  212. o.default do
  213. ENV.fetch("DD_TRACE_HTTPX_SERVICE_NAME", "httpx")
  214. end
  215. o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  216. end
  217. end
  218. 6 option :distributed_tracing, default: true
  219. 6 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.15.0")
  220. 6 option :error_handler do |o|
  221. 6 o.type :proc
  222. 6 o.default_proc(&DEFAULT_ERROR_HANDLER)
  223. end
  224. elsif Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  225. option :error_handler do |o|
  226. o.type :proc
  227. o.experimental_default_proc(&DEFAULT_ERROR_HANDLER)
  228. end
  229. else
  230. option :error_handler, default: DEFAULT_ERROR_HANDLER
  231. end
  232. end
  233. end
  234. # Patcher enables patching of 'httpx' with datadog components.
  235. #
  236. 6 module Patcher
  237. 6 include Datadog::Tracing::Contrib::Patcher
  238. 6 module_function
  239. 6 def target_version
  240. 12 Integration.version
  241. end
  242. # loads a session instannce with the datadog plugin, and replaces the
  243. # base HTTPX::Session with the patched session class.
  244. 6 def patch
  245. 6 datadog_session = ::HTTPX.plugin(Plugin)
  246. 6 ::HTTPX.send(:remove_const, :Session)
  247. 6 ::HTTPX.send(:const_set, :Session, datadog_session.class)
  248. end
  249. end
  250. # Datadog Integration for HTTPX.
  251. #
  252. 6 class Integration
  253. 6 include Contrib::Integration
  254. 6 MINIMUM_VERSION = Gem::Version.new("0.10.2")
  255. 6 register_as :httpx
  256. 6 def self.version
  257. 246 Gem.loaded_specs["httpx"] && Gem.loaded_specs["httpx"].version
  258. end
  259. 6 def self.loaded?
  260. 78 defined?(::HTTPX::Request)
  261. end
  262. 6 def self.compatible?
  263. 78 super && version >= MINIMUM_VERSION
  264. end
  265. 6 def new_configuration
  266. 156 Configuration::Settings.new
  267. end
  268. 6 def patcher
  269. 156 Patcher
  270. end
  271. end
  272. end
  273. end
  274. end

lib/httpx/adapters/faraday.rb

98.13% lines covered

160 relevant lines. 157 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 15 require "delegate"
  3. 15 require "httpx"
  4. 15 require "faraday"
  5. 15 module Faraday
  6. 15 class Adapter
  7. 15 class HTTPX < Faraday::Adapter
  8. 15 module RequestMixin
  9. 15 def build_connection(env)
  10. 287 return @connection if defined?(@connection)
  11. 287 @connection = ::HTTPX.plugin(:persistent).plugin(ReasonPlugin)
  12. 287 @connection = @connection.with(@connection_options) unless @connection_options.empty?
  13. 287 connection_opts = options_from_env(env)
  14. 287 if (bind = env.request.bind)
  15. 8 @bind = TCPSocket.new(bind[:host], bind[:port])
  16. 7 connection_opts[:io] = @bind
  17. end
  18. 287 @connection = @connection.with(connection_opts)
  19. 287 if (proxy = env.request.proxy)
  20. 8 proxy_options = { uri: proxy.uri }
  21. 8 proxy_options[:username] = proxy.user if proxy.user
  22. 8 proxy_options[:password] = proxy.password if proxy.password
  23. 8 @connection = @connection.plugin(:proxy).with(proxy: proxy_options)
  24. end
  25. 287 @connection = @connection.plugin(OnDataPlugin) if env.request.stream_response?
  26. 287 @connection = @config_block.call(@connection) || @connection if @config_block
  27. 287 @connection
  28. end
  29. 15 def close
  30. 294 @connection.close if @connection
  31. 294 @bind.close if @bind
  32. end
  33. 15 private
  34. 15 def connect(env, &blk)
  35. 287 connection(env, &blk)
  36. rescue ::HTTPX::TLSError => e
  37. 8 raise Faraday::SSLError, e
  38. rescue Errno::ECONNABORTED,
  39. Errno::ECONNREFUSED,
  40. Errno::ECONNRESET,
  41. Errno::EHOSTUNREACH,
  42. Errno::EINVAL,
  43. Errno::ENETUNREACH,
  44. Errno::EPIPE,
  45. ::HTTPX::ConnectionError => e
  46. 8 raise Faraday::ConnectionFailed, e
  47. end
  48. 15 def build_request(env)
  49. 295 meth = env[:method]
  50. 30 request_options = {
  51. 264 headers: env.request_headers,
  52. body: env.body,
  53. **options_from_env(env),
  54. }
  55. 295 [meth.to_s.upcase, env.url, request_options]
  56. end
  57. 15 def options_from_env(env)
  58. 582 timeout_options = {}
  59. 582 req_opts = env.request
  60. 582 if (sec = request_timeout(:read, req_opts))
  61. 28 timeout_options[:read_timeout] = sec
  62. end
  63. 582 if (sec = request_timeout(:write, req_opts))
  64. 14 timeout_options[:write_timeout] = sec
  65. end
  66. 582 if (sec = request_timeout(:open, req_opts))
  67. 14 timeout_options[:connect_timeout] = sec
  68. end
  69. 59 {
  70. 522 ssl: ssl_options_from_env(env),
  71. timeout: timeout_options,
  72. }
  73. end
  74. 15 if defined?(::OpenSSL)
  75. 15 def ssl_options_from_env(env)
  76. 582 ssl_options = {}
  77. 582 unless env.ssl.verify.nil?
  78. 28 ssl_options[:verify_mode] = env.ssl.verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
  79. end
  80. 582 ssl_options[:ca_file] = env.ssl.ca_file if env.ssl.ca_file
  81. 582 ssl_options[:ca_path] = env.ssl.ca_path if env.ssl.ca_path
  82. 582 ssl_options[:cert_store] = env.ssl.cert_store if env.ssl.cert_store
  83. 582 ssl_options[:cert] = env.ssl.client_cert if env.ssl.client_cert
  84. 582 ssl_options[:key] = env.ssl.client_key if env.ssl.client_key
  85. 582 ssl_options[:ssl_version] = env.ssl.version if env.ssl.version
  86. 582 ssl_options[:verify_depth] = env.ssl.verify_depth if env.ssl.verify_depth
  87. 582 ssl_options[:min_version] = env.ssl.min_version if env.ssl.min_version
  88. 582 ssl_options[:max_version] = env.ssl.max_version if env.ssl.max_version
  89. 582 ssl_options
  90. end
  91. else
  92. skipped # :nocov:
  93. skipped def ssl_options_from_env(*)
  94. skipped {}
  95. skipped end
  96. skipped # :nocov:
  97. end
  98. end
  99. 15 include RequestMixin
  100. 15 module OnDataPlugin
  101. 15 module RequestMethods
  102. 15 attr_writer :response_on_data
  103. 15 def response=(response)
  104. 16 super
  105. 16 return if response.is_a?(::HTTPX::ErrorResponse)
  106. 16 response.body.on_data = @response_on_data
  107. end
  108. end
  109. 15 module ResponseBodyMethods
  110. 15 attr_writer :on_data
  111. 15 def write(chunk)
  112. 51 return super unless @on_data
  113. 51 @on_data.call(chunk, chunk.bytesize)
  114. end
  115. end
  116. end
  117. 15 module ReasonPlugin
  118. 15 def self.load_dependencies(*)
  119. 287 require "net/http/status"
  120. end
  121. 15 module ResponseMethods
  122. 15 def reason
  123. 241 Net::HTTP::STATUS_CODES.fetch(@status, "Non-Standard status code")
  124. end
  125. end
  126. end
  127. 15 class ParallelManager
  128. 15 class ResponseHandler < SimpleDelegator
  129. 15 attr_reader :env
  130. 15 def initialize(env)
  131. 32 @env = env
  132. 32 super
  133. end
  134. 15 def on_response(&blk)
  135. 64 if blk
  136. 32 @on_response = ->(response) do
  137. 32 blk.call(response)
  138. end
  139. 32 self
  140. else
  141. 32 @on_response
  142. end
  143. end
  144. 15 def on_complete(&blk)
  145. 32 if blk
  146. @on_complete = blk
  147. self
  148. else
  149. 32 @on_complete
  150. end
  151. end
  152. end
  153. 15 include RequestMixin
  154. 15 def initialize(options)
  155. 32 @handlers = []
  156. 32 @connection_options = options
  157. end
  158. 15 def enqueue(request)
  159. 32 handler = ResponseHandler.new(request)
  160. 32 @handlers << handler
  161. 32 handler
  162. end
  163. 15 def run
  164. 32 return unless @handlers.last
  165. 24 env = @handlers.last.env
  166. 24 connect(env) do |session|
  167. 56 requests = @handlers.map { |handler| session.build_request(*build_request(handler.env)) }
  168. 24 if env.request.stream_response?
  169. 8 requests.each do |request|
  170. 8 request.response_on_data = env.request.on_data
  171. end
  172. end
  173. 24 responses = session.request(*requests)
  174. 24 Array(responses).each_with_index do |response, index|
  175. 32 handler = @handlers[index]
  176. 32 handler.on_response.call(response)
  177. 32 handler.on_complete.call(handler.env) if handler.on_complete
  178. end
  179. end
  180. rescue ::HTTPX::TimeoutError => e
  181. raise Faraday::TimeoutError, e
  182. end
  183. # from Faraday::Adapter#connection
  184. 15 def connection(env)
  185. 24 conn = build_connection(env)
  186. 24 return conn unless block_given?
  187. 24 yield conn
  188. end
  189. 15 private
  190. # from Faraday::Adapter#request_timeout
  191. 15 def request_timeout(type, options)
  192. 168 key = Faraday::Adapter::TIMEOUT_KEYS[type]
  193. 168 options[key] || options[:timeout]
  194. end
  195. end
  196. 15 self.supports_parallel = true
  197. 15 class << self
  198. 15 def setup_parallel_manager(options = {})
  199. 32 ParallelManager.new(options)
  200. end
  201. end
  202. 15 def call(env)
  203. 295 super
  204. 295 if parallel?(env)
  205. 32 handler = env[:parallel_manager].enqueue(env)
  206. 32 handler.on_response do |response|
  207. 32 if response.is_a?(::HTTPX::Response)
  208. 24 save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
  209. 24 response_headers.merge!(response.headers)
  210. end
  211. else
  212. 7 env[:error] = response.error
  213. 8 save_response(env, 0, "", {}, nil)
  214. end
  215. end
  216. 28 return handler
  217. end
  218. 263 response = connect_and_request(env)
  219. 217 save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
  220. 217 response_headers.merge!(response.headers)
  221. end
  222. 217 @app.call(env)
  223. end
  224. 15 private
  225. 15 def connect_and_request(env)
  226. 263 connect(env) do |session|
  227. 263 request = session.build_request(*build_request(env))
  228. 263 request.response_on_data = env.request.on_data if env.request.stream_response?
  229. 263 response = session.request(request)
  230. # do not call #raise_for_status for HTTP 4xx or 5xx, as faraday has a middleware for that.
  231. 263 response.raise_for_status unless response.is_a?(::HTTPX::Response)
  232. 217 response
  233. end
  234. rescue ::HTTPX::TimeoutError => e
  235. 24 raise Faraday::TimeoutError, e
  236. end
  237. 15 def parallel?(env)
  238. 295 env[:parallel_manager]
  239. end
  240. end
  241. 15 register_middleware httpx: HTTPX
  242. end
  243. end

lib/httpx/adapters/sentry.rb

100.0% lines covered

62 relevant lines. 62 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "sentry-ruby"
  3. 6 module HTTPX::Plugins
  4. 6 module Sentry
  5. 6 module Tracer
  6. 6 module_function
  7. 6 def call(request)
  8. 129 sentry_span = start_sentry_span
  9. 129 return unless sentry_span
  10. 129 set_sentry_trace_header(request, sentry_span)
  11. 129 request.on(:response, &method(:finish_sentry_span).curry(3)[sentry_span, request])
  12. end
  13. 6 def start_sentry_span
  14. 129 return unless ::Sentry.initialized? && (span = ::Sentry.get_current_scope.get_span)
  15. 129 return if span.sampled == false
  16. 129 span.start_child(op: "httpx.client", start_timestamp: ::Sentry.utc_now.to_f)
  17. end
  18. 6 def set_sentry_trace_header(request, sentry_span)
  19. 129 return unless sentry_span
  20. 129 config = ::Sentry.configuration
  21. 129 url = request.uri.to_s
  22. 258 return unless config.propagate_traces && config.trace_propagation_targets.any? { |target| url.match?(target) }
  23. 129 trace = ::Sentry.get_current_client.generate_sentry_trace(sentry_span)
  24. 129 request.headers[::Sentry::SENTRY_TRACE_HEADER_NAME] = trace if trace
  25. end
  26. 6 def finish_sentry_span(span, request, response)
  27. 137 return unless ::Sentry.initialized?
  28. 137 record_sentry_breadcrumb(request, response)
  29. 137 record_sentry_span(request, response, span)
  30. end
  31. 6 def record_sentry_breadcrumb(req, res)
  32. 137 return unless ::Sentry.configuration.breadcrumbs_logger.include?(:http_logger)
  33. 137 request_info = extract_request_info(req)
  34. 137 data = if res.is_a?(HTTPX::ErrorResponse)
  35. 13 { error: res.error.message, **request_info }
  36. else
  37. 124 { status: res.status, **request_info }
  38. end
  39. 137 crumb = ::Sentry::Breadcrumb.new(
  40. level: :info,
  41. category: "httpx",
  42. type: :info,
  43. data: data
  44. )
  45. 137 ::Sentry.add_breadcrumb(crumb)
  46. end
  47. 6 def record_sentry_span(req, res, sentry_span)
  48. 137 return unless sentry_span
  49. 137 request_info = extract_request_info(req)
  50. 137 sentry_span.set_description("#{request_info[:method]} #{request_info[:url]}")
  51. 137 if res.is_a?(HTTPX::ErrorResponse)
  52. 13 sentry_span.set_data(:error, res.error.message)
  53. else
  54. 124 sentry_span.set_data(:status, res.status)
  55. end
  56. 137 sentry_span.set_timestamp(::Sentry.utc_now.to_f)
  57. end
  58. 6 def extract_request_info(req)
  59. 274 uri = req.uri
  60. result = {
  61. 274 method: req.verb,
  62. }
  63. 274 if ::Sentry.configuration.send_default_pii
  64. 24 uri += "?#{req.query}" unless req.query.empty?
  65. 24 result[:body] = req.body.to_s unless req.body.empty? || req.body.unbounded_body?
  66. end
  67. 274 result[:url] = uri.to_s
  68. 274 result
  69. end
  70. end
  71. 6 module RequestMethods
  72. 6 def __sentry_enable_trace!
  73. 137 return if @__sentry_enable_trace
  74. 129 Tracer.call(self)
  75. 129 @__sentry_enable_trace = true
  76. end
  77. end
  78. 6 module ConnectionMethods
  79. 6 def send(request)
  80. 137 request.__sentry_enable_trace!
  81. 137 super
  82. end
  83. end
  84. end
  85. end
  86. 6 Sentry.register_patch(:httpx) do
  87. 30 sentry_session = HTTPX.plugin(HTTPX::Plugins::Sentry)
  88. 30 HTTPX.send(:remove_const, :Session)
  89. 30 HTTPX.send(:const_set, :Session, sentry_session.class)
  90. end

lib/httpx/adapters/webmock.rb

100.0% lines covered

93 relevant lines. 93 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module WebMock
  3. 8 module HttpLibAdapters
  4. 8 require "net/http/status"
  5. 8 HTTP_REASONS = Net::HTTP::STATUS_CODES
  6. #
  7. # HTTPX plugin for webmock.
  8. #
  9. # Requests are "hijacked" at the session, before they're distributed to a connection.
  10. #
  11. 8 module Plugin
  12. 8 class << self
  13. 8 def build_webmock_request_signature(request)
  14. 188 uri = WebMock::Util::URI.heuristic_parse(request.uri)
  15. 188 uri.query = request.query
  16. 188 uri.path = uri.normalized_path.gsub("[^:]//", "/")
  17. 188 WebMock::RequestSignature.new(
  18. request.verb.downcase.to_sym,
  19. uri.to_s,
  20. body: request.body.to_s,
  21. headers: request.headers.to_h
  22. )
  23. end
  24. 8 def build_webmock_response(_request, response)
  25. 6 webmock_response = WebMock::Response.new
  26. 6 webmock_response.status = [response.status, HTTP_REASONS[response.status]]
  27. 6 webmock_response.body = response.body.to_s
  28. 6 webmock_response.headers = response.headers.to_h
  29. 6 webmock_response
  30. end
  31. 8 def build_from_webmock_response(request, webmock_response)
  32. 158 return build_error_response(request, HTTPX::TimeoutError.new(1, "Timed out")) if webmock_response.should_timeout
  33. 140 return build_error_response(request, webmock_response.exception) if webmock_response.exception
  34. 133 request.options.response_class.new(request,
  35. webmock_response.status[0],
  36. "2.0",
  37. webmock_response.headers).tap do |res|
  38. 133 res.mocked = true
  39. end
  40. end
  41. 8 def build_error_response(request, exception)
  42. 25 HTTPX::ErrorResponse.new(request, exception)
  43. end
  44. end
  45. 8 module InstanceMethods
  46. 8 private
  47. 8 def do_init_connection(connection, selector)
  48. 170 super
  49. 170 connection.once(:unmock_connection) do
  50. 24 next unless connection.current_session == self
  51. 24 unless connection.addresses
  52. # reset Happy Eyeballs, fail early
  53. 24 connection.sibling = nil
  54. 24 deselect_connection(connection, selector)
  55. end
  56. 24 resolve_connection(connection, selector)
  57. end
  58. end
  59. end
  60. 8 module ResponseMethods
  61. 8 attr_accessor :mocked
  62. 8 def initialize(*)
  63. 157 super
  64. 157 @mocked = false
  65. end
  66. end
  67. 8 module ResponseBodyMethods
  68. 8 def decode_chunk(chunk)
  69. 96 return chunk if @response.mocked
  70. 42 super
  71. end
  72. end
  73. 8 module ConnectionMethods
  74. 8 def initialize(*)
  75. 170 super
  76. 170 @mocked = true
  77. end
  78. 8 def open?
  79. 194 return true if @mocked
  80. 24 super
  81. end
  82. 8 def interests
  83. 278 return if @mocked
  84. 246 super
  85. end
  86. 8 def terminate
  87. 145 force_reset
  88. end
  89. 8 def send(request)
  90. 188 request_signature = Plugin.build_webmock_request_signature(request)
  91. 188 WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)
  92. 188 if (mock_response = WebMock::StubRegistry.instance.response_for_request(request_signature))
  93. 158 response = Plugin.build_from_webmock_response(request, mock_response)
  94. 158 WebMock::CallbackRegistry.invoke_callbacks({ lib: :httpx }, request_signature, mock_response)
  95. 158 log { "mocking #{request.uri} with #{mock_response.inspect}" }
  96. 158 request.transition(:headers)
  97. 158 request.transition(:body)
  98. 158 request.transition(:trailers)
  99. 158 request.transition(:done)
  100. 158 response.finish!
  101. 158 request.response = response
  102. 158 request.emit(:response, response)
  103. 158 request_signature.headers = request.headers.to_h
  104. 158 response << mock_response.body.dup unless response.is_a?(HTTPX::ErrorResponse)
  105. 30 elsif WebMock.net_connect_allowed?(request_signature.uri)
  106. 24 if WebMock::CallbackRegistry.any_callbacks?
  107. 6 request.on(:response) do |resp|
  108. 6 unless resp.is_a?(HTTPX::ErrorResponse)
  109. 6 webmock_response = Plugin.build_webmock_response(request, resp)
  110. 6 WebMock::CallbackRegistry.invoke_callbacks(
  111. { lib: :httpx, real_request: true }, request_signature,
  112. webmock_response
  113. )
  114. end
  115. end
  116. end
  117. 24 @mocked = false
  118. 24 emit(:unmock_connection, self)
  119. 24 super
  120. else
  121. 6 raise WebMock::NetConnectNotAllowedError, request_signature
  122. end
  123. end
  124. end
  125. end
  126. 8 class HttpxAdapter < HttpLibAdapter
  127. 8 adapter_for :httpx
  128. 8 class << self
  129. 8 def enable!
  130. 370 @original_session ||= HTTPX::Session
  131. 370 webmock_session = HTTPX.plugin(Plugin)
  132. 370 HTTPX.send(:remove_const, :Session)
  133. 370 HTTPX.send(:const_set, :Session, webmock_session.class)
  134. end
  135. 8 def disable!
  136. 370 return unless @original_session
  137. 362 HTTPX.send(:remove_const, :Session)
  138. 362 HTTPX.send(:const_set, :Session, @original_session)
  139. end
  140. end
  141. end
  142. end
  143. end

lib/httpx/altsvc.rb

96.39% lines covered

83 relevant lines. 80 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "strscan"
  3. 27 module HTTPX
  4. 27 module AltSvc
  5. # makes connections able to accept requests destined to primary service.
  6. 27 module ConnectionMixin
  7. 27 using URIExtensions
  8. 27 def send(request)
  9. 8 request.headers["alt-used"] = @origin.authority if @parser && !@write_buffer.full? && match_altsvcs?(request.uri)
  10. 8 super
  11. end
  12. 27 def match?(uri, options)
  13. 8 return false if !used? && (@state == :closing || @state == :closed)
  14. 8 match_altsvcs?(uri) && match_altsvc_options?(uri, options)
  15. end
  16. 27 private
  17. # checks if this is connection is an alternative service of
  18. # +uri+
  19. 27 def match_altsvcs?(uri)
  20. 24 @origins.any? { |origin| altsvc_match?(uri, origin) } ||
  21. AltSvc.cached_altsvc(@origin).any? do |altsvc|
  22. origin = altsvc["origin"]
  23. altsvc_match?(origin, uri.origin)
  24. end
  25. end
  26. 27 def match_altsvc_options?(uri, options)
  27. 8 return @options == options unless @options.ssl.all? do |k, v|
  28. 8 v == (k == :hostname ? uri.host : options.ssl[k])
  29. end
  30. 8 @options.options_equals?(options, Options::REQUEST_BODY_IVARS + %i[@ssl])
  31. end
  32. 27 def altsvc_match?(uri, other_uri)
  33. 16 other_uri = URI(other_uri)
  34. 16 uri.origin == other_uri.origin || begin
  35. 7 case uri.scheme
  36. when "h2"
  37. (other_uri.scheme == "https" || other_uri.scheme == "h2") &&
  38. uri.host == other_uri.host &&
  39. uri.port == other_uri.port
  40. else
  41. 8 false
  42. end
  43. end
  44. end
  45. end
  46. 27 @altsvc_mutex = Thread::Mutex.new
  47. 48 @altsvcs = Hash.new { |h, k| h[k] = [] }
  48. 27 module_function
  49. 27 def cached_altsvc(origin)
  50. 40 now = Utils.now
  51. 40 @altsvc_mutex.synchronize do
  52. 40 lookup(origin, now)
  53. end
  54. end
  55. 27 def cached_altsvc_set(origin, entry)
  56. 24 now = Utils.now
  57. 24 @altsvc_mutex.synchronize do
  58. 24 return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
  59. 24 entry["TTL"] = Integer(entry["ma"]) + now if entry.key?("ma")
  60. 24 @altsvcs[origin] << entry
  61. 24 entry
  62. end
  63. end
  64. 27 def lookup(origin, ttl)
  65. 40 return [] unless @altsvcs.key?(origin)
  66. 28 @altsvcs[origin] = @altsvcs[origin].select do |entry|
  67. 24 !entry.key?("TTL") || entry["TTL"] > ttl
  68. end
  69. 48 @altsvcs[origin].reject { |entry| entry["noop"] }
  70. end
  71. 27 def emit(request, response)
  72. 7981 return unless response.respond_to?(:headers)
  73. # Alt-Svc
  74. 7955 return unless response.headers.key?("alt-svc")
  75. 80 origin = request.origin
  76. 80 host = request.uri.host
  77. 80 altsvc = response.headers["alt-svc"]
  78. # https://datatracker.ietf.org/doc/html/rfc7838#section-3
  79. # A field value containing the special value "clear" indicates that the
  80. # origin requests all alternatives for that origin to be invalidated
  81. # (including those specified in the same response, in case of an
  82. # invalid reply containing both "clear" and alternative services).
  83. 80 if altsvc == "clear"
  84. 8 @altsvc_mutex.synchronize do
  85. 8 @altsvcs[origin].clear
  86. end
  87. 7 return
  88. end
  89. 72 parse(altsvc) do |alt_origin, alt_params|
  90. 8 alt_origin.host ||= host
  91. 8 yield(alt_origin, origin, alt_params)
  92. end
  93. end
  94. 27 def parse(altsvc)
  95. 184 return enum_for(__method__, altsvc) unless block_given?
  96. 128 scanner = StringScanner.new(altsvc)
  97. 134 until scanner.eos?
  98. 128 alt_service = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
  99. 128 alt_params = []
  100. 128 loop do
  101. 152 alt_param = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
  102. 152 alt_params << alt_param.strip if alt_param
  103. 152 scanner.skip(/;/)
  104. 152 break if scanner.eos? || scanner.scan(/ *, */)
  105. end
  106. 256 alt_params = Hash[alt_params.map { |field| field.split("=", 2) }]
  107. 128 alt_proto, alt_authority = alt_service.split("=", 2)
  108. 128 alt_origin = parse_altsvc_origin(alt_proto, alt_authority)
  109. 128 return unless alt_origin
  110. 48 yield(alt_origin, alt_params.merge("proto" => alt_proto))
  111. end
  112. end
  113. 27 def parse_altsvc_scheme(alt_proto)
  114. 135 case alt_proto
  115. when "h2c"
  116. 8 "http"
  117. when "h2"
  118. 56 "https"
  119. end
  120. end
  121. 27 def parse_altsvc_origin(alt_proto, alt_origin)
  122. 128 alt_scheme = parse_altsvc_scheme(alt_proto)
  123. 128 return unless alt_scheme
  124. 48 alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
  125. 48 URI.parse("#{alt_scheme}://#{alt_origin}")
  126. end
  127. end
  128. end

lib/httpx/buffer.rb

100.0% lines covered

27 relevant lines. 27 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 module HTTPX
  4. # Internal class to abstract a string buffer, by wrapping a string and providing the
  5. # minimum possible API and functionality required.
  6. #
  7. # buffer = Buffer.new(640)
  8. # buffer.full? #=> false
  9. # buffer << "aa"
  10. # buffer.capacity #=> 638
  11. #
  12. 27 class Buffer
  13. 27 extend Forwardable
  14. 27 def_delegator :@buffer, :to_s
  15. 27 def_delegator :@buffer, :to_str
  16. 27 def_delegator :@buffer, :empty?
  17. 27 def_delegator :@buffer, :bytesize
  18. 27 def_delegator :@buffer, :clear
  19. 27 def_delegator :@buffer, :replace
  20. 27 attr_reader :limit
  21. 27 if RUBY_VERSION >= "3.4.0"
  22. 15 def initialize(limit)
  23. 2896 @buffer = String.new("", encoding: Encoding::BINARY, capacity: limit)
  24. 2896 @limit = limit
  25. end
  26. 15 def <<(chunk)
  27. 12169 @buffer.append_as_bytes(chunk)
  28. end
  29. else
  30. 12 def initialize(limit)
  31. 18876 @buffer = "".b
  32. 18876 @limit = limit
  33. end
  34. 12 def_delegator :@buffer, :<<
  35. end
  36. 27 def full?
  37. 11594195 @buffer.bytesize >= @limit
  38. end
  39. 27 def capacity
  40. 12 @limit - @buffer.bytesize
  41. end
  42. 27 def shift!(fin)
  43. 21982 @buffer = @buffer.byteslice(fin..-1) || "".b
  44. end
  45. end
  46. end

lib/httpx/callbacks.rb

100.0% lines covered

19 relevant lines. 19 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Callbacks
  4. 27 def on(type, &action)
  5. 353445 callbacks(type) << action
  6. 353445 action
  7. end
  8. 27 def once(type, &block)
  9. 192275 on(type) do |*args, &callback|
  10. 99015 block.call(*args, &callback)
  11. 98951 :delete
  12. end
  13. end
  14. 27 def emit(type, *args)
  15. 128804 log { "emit #{type.inspect} callbacks" } if respond_to?(:log)
  16. 270258 callbacks(type).delete_if { |pr| :delete == pr.call(*args) } # rubocop:disable Style/YodaCondition
  17. end
  18. 27 def callbacks_for?(type)
  19. 3046 @callbacks.key?(type) && @callbacks[type].any?
  20. end
  21. 27 protected
  22. 27 def callbacks(type = nil)
  23. 485606 return @callbacks unless type
  24. 700159 @callbacks ||= Hash.new { |h, k| h[k] = [] }
  25. 485519 @callbacks[type]
  26. end
  27. end
  28. end

lib/httpx/chainable.rb

95.65% lines covered

46 relevant lines. 44 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # Session mixin, implements most of the APIs that the users call.
  4. # delegates to a default session when extended.
  5. 27 module Chainable
  6. 27 %w[head get post put delete trace options connect patch].each do |meth|
  7. 234 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  8. 9 def #{meth}(*uri, **options) # def get(*uri, **options)
  9. 18 request("#{meth.upcase}", uri, **options) # request("GET", uri, **options)
  10. end # end
  11. MOD
  12. end
  13. # delegates to the default session (see HTTPX::Session#request).
  14. 27 def request(*args, **options)
  15. 2670 branch(default_options).request(*args, **options)
  16. end
  17. 27 def accept(type)
  18. 16 with(headers: { "accept" => String(type) })
  19. end
  20. # delegates to the default session (see HTTPX::Session#wrap).
  21. 27 def wrap(&blk)
  22. 89 branch(default_options).wrap(&blk)
  23. end
  24. # returns a new instance loaded with the +pl+ plugin and +options+.
  25. 27 def plugin(pl, options = nil, &blk)
  26. 5199 klass = is_a?(S) ? self.class : Session
  27. 5199 klass = Class.new(klass)
  28. 5199 klass.instance_variable_set(:@default_options, klass.default_options.merge(default_options))
  29. 5199 klass.plugin(pl, options, &blk).new
  30. end
  31. # returns a new instance loaded with +options+.
  32. 27 def with(options, &blk)
  33. 2737 branch(default_options.merge(options), &blk)
  34. end
  35. 27 private
  36. # returns default instance of HTTPX::Options.
  37. 27 def default_options
  38. 10751 @options || Session.default_options
  39. end
  40. # returns a default instance of HTTPX::Session.
  41. 27 def branch(options, &blk)
  42. 5480 return self.class.new(options, &blk) if is_a?(S)
  43. 3211 Session.new(options, &blk)
  44. end
  45. 27 def method_missing(meth, *args, **options, &blk)
  46. 652 case meth
  47. when /\Awith_(.+)/
  48. 729 option = Regexp.last_match(1)
  49. 729 return super unless option
  50. 729 with(option.to_sym => args.first || options)
  51. when /\Aon_(.+)/
  52. 9 callback = Regexp.last_match(1)
  53. 6 return super unless %w[
  54. connection_opened connection_closed
  55. request_error
  56. request_started request_body_chunk request_completed
  57. response_started response_body_chunk response_completed
  58. 2 ].include?(callback)
  59. 9 warn "DEPRECATION WARNING: calling `.#{meth}` on plain HTTPX sessions is deprecated. " \
  60. 1 "Use `HTTPX.plugin(:callbacks).#{meth}` instead."
  61. 9 plugin(:callbacks).__send__(meth, *args, **options, &blk)
  62. else
  63. super
  64. end
  65. end
  66. 27 def respond_to_missing?(meth, *)
  67. 49 case meth
  68. when /\Awith_(.+)/
  69. 40 option = Regexp.last_match(1)
  70. 40 default_options.respond_to?(option) || super
  71. when /\Aon_(.+)/
  72. 16 callback = Regexp.last_match(1)
  73. 12 %w[
  74. connection_opened connection_closed
  75. request_error
  76. request_started request_body_chunk request_completed
  77. response_started response_body_chunk response_completed
  78. 3 ].include?(callback) || super
  79. else
  80. super
  81. end
  82. end
  83. end
  84. 27 extend Chainable
  85. end

lib/httpx/connection.rb

92.52% lines covered

481 relevant lines. 445 lines covered and 36 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "forwardable"
  4. 27 require "httpx/io"
  5. 27 require "httpx/buffer"
  6. 27 module HTTPX
  7. # The Connection can be watched for IO events.
  8. #
  9. # It contains the +io+ object to read/write from, and knows what to do when it can.
  10. #
  11. # It defers connecting until absolutely necessary. Connection should be triggered from
  12. # the IO selector (until then, any request will be queued).
  13. #
  14. # A connection boots up its parser after connection is established. All pending requests
  15. # will be redirected there after connection.
  16. #
  17. # A connection can be prevented from closing by the parser, that is, if there are pending
  18. # requests. This will signal that the connection was prematurely closed, due to a possible
  19. # number of conditions:
  20. #
  21. # * Remote peer closed the connection ("Connection: close");
  22. # * Remote peer doesn't support pipelining;
  23. #
  24. # A connection may also route requests for a different host for which the +io+ was connected
  25. # to, provided that the IP is the same and the port and scheme as well. This will allow to
  26. # share the same socket to send HTTP/2 requests to different hosts.
  27. #
  28. 27 class Connection
  29. 27 extend Forwardable
  30. 27 include Loggable
  31. 27 include Callbacks
  32. 27 using URIExtensions
  33. 27 require "httpx/connection/http2"
  34. 27 require "httpx/connection/http1"
  35. 27 def_delegator :@io, :closed?
  36. 27 def_delegator :@write_buffer, :empty?
  37. 27 attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling
  38. 27 attr_writer :current_selector
  39. 27 attr_accessor :current_session, :family
  40. 27 protected :sibling
  41. 27 def initialize(uri, options)
  42. 7384 @current_session = @current_selector = @sibling = @coalesced_connection = nil
  43. 7384 @exhausted = @cloned = @main_sibling = false
  44. 7384 @options = Options.new(options)
  45. 7384 @type = initialize_type(uri, @options)
  46. 7384 @origins = [uri.origin]
  47. 7384 @origin = Utils.to_uri(uri.origin)
  48. 7384 @window_size = @options.window_size
  49. 7384 @read_buffer = Buffer.new(@options.buffer_size)
  50. 7384 @write_buffer = Buffer.new(@options.buffer_size)
  51. 7384 @pending = []
  52. 7384 on(:error, &method(:on_error))
  53. 7384 if @options.io
  54. # if there's an already open IO, get its
  55. # peer address, and force-initiate the parser
  56. 66 transition(:already_open)
  57. 66 @io = build_socket
  58. 66 parser
  59. else
  60. 7318 transition(:idle)
  61. end
  62. 7384 on(:close) do
  63. 7567 next if @exhausted # it'll reset
  64. # may be called after ":close" above, so after the connection has been checked back in.
  65. # next unless @current_session
  66. 7559 next unless @current_session
  67. 7559 @current_session.deselect_connection(self, @current_selector, @cloned)
  68. end
  69. 7384 on(:terminate) do
  70. 2807 next if @exhausted # it'll reset
  71. 2799 current_session = @current_session
  72. 2799 current_selector = @current_selector
  73. # may be called after ":close" above, so after the connection has been checked back in.
  74. 2799 next unless current_session && current_selector
  75. 16 current_session.deselect_connection(self, current_selector)
  76. end
  77. 7384 on(:altsvc) do |alt_origin, origin, alt_params|
  78. 8 build_altsvc_connection(alt_origin, origin, alt_params)
  79. end
  80. 7384 @inflight = 0
  81. 7384 @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
  82. 7384 self.addresses = @options.addresses if @options.addresses
  83. end
  84. 27 def peer
  85. 14962 @origin
  86. end
  87. # this is a semi-private method, to be used by the resolver
  88. # to initiate the io object.
  89. 27 def addresses=(addrs)
  90. 7170 if @io
  91. 241 @io.add_addresses(addrs)
  92. else
  93. 6929 @io = build_socket(addrs)
  94. end
  95. end
  96. 27 def addresses
  97. 14751 @io && @io.addresses
  98. end
  99. 27 def match?(uri, options)
  100. 2090 return false if !used? && (@state == :closing || @state == :closed)
  101. 314 (
  102. 1815 @origins.include?(uri.origin) &&
  103. # if there is more than one origin to match, it means that this connection
  104. # was the result of coalescing. To prevent blind trust in the case where the
  105. # origin came from an ORIGIN frame, we're going to verify the hostname with the
  106. # SSL certificate
  107. 1885 (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
  108. ) && @options == options
  109. end
  110. 27 def expired?
  111. return false unless @io
  112. @io.expired?
  113. end
  114. 27 def mergeable?(connection)
  115. 319 return false if @state == :closing || @state == :closed || !@io
  116. 64 return false unless connection.addresses
  117. 2 (
  118. 64 (open? && @origin == connection.origin) ||
  119. 64 !(@io.addresses & (connection.addresses || [])).empty?
  120. ) && @options == connection.options
  121. end
  122. # coalescable connections need to be mergeable!
  123. # but internally, #mergeable? is called before #coalescable?
  124. 27 def coalescable?(connection)
  125. 27 if @io.protocol == "h2" &&
  126. @origin.scheme == "https" &&
  127. connection.origin.scheme == "https" &&
  128. @io.can_verify_peer?
  129. 13 @io.verify_hostname(connection.origin.host)
  130. else
  131. 14 @origin == connection.origin
  132. end
  133. end
  134. 27 def create_idle(options = {})
  135. self.class.new(@origin, @options.merge(options))
  136. end
  137. 27 def merge(connection)
  138. 34 @origins |= connection.instance_variable_get(:@origins)
  139. 37 if connection.ssl_session
  140. 14 @ssl_session = connection.ssl_session
  141. 2 @io.session_new_cb do |sess|
  142. 21 @ssl_session = sess
  143. 14 end if @io
  144. end
  145. 37 connection.purge_pending do |req|
  146. 7 send(req)
  147. end
  148. end
  149. 27 def purge_pending(&block)
  150. 37 pendings = []
  151. 37 if @parser
  152. 21 @inflight -= @parser.pending.size
  153. 24 pendings << @parser.pending
  154. end
  155. 37 pendings << @pending
  156. 37 pendings.each do |pending|
  157. 61 pending.reject!(&block)
  158. end
  159. end
  160. 27 def io_connected?
  161. return @coalesced_connection.io_connected? if @coalesced_connection
  162. @io && @io.state == :connected
  163. end
  164. 27 def connecting?
  165. 11629667 @state == :idle
  166. end
  167. 27 def inflight?
  168. 2921 @parser && (
  169. # parser may be dealing with other requests (possibly started from a different fiber)
  170. 2451 !@parser.empty? ||
  171. # connection may be doing connection termination handshake
  172. !@write_buffer.empty?
  173. )
  174. end
  175. 27 def interests
  176. # connecting
  177. 11617982 if connecting?
  178. 11385 connect
  179. 11384 return @io.interests if connecting?
  180. end
  181. # if the write buffer is full, we drain it
  182. 11607163 return :w unless @write_buffer.empty?
  183. 11565535 return @parser.interests if @parser
  184. 21 nil
  185. rescue StandardError => e
  186. emit(:error, e)
  187. nil
  188. end
  189. 27 def to_io
  190. 22930 @io.to_io
  191. end
  192. 27 def call
  193. 20281 case @state
  194. when :idle
  195. 10670 connect
  196. 10655 consume
  197. when :closed
  198. return
  199. when :closing
  200. consume
  201. transition(:closed)
  202. when :open
  203. 11314 consume
  204. end
  205. 8681 nil
  206. rescue StandardError => e
  207. 23 emit(:error, e)
  208. 23 raise e
  209. end
  210. 27 def close
  211. 2759 transition(:active) if @state == :inactive
  212. 2759 @parser.close if @parser
  213. end
  214. 27 def terminate
  215. 2759 @connected_at = nil if @state == :closed
  216. 2759 close
  217. end
  218. # bypasses the state machine to force closing of connections still connecting.
  219. # **only** used for Happy Eyeballs v2.
  220. 27 def force_reset(cloned = false)
  221. 292 @state = :closing
  222. 292 @cloned = cloned
  223. 292 transition(:closed)
  224. end
  225. 27 def reset
  226. 7480 return if @state == :closing || @state == :closed
  227. 7424 transition(:closing)
  228. 7424 transition(:closed)
  229. end
  230. 27 def send(request)
  231. 8970 return @coalesced_connection.send(request) if @coalesced_connection
  232. 8964 if @parser && !@write_buffer.full?
  233. 427 if @response_received_at && @keep_alive_timeout &&
  234. Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
  235. # when pushing a request into an existing connection, we have to check whether there
  236. # is the possibility that the connection might have extended the keep alive timeout.
  237. # for such cases, we want to ping for availability before deciding to shovel requests.
  238. 20 log(level: 3) { "keep alive timeout expired, pinging connection..." }
  239. 20 @pending << request
  240. 20 transition(:active) if @state == :inactive
  241. 20 parser.ping
  242. 20 request.ping!
  243. 18 return
  244. end
  245. 407 send_request_to_parser(request)
  246. else
  247. 8537 @pending << request
  248. end
  249. end
  250. 27 def timeout
  251. 11492307 return if @state == :closed || @state == :inactive
  252. 11492307 return @timeout if @timeout
  253. 11479577 return @options.timeout[:connect_timeout] if @state == :idle
  254. 11479577 @options.timeout[:operation_timeout]
  255. end
  256. 27 def idling
  257. 819 purge_after_closed
  258. 819 @write_buffer.clear
  259. 819 transition(:idle)
  260. 819 @parser = nil if @parser
  261. end
  262. 27 def used?
  263. 2309 @connected_at
  264. end
  265. 27 def deactivate
  266. 366 transition(:inactive)
  267. end
  268. 27 def open?
  269. 7227 @state == :open || @state == :inactive
  270. end
  271. 27 def handle_socket_timeout(interval)
  272. 32 error = OperationTimeoutError.new(interval, "timed out while waiting on select")
  273. 32 error.set_backtrace(caller)
  274. 32 on_error(error)
  275. end
  276. 27 def coalesced_connection=(connection)
  277. 13 @coalesced_connection = connection
  278. 13 close_sibling
  279. 13 connection.merge(self)
  280. end
  281. 27 def sibling=(connection)
  282. 24 @sibling = connection
  283. 24 return unless connection
  284. @main_sibling = connection.sibling.nil?
  285. return unless @main_sibling
  286. connection.sibling = self
  287. end
  288. 27 def handle_connect_error(error)
  289. 268 @connect_error = error
  290. 268 return handle_error(error) unless @sibling && @sibling.connecting?
  291. @sibling.merge(self)
  292. force_reset(true)
  293. end
  294. 27 def disconnect
  295. 7689 return unless @current_session && @current_selector
  296. 7567 emit(:close)
  297. 7551 @current_session = nil
  298. 7551 @current_selector = nil
  299. end
  300. skipped # :nocov:
  301. skipped def inspect
  302. skipped "#<#{self.class}:#{object_id} " \
  303. skipped "@origin=#{@origin} " \
  304. skipped "@state=#{@state} " \
  305. skipped "@pending=#{@pending.size} " \
  306. skipped "@io=#{@io}>"
  307. skipped end
  308. skipped # :nocov:
  309. 27 private
  310. 27 def connect
  311. 21047 transition(:open)
  312. end
  313. 27 def consume
  314. 25229 return unless @io
  315. 25229 catch(:called) do
  316. 25229 epiped = false
  317. 25229 loop do
  318. # connection may have
  319. 45607 return if @state == :idle
  320. 42298 parser.consume
  321. # we exit if there's no more requests to process
  322. #
  323. # this condition takes into account:
  324. #
  325. # * the number of inflight requests
  326. # * the number of pending requests
  327. # * whether the write buffer has bytes (i.e. for close handshake)
  328. 42282 if @pending.empty? && @inflight.zero? && @write_buffer.empty?
  329. 2906 log(level: 3) { "NO MORE REQUESTS..." }
  330. 2890 return
  331. end
  332. 39392 @timeout = @current_timeout
  333. 39392 read_drained = false
  334. 39392 write_drained = nil
  335. #
  336. # tight read loop.
  337. #
  338. # read as much of the socket as possible.
  339. #
  340. # this tight loop reads all the data it can from the socket and pipes it to
  341. # its parser.
  342. #
  343. 7564 loop do
  344. 51067 siz = @io.read(@window_size, @read_buffer)
  345. 51181 log(level: 3, color: :cyan) { "IO READ: #{siz} bytes... (wsize: #{@window_size}, rbuffer: #{@read_buffer.bytesize})" }
  346. 51067 unless siz
  347. 19 @write_buffer.clear
  348. 19 ex = EOFError.new("descriptor closed")
  349. 19 ex.set_backtrace(caller)
  350. 19 on_error(ex)
  351. 19 return
  352. end
  353. # socket has been drained. mark and exit the read loop.
  354. 51048 if siz.zero?
  355. 10813 read_drained = @read_buffer.empty?
  356. 10813 epiped = false
  357. 10813 break
  358. end
  359. 40235 parser << @read_buffer.to_s
  360. # continue reading if possible.
  361. 35815 break if interests == :w && !epiped
  362. # exit the read loop if connection is preparing to be closed
  363. 28335 break if @state == :closing || @state == :closed
  364. # exit #consume altogether if all outstanding requests have been dealt with
  365. 28320 return if @pending.empty? && @inflight.zero?
  366. 39392 end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
  367. #
  368. # tight write loop.
  369. #
  370. # flush as many bytes as the sockets allow.
  371. #
  372. 6145 loop do
  373. # buffer has been drainned, mark and exit the write loop.
  374. 24117 if @write_buffer.empty?
  375. # we only mark as drained on the first loop
  376. 2839 write_drained = write_drained.nil? && @inflight.positive?
  377. 2839 break
  378. end
  379. 2086 begin
  380. 21278 siz = @io.write(@write_buffer)
  381. rescue Errno::EPIPE
  382. # this can happen if we still have bytes in the buffer to send to the server, but
  383. # the server wants to respond immediately with some message, or an error. An example is
  384. # when one's uploading a big file to an unintended endpoint, and the server stops the
  385. # consumption, and responds immediately with an authorization of even method not allowed error.
  386. # at this point, we have to let the connection switch to read-mode.
  387. 11 log(level: 2) { "pipe broken, could not flush buffer..." }
  388. 11 epiped = true
  389. 11 read_drained = false
  390. 11 break
  391. end
  392. 21343 log(level: 3, color: :cyan) { "IO WRITE: #{siz} bytes..." }
  393. 21257 unless siz
  394. @write_buffer.clear
  395. ex = EOFError.new("descriptor closed")
  396. ex.set_backtrace(caller)
  397. on_error(ex)
  398. return
  399. end
  400. # socket closed for writing. mark and exit the write loop.
  401. 21257 if siz.zero?
  402. 16 write_drained = !@write_buffer.empty?
  403. 16 break
  404. end
  405. # exit write loop if marked to consume from peer, or is closing.
  406. 21241 break if interests == :r || @state == :closing || @state == :closed
  407. 3074 write_drained = false
  408. 31855 end unless (ints = interests) == :r
  409. 31845 send_pending if @state == :open
  410. # return if socket is drained
  411. 31845 next unless (ints != :r || read_drained) && (ints != :w || write_drained)
  412. # gotta go back to the event loop. It happens when:
  413. #
  414. # * the socket is drained of bytes or it's not the interest of the conn to read;
  415. # * theres nothing more to write, or it's not in the interest of the conn to write;
  416. 11504 log(level: 3) { "(#{ints}): WAITING FOR EVENTS..." }
  417. 11467 return
  418. end
  419. end
  420. end
  421. 27 def send_pending
  422. 82055 while !@write_buffer.full? && (request = @pending.shift)
  423. 18707 send_request_to_parser(request)
  424. end
  425. end
  426. 27 def parser
  427. 110293 @parser ||= build_parser
  428. end
  429. 27 def send_request_to_parser(request)
  430. 18188 @inflight += 1
  431. 19114 request.peer_address = @io.ip
  432. 19114 set_request_timeouts(request)
  433. 19114 parser.send(request)
  434. 19114 return unless @state == :inactive
  435. 7 transition(:active)
  436. end
  437. 27 def build_parser(protocol = @io.protocol)
  438. 7473 parser = parser_type(protocol).new(@write_buffer, @options)
  439. 7473 set_parser_callbacks(parser)
  440. 7473 parser
  441. end
  442. 27 def set_parser_callbacks(parser)
  443. 7584 parser.on(:response) do |request, response|
  444. 7973 AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
  445. 8 emit(:altsvc, alt_origin, origin, alt_params)
  446. end
  447. 7973 @response_received_at = Utils.now
  448. 7125 @inflight -= 1
  449. 7973 response.finish!
  450. 7973 request.emit(:response, response)
  451. end
  452. 7584 parser.on(:altsvc) do |alt_origin, origin, alt_params|
  453. emit(:altsvc, alt_origin, origin, alt_params)
  454. end
  455. 7584 parser.on(:pong, &method(:send_pending))
  456. 7584 parser.on(:promise) do |request, stream|
  457. 24 request.emit(:promise, parser, stream)
  458. end
  459. 7584 parser.on(:exhausted) do
  460. 8 @exhausted = true
  461. 8 current_session = @current_session
  462. 8 current_selector = @current_selector
  463. begin
  464. 8 parser.close
  465. 8 @pending.concat(parser.pending)
  466. ensure
  467. 8 @current_session = current_session
  468. 8 @current_selector = current_selector
  469. end
  470. 7 case @state
  471. when :closed
  472. 8 idling
  473. 8 @exhausted = false
  474. when :closing
  475. once(:closed) do
  476. idling
  477. @exhausted = false
  478. end
  479. end
  480. end
  481. 7584 parser.on(:origin) do |origin|
  482. @origins |= [origin]
  483. end
  484. 7584 parser.on(:close) do |force|
  485. 2807 if force
  486. 2807 reset
  487. 2799 emit(:terminate)
  488. end
  489. end
  490. 7584 parser.on(:close_handshake) do
  491. 8 consume
  492. end
  493. 7584 parser.on(:reset) do
  494. 4037 @pending.concat(parser.pending) unless parser.empty?
  495. 4037 current_session = @current_session
  496. 4037 current_selector = @current_selector
  497. 4037 reset
  498. 4029 unless @pending.empty?
  499. 179 idling
  500. 179 @current_session = current_session
  501. 179 @current_selector = current_selector
  502. end
  503. end
  504. 7584 parser.on(:current_timeout) do
  505. 3202 @current_timeout = @timeout = parser.timeout
  506. end
  507. 7584 parser.on(:timeout) do |tout|
  508. 2743 @timeout = tout
  509. end
  510. 7584 parser.on(:error) do |request, error|
  511. 58 case error
  512. when :http_1_1_required
  513. 16 current_session = @current_session
  514. 16 current_selector = @current_selector
  515. 16 parser.close
  516. 16 other_connection = current_session.find_connection(@origin, current_selector,
  517. @options.merge(ssl: { alpn_protocols: %w[http/1.1] }))
  518. 16 other_connection.merge(self)
  519. 16 request.transition(:idle)
  520. 16 other_connection.send(request)
  521. 16 next
  522. when OperationTimeoutError
  523. # request level timeouts should take precedence
  524. next unless request.active_timeouts.empty?
  525. end
  526. 44 @inflight -= 1
  527. 49 response = ErrorResponse.new(request, error)
  528. 49 request.response = response
  529. 49 request.emit(:response, response)
  530. end
  531. end
  532. 27 def transition(nextstate)
  533. 46370 handle_transition(nextstate)
  534. rescue Errno::ECONNABORTED,
  535. Errno::ECONNREFUSED,
  536. Errno::ECONNRESET,
  537. Errno::EADDRNOTAVAIL,
  538. Errno::EHOSTUNREACH,
  539. Errno::EINVAL,
  540. Errno::ENETUNREACH,
  541. Errno::EPIPE,
  542. Errno::ENOENT,
  543. SocketError,
  544. IOError => e
  545. # connect errors, exit gracefully
  546. 81 error = ConnectionError.new(e.message)
  547. 81 error.set_backtrace(e.backtrace)
  548. 81 handle_connect_error(error) if connecting?
  549. 81 @state = :closed
  550. 81 purge_after_closed
  551. 81 disconnect
  552. rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
  553. # connect errors, exit gracefully
  554. 25 handle_error(e)
  555. 25 handle_connect_error(e) if connecting?
  556. 25 @state = :closed
  557. 25 purge_after_closed
  558. 25 disconnect
  559. end
  560. 27 def handle_transition(nextstate)
  561. 40954 case nextstate
  562. when :idle
  563. 8153 @timeout = @current_timeout = @options.timeout[:connect_timeout]
  564. 8153 @connected_at = @response_received_at = nil
  565. when :open
  566. 21343 return if @state == :closed
  567. 21343 @io.connect
  568. 21238 close_sibling if @io.state == :connected
  569. 21238 return unless @io.connected?
  570. 7478 @connected_at = Utils.now
  571. 7478 send_pending
  572. 7478 @timeout = @current_timeout = parser.timeout
  573. 7478 emit(:open)
  574. when :inactive
  575. 366 return unless @state == :open
  576. # do not deactivate connection in use
  577. 365 return if @inflight.positive?
  578. when :closing
  579. 7432 return unless @state == :idle || @state == :open
  580. 7432 unless @write_buffer.empty?
  581. # preset state before handshake, as error callbacks
  582. # may take it back here.
  583. 2765 @state = nextstate
  584. # handshakes, try sending
  585. 2765 consume
  586. 2764 @write_buffer.clear
  587. 2764 return
  588. end
  589. when :closed
  590. 7724 return unless @state == :closing
  591. 7723 return unless @write_buffer.empty?
  592. 7698 purge_after_closed
  593. 7698 disconnect if @pending.empty?
  594. when :already_open
  595. 66 nextstate = :open
  596. # the first check for given io readiness must still use a timeout.
  597. # connect is the reasonable choice in such a case.
  598. 66 @timeout = @options.timeout[:connect_timeout]
  599. 66 send_pending
  600. when :active
  601. 197 return unless @state == :inactive
  602. 197 nextstate = :open
  603. # activate
  604. 197 @current_session.select_connection(self, @current_selector)
  605. end
  606. 29177 @state = nextstate
  607. end
  608. 27 def close_sibling
  609. 9889 return unless @sibling
  610. if @sibling.io_connected?
  611. reset
  612. # TODO: transition connection to closed
  613. end
  614. unless @sibling.state == :closed
  615. merge(@sibling) unless @main_sibling
  616. @sibling.force_reset(true)
  617. end
  618. @sibling = nil
  619. end
  620. 27 def purge_after_closed
  621. 8631 @io.close if @io
  622. 8631 @read_buffer.clear
  623. 8631 @timeout = nil
  624. end
  625. 27 def initialize_type(uri, options)
  626. 7018 options.transport || begin
  627. 6219 case uri.scheme
  628. when "http"
  629. 3989 "tcp"
  630. when "https"
  631. 3001 "ssl"
  632. else
  633. raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
  634. end
  635. end
  636. end
  637. # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
  638. 27 def build_altsvc_connection(alt_origin, origin, alt_params)
  639. # do not allow security downgrades on altsvc negotiation
  640. 8 return if @origin.scheme == "https" && alt_origin.scheme != "https"
  641. 8 altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
  642. # altsvc already exists, somehow it wasn't advertised, probably noop
  643. 8 return unless altsvc
  644. 8 alt_options = @options.merge(ssl: @options.ssl.merge(hostname: URI(origin).host))
  645. 8 connection = @current_session.find_connection(alt_origin, @current_selector, alt_options)
  646. # advertised altsvc is the same origin being used, ignore
  647. 8 return if connection == self
  648. 8 connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
  649. 8 log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
  650. 8 connection.merge(self)
  651. 8 terminate
  652. rescue UnsupportedSchemeError
  653. altsvc["noop"] = true
  654. nil
  655. end
  656. 27 def build_socket(addrs = nil)
  657. 6194 case @type
  658. when "tcp"
  659. 4061 TCP.new(peer, addrs, @options)
  660. when "ssl"
  661. 2906 SSL.new(peer, addrs, @options) do |sock|
  662. 2884 sock.ssl_session = @ssl_session
  663. 2884 sock.session_new_cb do |sess|
  664. 5020 @ssl_session = sess
  665. 5020 sock.ssl_session = sess
  666. end
  667. end
  668. when "unix"
  669. 28 path = Array(addrs).first
  670. 28 path = String(path) if path
  671. 28 UNIX.new(peer, path, @options)
  672. else
  673. raise Error, "unsupported transport (#{@type})"
  674. end
  675. end
  676. 27 def on_error(error, request = nil)
  677. 612 if error.is_a?(OperationTimeoutError)
  678. # inactive connections do not contribute to the select loop, therefore
  679. # they should not fail due to such errors.
  680. 32 return if @state == :inactive
  681. 32 if @timeout
  682. 28 @timeout -= error.timeout
  683. 32 return unless @timeout <= 0
  684. end
  685. 32 error = error.to_connection_error if connecting?
  686. end
  687. 612 handle_error(error, request)
  688. 612 reset
  689. end
  690. 27 def handle_error(error, request = nil)
  691. 905 parser.handle_error(error, request) if @parser && parser.respond_to?(:handle_error)
  692. 2004 while (req = @pending.shift)
  693. 420 next if request && req == request
  694. 420 response = ErrorResponse.new(req, error)
  695. 420 req.response = response
  696. 420 req.emit(:response, response)
  697. end
  698. 905 return unless request
  699. 359 @inflight -= 1
  700. 410 response = ErrorResponse.new(request, error)
  701. 410 request.response = response
  702. 410 request.emit(:response, response)
  703. end
  704. 27 def set_request_timeouts(request)
  705. 19114 set_request_write_timeout(request)
  706. 19114 set_request_read_timeout(request)
  707. 19114 set_request_request_timeout(request)
  708. end
  709. 27 def set_request_read_timeout(request)
  710. 19114 read_timeout = request.read_timeout
  711. 19114 return if read_timeout.nil? || read_timeout.infinite?
  712. 18820 set_request_timeout(:read_timeout, request, read_timeout, :done, :response) do
  713. 24 read_timeout_callback(request, read_timeout)
  714. end
  715. end
  716. 27 def set_request_write_timeout(request)
  717. 19114 write_timeout = request.write_timeout
  718. 19114 return if write_timeout.nil? || write_timeout.infinite?
  719. 19114 set_request_timeout(:write_timeout, request, write_timeout, :headers, %i[done response]) do
  720. 16 write_timeout_callback(request, write_timeout)
  721. end
  722. end
  723. 27 def set_request_request_timeout(request)
  724. 18830 request_timeout = request.request_timeout
  725. 18830 return if request_timeout.nil? || request_timeout.infinite?
  726. 541 set_request_timeout(:request_timeout, request, request_timeout, :headers, :complete) do
  727. 402 read_timeout_callback(request, request_timeout, RequestTimeoutError)
  728. end
  729. end
  730. 27 def write_timeout_callback(request, write_timeout)
  731. 16 return if request.state == :done
  732. 16 @write_buffer.clear
  733. 16 error = WriteTimeoutError.new(request, nil, write_timeout)
  734. 16 on_error(error, request)
  735. end
  736. 27 def read_timeout_callback(request, read_timeout, error_type = ReadTimeoutError)
  737. 426 response = request.response
  738. 426 return if response && response.finished?
  739. 394 @write_buffer.clear
  740. 394 error = error_type.new(request, request.response, read_timeout)
  741. 394 on_error(error, request)
  742. end
  743. 27 def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
  744. 38555 request.set_timeout_callback(start_event) do
  745. 38355 timer = @current_selector.after(timeout, callback)
  746. 38355 request.active_timeouts << label
  747. 38355 Array(finish_events).each do |event|
  748. # clean up request timeouts if the connection errors out
  749. 57461 request.set_timeout_callback(event) do
  750. 57009 timer.cancel
  751. 57009 request.active_timeouts.delete(label)
  752. end
  753. end
  754. end
  755. end
  756. 27 def parser_type(protocol)
  757. 6741 case protocol
  758. 3210 when "h2" then HTTP2
  759. 4399 when "http/1.1" then HTTP1
  760. else
  761. raise Error, "unsupported protocol (##{protocol})"
  762. end
  763. end
  764. end
  765. end

lib/httpx/connection/http1.rb

89.14% lines covered

221 relevant lines. 197 lines covered and 24 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "httpx/parser/http1"
  3. 27 module HTTPX
  4. 27 class Connection::HTTP1
  5. 27 include Callbacks
  6. 27 include Loggable
  7. 27 MAX_REQUESTS = 200
  8. 27 CRLF = "\r\n"
  9. 27 attr_reader :pending, :requests
  10. 27 attr_accessor :max_concurrent_requests
  11. 27 def initialize(buffer, options)
  12. 4399 @options = options
  13. 4399 @max_concurrent_requests = @options.max_concurrent_requests || MAX_REQUESTS
  14. 4399 @max_requests = @options.max_requests
  15. 4399 @parser = Parser::HTTP1.new(self)
  16. 4399 @buffer = buffer
  17. 4399 @version = [1, 1]
  18. 4399 @pending = []
  19. 4399 @requests = []
  20. 4399 @handshake_completed = false
  21. end
  22. 27 def timeout
  23. 4282 @options.timeout[:operation_timeout]
  24. end
  25. 27 def interests
  26. # this means we're processing incoming response already
  27. 30650 return :r if @request
  28. 25564 return if @requests.empty?
  29. 25542 request = @requests.first
  30. 25542 return unless request
  31. 25542 return :w if request.interests == :w || !@buffer.empty?
  32. 22612 :r
  33. end
  34. 27 def reset
  35. 4166 @max_requests = @options.max_requests || MAX_REQUESTS
  36. 4166 @parser.reset!
  37. 4166 @handshake_completed = false
  38. 4166 @pending.concat(@requests) unless @requests.empty?
  39. end
  40. 27 def close
  41. 81 reset
  42. 81 emit(:close, true)
  43. end
  44. 27 def exhausted?
  45. 528 !@max_requests.positive?
  46. end
  47. 27 def empty?
  48. # this means that for every request there's an available
  49. # partial response, so there are no in-flight requests waiting.
  50. 4085 @requests.empty? || (
  51. # checking all responses can be time-consuming. Alas, as in HTTP/1, responses
  52. # do not come out of order, we can get away with checking first and last.
  53. 185 !@requests.first.response.nil? &&
  54. (@requests.size == 1 || !@requests.last.response.nil?)
  55. )
  56. end
  57. 27 def <<(data)
  58. 7149 @parser << data
  59. end
  60. 27 def send(request)
  61. 15496 unless @max_requests.positive?
  62. @pending << request
  63. return
  64. end
  65. 15496 return if @requests.include?(request)
  66. 15496 @requests << request
  67. 15496 @pipelining = true if @requests.size > 1
  68. end
  69. 27 def consume
  70. 16164 requests_limit = [@max_requests, @requests.size].min
  71. 16164 concurrent_requests_limit = [@max_concurrent_requests, requests_limit].min
  72. 16164 @requests.each_with_index do |request, idx|
  73. 18733 break if idx >= concurrent_requests_limit
  74. 16140 next unless request.can_buffer?
  75. 6130 handle(request)
  76. end
  77. end
  78. # HTTP Parser callbacks
  79. #
  80. # must be public methods, or else they won't be reachable
  81. 27 def on_start
  82. 4790 log(level: 2) { "parsing begins" }
  83. end
  84. 27 def on_headers(h)
  85. 4766 @request = @requests.first
  86. 4766 return if @request.response
  87. 4790 log(level: 2) { "headers received" }
  88. 4766 headers = @request.options.headers_class.new(h)
  89. 4766 response = @request.options.response_class.new(@request,
  90. @parser.status_code,
  91. @parser.http_version.join("."),
  92. headers)
  93. 4790 log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
  94. 4982 log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
  95. 4766 @request.response = response
  96. 4758 on_complete if response.finished?
  97. end
  98. 27 def on_trailers(h)
  99. 8 return unless @request
  100. 8 response = @request.response
  101. 8 log(level: 2) { "trailer headers received" }
  102. 8 log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{v.join(", ")}" }.join("\n") }
  103. 8 response.merge_headers(h)
  104. end
  105. 27 def on_data(chunk)
  106. 5545 request = @request
  107. 5545 return unless request
  108. 5569 log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
  109. 5569 log(level: 2, color: :green) { "-> #{chunk.inspect}" }
  110. 5545 response = request.response
  111. 5545 response << chunk
  112. rescue StandardError => e
  113. 14 error_response = ErrorResponse.new(request, e)
  114. 14 request.response = error_response
  115. 14 dispatch
  116. end
  117. 27 def on_complete
  118. 4728 request = @request
  119. 4728 return unless request
  120. 4752 log(level: 2) { "parsing complete" }
  121. 4728 dispatch
  122. end
  123. 27 def dispatch
  124. 4742 request = @request
  125. 4742 if request.expects?
  126. 72 @parser.reset!
  127. 63 return handle(request)
  128. end
  129. 4670 @request = nil
  130. 4670 @requests.shift
  131. 4670 response = request.response
  132. 4670 emit(:response, request, response)
  133. 4613 if @parser.upgrade?
  134. 32 response << @parser.upgrade_data
  135. 32 throw(:called)
  136. end
  137. 4581 @parser.reset!
  138. 4113 @max_requests -= 1
  139. 4581 if response.is_a?(ErrorResponse)
  140. 14 disable
  141. else
  142. 4567 manage_connection(request, response)
  143. end
  144. 528 if exhausted?
  145. @pending.concat(@requests)
  146. @requests.clear
  147. emit(:exhausted)
  148. else
  149. 528 send(@pending.shift) unless @pending.empty?
  150. end
  151. end
  152. 27 def handle_error(ex, request = nil)
  153. 209 if (ex.is_a?(EOFError) || ex.is_a?(TimeoutError)) && @request && @request.response &&
  154. !@request.response.headers.key?("content-length") &&
  155. !@request.response.headers.key?("transfer-encoding")
  156. # if the response does not contain a content-length header, the server closing the
  157. # connnection is the indicator of response consumed.
  158. # https://greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4
  159. 32 catch(:called) { on_complete }
  160. 14 return
  161. end
  162. 193 if @pipelining
  163. catch(:called) { disable }
  164. else
  165. 193 @requests.each do |req|
  166. 185 next if request && request == req
  167. emit(:error, req, ex)
  168. end
  169. 193 @pending.each do |req|
  170. next if request && request == req
  171. emit(:error, req, ex)
  172. end
  173. end
  174. end
  175. 27 def ping
  176. reset
  177. emit(:reset)
  178. emit(:exhausted)
  179. end
  180. 27 private
  181. 27 def manage_connection(request, response)
  182. 4567 connection = response.headers["connection"]
  183. 4100 case connection
  184. when /keep-alive/i
  185. 528 if @handshake_completed
  186. if @max_requests.zero?
  187. @pending.concat(@requests)
  188. @requests.clear
  189. emit(:exhausted)
  190. end
  191. return
  192. end
  193. 528 keep_alive = response.headers["keep-alive"]
  194. 528 return unless keep_alive
  195. 108 parameters = Hash[keep_alive.split(/ *, */).map do |pair|
  196. 108 pair.split(/ *= */, 2)
  197. end]
  198. 108 @max_requests = parameters["max"].to_i - 1 if parameters.key?("max")
  199. 108 if parameters.key?("timeout")
  200. keep_alive_timeout = parameters["timeout"].to_i
  201. emit(:timeout, keep_alive_timeout)
  202. end
  203. 108 @handshake_completed = true
  204. when /close/i
  205. 4039 disable
  206. when nil
  207. # In HTTP/1.1, it's keep alive by default
  208. return if response.version == "1.1" && request.headers["connection"] != "close"
  209. disable
  210. end
  211. end
  212. 27 def disable
  213. 4053 disable_pipelining
  214. 4053 reset
  215. 4053 emit(:reset)
  216. 4045 throw(:called)
  217. end
  218. 27 def disable_pipelining
  219. 4053 return if @requests.empty?
  220. # do not disable pipelining if already set to 1 request at a time
  221. 187 return if @max_concurrent_requests == 1
  222. 29 @requests.each do |r|
  223. 29 r.transition(:idle)
  224. # when we disable pipelining, we still want to try keep-alive.
  225. # only when keep-alive with one request fails, do we fallback to
  226. # connection: close.
  227. 29 r.headers["connection"] = "close" if @max_concurrent_requests == 1
  228. end
  229. # server doesn't handle pipelining, and probably
  230. # doesn't support keep-alive. Fallback to send only
  231. # 1 keep alive request.
  232. 29 @max_concurrent_requests = 1
  233. 29 @pipelining = false
  234. end
  235. 27 def set_protocol_headers(request)
  236. 4908 if !request.headers.key?("content-length") &&
  237. request.body.bytesize == Float::INFINITY
  238. 32 request.body.chunk!
  239. end
  240. 4908 extra_headers = {}
  241. 4908 unless request.headers.key?("connection")
  242. 4884 connection_value = if request.persistent?
  243. # when in a persistent connection, the request can't be at
  244. # the edge of a renegotiation
  245. 97 if @requests.index(request) + 1 < @max_requests
  246. 97 "keep-alive"
  247. else
  248. "close"
  249. end
  250. else
  251. # when it's not a persistent connection, it sets "Connection: close" always
  252. # on the last request of the possible batch (either allowed max requests,
  253. # or if smaller, the size of the batch itself)
  254. 4787 requests_limit = [@max_requests, @requests.size].min
  255. 4787 if request == @requests[requests_limit - 1]
  256. 4222 "close"
  257. else
  258. 565 "keep-alive"
  259. end
  260. end
  261. 4380 extra_headers["connection"] = connection_value
  262. end
  263. 4908 extra_headers["host"] = request.authority unless request.headers.key?("host")
  264. 4908 extra_headers
  265. end
  266. 27 def handle(request)
  267. 6202 catch(:buffer_full) do
  268. 6202 request.transition(:headers)
  269. 6194 join_headers(request) if request.state == :headers
  270. 6194 request.transition(:body)
  271. 6194 join_body(request) if request.state == :body
  272. 5102 request.transition(:trailers)
  273. # HTTP/1.1 trailers should only work for chunked encoding
  274. 5102 join_trailers(request) if request.body.chunked? && request.state == :trailers
  275. 5102 request.transition(:done)
  276. end
  277. end
  278. 27 def join_headline(request)
  279. 4331 "#{request.verb} #{request.path} HTTP/#{@version.join(".")}"
  280. end
  281. 27 def join_headers(request)
  282. 4908 headline = join_headline(request)
  283. 4908 @buffer << headline << CRLF
  284. 4932 log(color: :yellow) { "<- HEADLINE: #{headline.chomp.inspect}" }
  285. 4908 extra_headers = set_protocol_headers(request)
  286. 4908 join_headers2(request.headers.each(extra_headers))
  287. 4932 log { "<- " }
  288. 4908 @buffer << CRLF
  289. end
  290. 27 def join_body(request)
  291. 5984 return if request.body.empty?
  292. 6565 while (chunk = request.drain_body)
  293. 3280 log(color: :green) { "<- DATA: #{chunk.bytesize} bytes..." }
  294. 3280 log(level: 2, color: :green) { "<- #{chunk.inspect}" }
  295. 3280 @buffer << chunk
  296. 3280 throw(:buffer_full, request) if @buffer.full?
  297. end
  298. 1556 return unless (error = request.drain_error)
  299. raise error
  300. end
  301. 27 def join_trailers(request)
  302. 96 return unless request.trailers? && request.callbacks_for?(:trailers)
  303. 32 join_headers2(request.trailers)
  304. 32 log { "<- " }
  305. 32 @buffer << CRLF
  306. end
  307. 27 def join_headers2(headers)
  308. 4940 headers.each do |field, value|
  309. 30322 buffer = "#{capitalized(field)}: #{value}#{CRLF}"
  310. 30442 log(color: :yellow) { "<- HEADER: #{buffer.chomp}" }
  311. 30322 @buffer << buffer
  312. end
  313. end
  314. 27 UPCASED = {
  315. "www-authenticate" => "WWW-Authenticate",
  316. "http2-settings" => "HTTP2-Settings",
  317. "content-md5" => "Content-MD5",
  318. }.freeze
  319. 27 def capitalized(field)
  320. 30322 UPCASED[field] || field.split("-").map(&:capitalize).join("-")
  321. end
  322. end
  323. end

lib/httpx/connection/http2.rb

94.98% lines covered

259 relevant lines. 246 lines covered and 13 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "securerandom"
  3. 27 require "http/2"
  4. 27 module HTTPX
  5. 27 class Connection::HTTP2
  6. 27 include Callbacks
  7. 27 include Loggable
  8. 27 MAX_CONCURRENT_REQUESTS = ::HTTP2::DEFAULT_MAX_CONCURRENT_STREAMS
  9. 27 class Error < Error
  10. 27 def initialize(id, code)
  11. 36 super("stream #{id} closed with error: #{code}")
  12. end
  13. end
  14. 27 class PingError < Error
  15. 27 def initialize
  16. super(0, :ping_error)
  17. end
  18. end
  19. 27 class GoawayError < Error
  20. 27 def initialize
  21. 17 super(0, :no_error)
  22. end
  23. end
  24. 27 attr_reader :streams, :pending
  25. 27 def initialize(buffer, options)
  26. 3242 @options = options
  27. 3242 @settings = @options.http2_settings
  28. 3242 @pending = []
  29. 3242 @streams = {}
  30. 3242 @drains = {}
  31. 3242 @pings = []
  32. 3242 @buffer = buffer
  33. 3242 @handshake_completed = false
  34. 3242 @wait_for_handshake = @settings.key?(:wait_for_handshake) ? @settings.delete(:wait_for_handshake) : true
  35. 3242 @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
  36. 3242 @max_requests = @options.max_requests
  37. 3242 init_connection
  38. end
  39. 27 def timeout
  40. 6398 return @options.timeout[:operation_timeout] if @handshake_completed
  41. 3196 @options.timeout[:settings_timeout]
  42. end
  43. 27 def interests
  44. # waiting for WINDOW_UPDATE frames
  45. 11534804 return :r if @buffer.full?
  46. 11534804 if @connection.state == :closed
  47. 2909 return unless @handshake_completed
  48. 2521 return :w
  49. end
  50. 11531895 unless @connection.state == :connected && @handshake_completed
  51. 12331 return @buffer.empty? ? :r : :rw
  52. end
  53. 11518087 return :w if !@pending.empty? && can_buffer_more_requests?
  54. 11518087 return :w unless @drains.empty?
  55. 11517109 if @buffer.empty?
  56. 11517109 return if @streams.empty? && @pings.empty?
  57. 39354 return :r
  58. end
  59. :rw
  60. end
  61. 27 def close
  62. 2751 unless @connection.state == :closed
  63. 2743 @connection.goaway
  64. 2743 emit(:timeout, @options.timeout[:close_handshake_timeout])
  65. end
  66. 2751 emit(:close, true)
  67. end
  68. 27 def empty?
  69. 2744 @connection.state == :closed || @streams.empty?
  70. end
  71. 27 def exhausted?
  72. 3229 !@max_requests.positive?
  73. end
  74. 27 def <<(data)
  75. 32782 @connection << data
  76. end
  77. 27 def send(request, head = false)
  78. 7068 unless can_buffer_more_requests?
  79. 3433 head ? @pending.unshift(request) : @pending << request
  80. 3433 return false
  81. end
  82. 3635 unless (stream = @streams[request])
  83. 3635 stream = @connection.new_stream
  84. 3635 handle_stream(stream, request)
  85. 3226 @streams[request] = stream
  86. 3226 @max_requests -= 1
  87. end
  88. 3635 handle(request, stream)
  89. 3619 true
  90. rescue ::HTTP2::Error::StreamLimitExceeded
  91. @pending.unshift(request)
  92. false
  93. end
  94. 27 def consume
  95. 25252 @streams.each do |request, stream|
  96. 9955 next unless request.can_buffer?
  97. 1116 handle(request, stream)
  98. end
  99. end
  100. 27 def handle_error(ex, request = nil)
  101. 276 if ex.is_a?(OperationTimeoutError) && !@handshake_completed && @connection.state != :closed
  102. 8 @connection.goaway(:settings_timeout, "closing due to settings timeout")
  103. 8 emit(:close_handshake)
  104. 8 settings_ex = SettingsTimeoutError.new(ex.timeout, ex.message)
  105. 8 settings_ex.set_backtrace(ex.backtrace)
  106. 8 ex = settings_ex
  107. end
  108. 276 @streams.each_key do |req|
  109. 226 next if request && request == req
  110. 16 emit(:error, req, ex)
  111. end
  112. 519 while (req = @pending.shift)
  113. 33 next if request && request == req
  114. 33 emit(:error, req, ex)
  115. end
  116. end
  117. 27 def ping
  118. 20 ping = SecureRandom.gen_random(8)
  119. 20 @connection.ping(ping.dup)
  120. ensure
  121. 20 @pings << ping
  122. end
  123. 27 private
  124. 27 def can_buffer_more_requests?
  125. 7621 (@handshake_completed || !@wait_for_handshake) &&
  126. @streams.size < @max_concurrent_requests &&
  127. @streams.size < @max_requests
  128. end
  129. 27 def send_pending
  130. 8541 while (request = @pending.shift)
  131. 3307 break unless send(request, true)
  132. end
  133. end
  134. 27 def handle(request, stream)
  135. 4855 catch(:buffer_full) do
  136. 4855 request.transition(:headers)
  137. 4847 join_headers(stream, request) if request.state == :headers
  138. 4847 request.transition(:body)
  139. 4847 join_body(stream, request) if request.state == :body
  140. 3805 request.transition(:trailers)
  141. 3805 join_trailers(stream, request) if request.state == :trailers && !request.body.empty?
  142. 3805 request.transition(:done)
  143. end
  144. end
  145. 27 def init_connection
  146. 3242 @connection = ::HTTP2::Client.new(@settings)
  147. 3242 @connection.on(:frame, &method(:on_frame))
  148. 3242 @connection.on(:frame_sent, &method(:on_frame_sent))
  149. 3242 @connection.on(:frame_received, &method(:on_frame_received))
  150. 3242 @connection.on(:origin, &method(:on_origin))
  151. 3242 @connection.on(:promise, &method(:on_promise))
  152. 3242 @connection.on(:altsvc) { |frame| on_altsvc(frame[:origin], frame) }
  153. 3242 @connection.on(:settings_ack, &method(:on_settings))
  154. 3242 @connection.on(:ack, &method(:on_pong))
  155. 3242 @connection.on(:goaway, &method(:on_close))
  156. #
  157. # Some servers initiate HTTP/2 negotiation right away, some don't.
  158. # As such, we have to check the socket buffer. If there is something
  159. # to read, the server initiated the negotiation. If not, we have to
  160. # initiate it.
  161. #
  162. 3242 @connection.send_connection_preface
  163. end
  164. 27 alias_method :reset, :init_connection
  165. 27 public :reset
  166. 27 def handle_stream(stream, request)
  167. 3651 request.on(:refuse, &method(:on_stream_refuse).curry(3)[stream, request])
  168. 3651 stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
  169. 3651 stream.on(:half_close) do
  170. 3629 log(level: 2) { "#{stream.id}: waiting for response..." }
  171. end
  172. 3651 stream.on(:altsvc, &method(:on_altsvc).curry(2)[request.origin])
  173. 3651 stream.on(:headers, &method(:on_stream_headers).curry(3)[stream, request])
  174. 3651 stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
  175. end
  176. 27 def set_protocol_headers(request)
  177. 408 {
  178. 3218 ":scheme" => request.scheme,
  179. ":method" => request.verb,
  180. ":path" => request.path,
  181. ":authority" => request.authority,
  182. }
  183. end
  184. 27 def join_headers(stream, request)
  185. 3627 extra_headers = set_protocol_headers(request)
  186. 3627 if request.headers.key?("host")
  187. 8 log { "forbidden \"host\" header found (#{request.headers["host"]}), will use it as authority..." }
  188. 7 extra_headers[":authority"] = request.headers["host"]
  189. end
  190. 3627 log(level: 1, color: :yellow) do
  191. 128 request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
  192. end
  193. 3627 stream.headers(request.headers.each(extra_headers), end_stream: request.body.empty?)
  194. end
  195. 27 def join_trailers(stream, request)
  196. 1517 unless request.trailers?
  197. 1509 stream.data("", end_stream: true) if request.callbacks_for?(:trailers)
  198. 1346 return
  199. end
  200. 8 log(level: 1, color: :yellow) do
  201. 15 request.trailers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
  202. end
  203. 8 stream.headers(request.trailers.each, end_stream: true)
  204. end
  205. 27 def join_body(stream, request)
  206. 4661 return if request.body.empty?
  207. 2559 chunk = @drains.delete(request) || request.drain_body
  208. 2753 while chunk
  209. 2787 next_chunk = request.drain_body
  210. 2787 send_chunk(request, stream, chunk, next_chunk)
  211. 2739 if next_chunk && (@buffer.full? || request.body.unbounded_body?)
  212. 874 @drains[request] = next_chunk
  213. 994 throw(:buffer_full)
  214. end
  215. 1745 chunk = next_chunk
  216. end
  217. 1517 return unless (error = request.drain_error)
  218. 12 on_stream_refuse(stream, request, error)
  219. end
  220. 27 def send_chunk(request, stream, chunk, next_chunk)
  221. 2808 log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
  222. 2808 log(level: 2, color: :green) { "#{stream.id}: -> #{chunk.inspect}" }
  223. 2787 stream.data(chunk, end_stream: end_stream?(request, next_chunk))
  224. end
  225. 27 def end_stream?(request, next_chunk)
  226. 2739 !(next_chunk || request.trailers? || request.callbacks_for?(:trailers))
  227. end
  228. ######
  229. # HTTP/2 Callbacks
  230. ######
  231. 27 def on_stream_headers(stream, request, h)
  232. 3596 response = request.response
  233. 3596 if response.is_a?(Response) && response.version == "2.0"
  234. 114 on_stream_trailers(stream, response, h)
  235. 114 return
  236. end
  237. 3482 log(color: :yellow) do
  238. 128 h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
  239. end
  240. 3482 _, status = h.shift
  241. 3482 headers = request.options.headers_class.new(h)
  242. 3482 response = request.options.response_class.new(request, status, "2.0", headers)
  243. 3482 request.response = response
  244. 3083 @streams[request] = stream
  245. 3474 handle(request, stream) if request.expects?
  246. end
  247. 27 def on_stream_trailers(stream, response, h)
  248. 114 log(color: :yellow) do
  249. h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
  250. end
  251. 114 response.merge_headers(h)
  252. end
  253. 27 def on_stream_data(stream, request, data)
  254. 6330 log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
  255. 6330 log(level: 2, color: :green) { "#{stream.id}: <- #{data.inspect}" }
  256. 6311 request.response << data
  257. end
  258. 27 def on_stream_refuse(stream, request, error)
  259. 12 on_stream_close(stream, request, error)
  260. 12 stream.close
  261. end
  262. 27 def on_stream_close(stream, request, error)
  263. 3388 return if error == :stream_closed && !@streams.key?(request)
  264. 3390 log(level: 2) { "#{stream.id}: closing stream" }
  265. 3376 @drains.delete(request)
  266. 3376 @streams.delete(request)
  267. 3376 if error
  268. 12 case error
  269. when :http_1_1_required
  270. emit(:error, request, error)
  271. else
  272. 12 ex = Error.new(stream.id, error)
  273. 12 ex.set_backtrace(caller)
  274. 12 response = ErrorResponse.new(request, ex)
  275. 12 request.response = response
  276. 12 emit(:response, request, response)
  277. end
  278. else
  279. 3364 response = request.response
  280. 3364 if response && response.is_a?(Response) && response.status == 421
  281. 8 emit(:error, request, :http_1_1_required)
  282. else
  283. 3356 emit(:response, request, response)
  284. end
  285. end
  286. 3368 send(@pending.shift) unless @pending.empty?
  287. 3368 return unless @streams.empty? && exhausted?
  288. 8 if @pending.empty?
  289. close
  290. else
  291. 8 emit(:exhausted)
  292. end
  293. end
  294. 27 def on_frame(bytes)
  295. 20037 @buffer << bytes
  296. end
  297. 27 def on_settings(*)
  298. 3202 @handshake_completed = true
  299. 3202 emit(:current_timeout)
  300. 3202 @max_concurrent_requests = [@max_concurrent_requests, @connection.remote_settings[:settings_max_concurrent_streams]].min
  301. 3202 send_pending
  302. end
  303. 27 def on_close(_last_frame, error, _payload)
  304. 32 is_connection_closed = @connection.state == :closed
  305. 32 if error
  306. 32 @buffer.clear if is_connection_closed
  307. 28 case error
  308. when :http_1_1_required
  309. 21 while (request = @pending.shift)
  310. 8 emit(:error, request, error)
  311. end
  312. when :no_error
  313. 17 ex = GoawayError.new
  314. 17 @pending.unshift(*@streams.keys)
  315. 17 @drains.clear
  316. 17 @streams.clear
  317. else
  318. 7 ex = Error.new(0, error)
  319. end
  320. 32 if ex
  321. 24 ex.set_backtrace(caller)
  322. 24 handle_error(ex)
  323. end
  324. end
  325. 32 return unless is_connection_closed && @streams.empty?
  326. 32 emit(:close, is_connection_closed)
  327. end
  328. 27 def on_frame_sent(frame)
  329. 16868 log(level: 2) { "#{frame[:stream]}: frame was sent!" }
  330. 16784 log(level: 2, color: :blue) do
  331. 96 payload = frame
  332. 96 payload = payload.merge(payload: frame[:payload].bytesize) if frame[:type] == :data
  333. 84 "#{frame[:stream]}: #{payload}"
  334. end
  335. end
  336. 27 def on_frame_received(frame)
  337. 17710 log(level: 2) { "#{frame[:stream]}: frame was received!" }
  338. 17649 log(level: 2, color: :magenta) do
  339. 69 payload = frame
  340. 69 payload = payload.merge(payload: frame[:payload].bytesize) if frame[:type] == :data
  341. 61 "#{frame[:stream]}: #{payload}"
  342. end
  343. end
  344. 27 def on_altsvc(origin, frame)
  345. log(level: 2) { "#{frame[:stream]}: altsvc frame was received" }
  346. log(level: 2) { "#{frame[:stream]}: #{frame.inspect}" }
  347. alt_origin = URI.parse("#{frame[:proto]}://#{frame[:host]}:#{frame[:port]}")
  348. params = { "ma" => frame[:max_age] }
  349. emit(:altsvc, origin, alt_origin, origin, params)
  350. end
  351. 27 def on_promise(stream)
  352. 24 emit(:promise, @streams.key(stream.parent), stream)
  353. end
  354. 27 def on_origin(origin)
  355. emit(:origin, origin)
  356. end
  357. 27 def on_pong(ping)
  358. 8 raise PingError unless @pings.delete(ping.to_s)
  359. 8 emit(:pong)
  360. end
  361. end
  362. end

lib/httpx/domain_name.rb

95.56% lines covered

45 relevant lines. 43 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. #
  3. # domain_name.rb - Domain Name manipulation library for Ruby
  4. #
  5. # Copyright (C) 2011-2017 Akinori MUSHA, All rights reserved.
  6. #
  7. # Redistribution and use in source and binary forms, with or without
  8. # modification, are permitted provided that the following conditions
  9. # are met:
  10. # 1. Redistributions of source code must retain the above copyright
  11. # notice, this list of conditions and the following disclaimer.
  12. # 2. Redistributions in binary form must reproduce the above copyright
  13. # notice, this list of conditions and the following disclaimer in the
  14. # documentation and/or other materials provided with the distribution.
  15. #
  16. # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17. # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19. # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20. # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24. # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25. # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26. # SUCH DAMAGE.
  27. 27 require "ipaddr"
  28. 27 module HTTPX
  29. # Represents a domain name ready for extracting its registered domain
  30. # and TLD.
  31. 27 class DomainName
  32. 27 include Comparable
  33. # The full host name normalized, ASCII-ized and downcased using the
  34. # Unicode NFC rules and the Punycode algorithm. If initialized with
  35. # an IP address, the string representation of the IP address
  36. # suitable for opening a connection to.
  37. 27 attr_reader :hostname
  38. # The Unicode representation of the #hostname property.
  39. #
  40. # :attr_reader: hostname_idn
  41. # The least "universally original" domain part of this domain name.
  42. # For example, "example.co.uk" for "www.sub.example.co.uk". This
  43. # may be nil if the hostname does not have one, like when it is an
  44. # IP address, an effective TLD or higher itself, or of a
  45. # non-canonical domain.
  46. 27 attr_reader :domain
  47. 27 class << self
  48. 27 def new(domain)
  49. 856 return domain if domain.is_a?(self)
  50. 792 super(domain)
  51. end
  52. # Normalizes a _domain_ using the Punycode algorithm as necessary.
  53. # The result will be a downcased, ASCII-only string.
  54. 27 def normalize(domain)
  55. 760 unless domain.ascii_only?
  56. domain = domain.chomp(".").unicode_normalize(:nfc)
  57. domain = Punycode.encode_hostname(domain)
  58. end
  59. 760 domain.downcase
  60. end
  61. end
  62. # Parses _hostname_ into a DomainName object. An IP address is also
  63. # accepted. An IPv6 address may be enclosed in square brackets.
  64. 27 def initialize(hostname)
  65. 792 hostname = String(hostname)
  66. 792 raise ArgumentError, "domain name must not start with a dot: #{hostname}" if hostname.start_with?(".")
  67. 98 begin
  68. 792 @ipaddr = IPAddr.new(hostname)
  69. 32 @hostname = @ipaddr.to_s
  70. 32 return
  71. rescue IPAddr::Error
  72. 760 nil
  73. end
  74. 760 @hostname = DomainName.normalize(hostname)
  75. 760 tld = if (last_dot = @hostname.rindex("."))
  76. 184 @hostname[(last_dot + 1)..-1]
  77. else
  78. 576 @hostname
  79. end
  80. # unknown/local TLD
  81. 760 @domain = if last_dot
  82. # fallback - accept cookies down to second level
  83. # cf. http://www.dkim-reputation.org/regdom-libs/
  84. 184 if (penultimate_dot = @hostname.rindex(".", last_dot - 1))
  85. 48 @hostname[(penultimate_dot + 1)..-1]
  86. else
  87. 136 @hostname
  88. end
  89. else
  90. # no domain part - must be a local hostname
  91. 576 tld
  92. end
  93. end
  94. # Checks if the server represented by this domain is qualified to
  95. # send and receive cookies with a domain attribute value of
  96. # _domain_. A true value given as the second argument represents
  97. # cookies without a domain attribute value, in which case only
  98. # hostname equality is checked.
  99. 27 def cookie_domain?(domain, host_only = false)
  100. # RFC 6265 #5.3
  101. # When the user agent "receives a cookie":
  102. 32 return self == @domain if host_only
  103. 32 domain = DomainName.new(domain)
  104. # RFC 6265 #5.1.3
  105. # Do not perform subdomain matching against IP addresses.
  106. 32 @hostname == domain.hostname if @ipaddr
  107. # RFC 6265 #4.1.1
  108. # Domain-value must be a subdomain.
  109. 32 @domain && self <= domain && domain <= @domain
  110. end
  111. 27 def <=>(other)
  112. 48 other = DomainName.new(other)
  113. 48 othername = other.hostname
  114. 48 if othername == @hostname
  115. 16 0
  116. 31 elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == "."
  117. # The other is higher
  118. 16 -1
  119. else
  120. # The other is lower
  121. 16 1
  122. end
  123. end
  124. end
  125. end

lib/httpx/errors.rb

97.62% lines covered

42 relevant lines. 41 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # the default exception class for exceptions raised by HTTPX.
  4. 27 class Error < StandardError; end
  5. 27 class UnsupportedSchemeError < Error; end
  6. 27 class ConnectionError < Error; end
  7. # Error raised when there was a timeout. Its subclasses allow for finer-grained
  8. # control of which timeout happened.
  9. 27 class TimeoutError < Error
  10. # The timeout value which caused this error to be raised.
  11. 27 attr_reader :timeout
  12. # initializes the timeout exception with the +timeout+ causing the error, and the
  13. # error +message+ for it.
  14. 27 def initialize(timeout, message)
  15. 521 @timeout = timeout
  16. 521 super(message)
  17. end
  18. # clones this error into a HTTPX::ConnectionTimeoutError.
  19. 27 def to_connection_error
  20. 24 ex = ConnectTimeoutError.new(@timeout, message)
  21. 24 ex.set_backtrace(backtrace)
  22. 24 ex
  23. end
  24. end
  25. # Raise when it can't acquire a connection from the pool.
  26. 27 class PoolTimeoutError < TimeoutError; end
  27. # Error raised when there was a timeout establishing the connection to a server.
  28. # This may be raised due to timeouts during TCP and TLS (when applicable) connection
  29. # establishment.
  30. 27 class ConnectTimeoutError < TimeoutError; end
  31. # Error raised when there was a timeout while sending a request, or receiving a response
  32. # from the server.
  33. 27 class RequestTimeoutError < TimeoutError
  34. # The HTTPX::Request request object this exception refers to.
  35. 27 attr_reader :request
  36. # initializes the exception with the +request+ and +response+ it refers to, and the
  37. # +timeout+ causing the error, and the
  38. 27 def initialize(request, response, timeout)
  39. 410 @request = request
  40. 410 @response = response
  41. 410 super(timeout, "Timed out after #{timeout} seconds")
  42. end
  43. 27 def marshal_dump
  44. [message]
  45. end
  46. end
  47. # Error raised when there was a timeout while receiving a response from the server.
  48. 27 class ReadTimeoutError < RequestTimeoutError; end
  49. # Error raised when there was a timeout while sending a request from the server.
  50. 27 class WriteTimeoutError < RequestTimeoutError; end
  51. # Error raised when there was a timeout while waiting for the HTTP/2 settings frame from the server.
  52. 27 class SettingsTimeoutError < TimeoutError; end
  53. # Error raised when there was a timeout while resolving a domain to an IP.
  54. 27 class ResolveTimeoutError < TimeoutError; end
  55. # Error raise when there was a timeout waiting for readiness of the socket the request is related to.
  56. 27 class OperationTimeoutError < TimeoutError; end
  57. # Error raised when there was an error while resolving a domain to an IP.
  58. 27 class ResolveError < Error; end
  59. # Error raised when there was an error while resolving a domain to an IP
  60. # using a HTTPX::Resolver::Native resolver.
  61. 27 class NativeResolveError < ResolveError
  62. 27 attr_reader :connection, :host
  63. # initializes the exception with the +connection+ it refers to, the +host+ domain
  64. # which failed to resolve, and the error +message+.
  65. 27 def initialize(connection, host, message = "Can't resolve #{host}")
  66. 120 @connection = connection
  67. 120 @host = host
  68. 120 super(message)
  69. end
  70. end
  71. # The exception class for HTTP responses with 4xx or 5xx status.
  72. 27 class HTTPError < Error
  73. # The HTTPX::Response response object this exception refers to.
  74. 27 attr_reader :response
  75. # Creates the instance and assigns the HTTPX::Response +response+.
  76. 27 def initialize(response)
  77. 84 @response = response
  78. 84 super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
  79. end
  80. # The HTTP response status.
  81. #
  82. # error.status #=> 404
  83. 27 def status
  84. 16 @response.status
  85. end
  86. end
  87. end

lib/httpx/extensions.rb

67.86% lines covered

28 relevant lines. 19 lines covered and 9 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "uri"
  3. 27 module HTTPX
  4. 27 module ArrayExtensions
  5. 27 module FilterMap
  6. refine Array do
  7. # Ruby 2.7 backport
  8. def filter_map
  9. return to_enum(:filter_map) unless block_given?
  10. each_with_object([]) do |item, res|
  11. processed = yield(item)
  12. res << processed if processed
  13. end
  14. end
  15. 26 end unless Array.method_defined?(:filter_map)
  16. end
  17. 27 module Intersect
  18. refine Array do
  19. # Ruby 3.1 backport
  20. 4 def intersect?(arr)
  21. if size < arr.size
  22. smaller = self
  23. else
  24. smaller, arr = arr, self
  25. end
  26. (arr & smaller).size > 0
  27. end
  28. 26 end unless Array.method_defined?(:intersect?)
  29. end
  30. end
  31. 27 module URIExtensions
  32. # uri 0.11 backport, ships with ruby 3.1
  33. 27 refine URI::Generic do
  34. 27 def non_ascii_hostname
  35. 439 @non_ascii_hostname
  36. end
  37. 27 def non_ascii_hostname=(hostname)
  38. 32 @non_ascii_hostname = hostname
  39. end
  40. def authority
  41. 5666 return host if port == default_port
  42. 586 "#{host}:#{port}"
  43. 26 end unless URI::HTTP.method_defined?(:authority)
  44. def origin
  45. 4610 "#{scheme}://#{authority}"
  46. 26 end unless URI::HTTP.method_defined?(:origin)
  47. end
  48. end
  49. end

lib/httpx/headers.rb

100.0% lines covered

71 relevant lines. 71 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 class Headers
  4. 27 class << self
  5. 27 def new(headers = nil)
  6. 25779 return headers if headers.is_a?(self)
  7. 11798 super
  8. end
  9. end
  10. 27 def initialize(headers = nil)
  11. 11798 if headers.nil? || headers.empty?
  12. 1896 @headers = headers.to_h
  13. 1677 return
  14. end
  15. 9902 @headers = {}
  16. 9902 headers.each do |field, value|
  17. 61403 field = downcased(field)
  18. 61403 value = array_value(value)
  19. 61403 current = @headers[field]
  20. 61403 if current.nil?
  21. 54583 @headers[field] = value
  22. else
  23. 53 current.concat(value)
  24. end
  25. end
  26. end
  27. # cloned initialization
  28. 27 def initialize_clone(orig, **kwargs)
  29. 8 super
  30. 8 @headers = orig.instance_variable_get(:@headers).clone(**kwargs)
  31. end
  32. # dupped initialization
  33. 27 def initialize_dup(orig)
  34. 15339 super
  35. 15339 @headers = orig.instance_variable_get(:@headers).dup
  36. end
  37. # freezes the headers hash
  38. 27 def freeze
  39. 19875 @headers.freeze
  40. 19875 super
  41. end
  42. # merges headers with another header-quack.
  43. # the merge rule is, if the header already exists,
  44. # ignore what the +other+ headers has. Otherwise, set
  45. #
  46. 27 def merge(other)
  47. 4811 headers = dup
  48. 4811 other.each do |field, value|
  49. 4338 headers[downcased(field)] = value
  50. end
  51. 4811 headers
  52. end
  53. # returns the comma-separated values of the header field
  54. # identified by +field+, or nil otherwise.
  55. #
  56. 27 def [](field)
  57. 94606 a = @headers[downcased(field)] || return
  58. 28304 a.join(", ")
  59. end
  60. # sets +value+ (if not nil) as single value for the +field+ header.
  61. #
  62. 27 def []=(field, value)
  63. 41580 return unless value
  64. 37256 @headers[downcased(field)] = array_value(value)
  65. end
  66. # deletes all values associated with +field+ header.
  67. #
  68. 27 def delete(field)
  69. 265 canonical = downcased(field)
  70. 265 @headers.delete(canonical) if @headers.key?(canonical)
  71. end
  72. # adds additional +value+ to the existing, for header +field+.
  73. #
  74. 27 def add(field, value)
  75. 592 (@headers[downcased(field)] ||= []) << String(value)
  76. end
  77. # helper to be used when adding an header field as a value to another field
  78. #
  79. # h2_headers.add_header("vary", "accept-encoding")
  80. # h2_headers["vary"] #=> "accept-encoding"
  81. # h1_headers.add_header("vary", "accept-encoding")
  82. # h1_headers["vary"] #=> "Accept-Encoding"
  83. #
  84. 27 alias_method :add_header, :add
  85. # returns the enumerable headers store in pairs of header field + the values in
  86. # the comma-separated string format
  87. #
  88. 27 def each(extra_headers = nil)
  89. 67335 return enum_for(__method__, extra_headers) { @headers.size } unless block_given?
  90. 35836 @headers.each do |field, value|
  91. 48838 yield(field, value.join(", ")) unless value.empty?
  92. end
  93. 5975 extra_headers.each do |field, value|
  94. 24387 yield(field, value) unless value.empty?
  95. 35835 end if extra_headers
  96. end
  97. 27 def ==(other)
  98. 20892 other == to_hash
  99. end
  100. 27 def empty?
  101. 312 @headers.empty?
  102. end
  103. # the headers store in Hash format
  104. 27 def to_hash
  105. 22733 Hash[to_a]
  106. end
  107. 27 alias_method :to_h, :to_hash
  108. # the headers store in array of pairs format
  109. 27 def to_a
  110. 22756 Array(each)
  111. end
  112. # headers as string
  113. 27 def to_s
  114. 1896 @headers.to_s
  115. end
  116. skipped # :nocov:
  117. skipped def inspect
  118. skipped "#<#{self.class}:#{object_id} " \
  119. skipped "#{to_hash.inspect}>"
  120. skipped end
  121. skipped # :nocov:
  122. # this is internal API and doesn't abide to other public API
  123. # guarantees, like downcasing strings.
  124. # Please do not use this outside of core!
  125. #
  126. 27 def key?(downcased_key)
  127. 64310 @headers.key?(downcased_key)
  128. end
  129. # returns the values for the +field+ header in array format.
  130. # This method is more internal, and for this reason doesn't try
  131. # to "correct" the user input, i.e. it doesn't downcase the key.
  132. #
  133. 27 def get(field)
  134. 279 @headers[field] || EMPTY
  135. end
  136. 27 private
  137. 27 def array_value(value)
  138. 102983 Array(value)
  139. end
  140. 27 def downcased(field)
  141. 203204 String(field).downcase
  142. end
  143. end
  144. end

lib/httpx/io.rb

100.0% lines covered

5 relevant lines. 5 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "socket"
  3. 27 require "httpx/io/udp"
  4. 27 require "httpx/io/tcp"
  5. 27 require "httpx/io/unix"
  6. begin
  7. 27 require "httpx/io/ssl"
  8. rescue LoadError
  9. end

lib/httpx/io/ssl.rb

96.55% lines covered

87 relevant lines. 84 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "openssl"
  3. 27 module HTTPX
  4. 27 TLSError = OpenSSL::SSL::SSLError
  5. 27 class SSL < TCP
  6. # rubocop:disable Style/MutableConstant
  7. 27 TLS_OPTIONS = { alpn_protocols: %w[h2 http/1.1].freeze }
  8. # https://github.com/jruby/jruby-openssl/issues/284
  9. 27 TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby"
  10. # rubocop:enable Style/MutableConstant
  11. 27 TLS_OPTIONS.freeze
  12. 27 attr_writer :ssl_session
  13. 27 def initialize(_, _, options)
  14. 2995 super
  15. 2995 ctx_options = TLS_OPTIONS.merge(options.ssl)
  16. 2995 @sni_hostname = ctx_options.delete(:hostname) || @hostname
  17. 2995 if @keep_open && @io.is_a?(OpenSSL::SSL::SSLSocket)
  18. # externally initiated ssl socket
  19. 22 @ctx = @io.context
  20. 22 @state = :negotiated
  21. else
  22. 2973 @ctx = OpenSSL::SSL::SSLContext.new
  23. 2973 @ctx.set_params(ctx_options) unless ctx_options.empty?
  24. 2973 unless @ctx.session_cache_mode.nil? # a dummy method on JRuby
  25. 2639 @ctx.session_cache_mode =
  26. OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
  27. end
  28. 2973 yield(self) if block_given?
  29. end
  30. 2995 @verify_hostname = @ctx.verify_hostname
  31. end
  32. 27 if OpenSSL::SSL::SSLContext.method_defined?(:session_new_cb=)
  33. 26 def session_new_cb(&pr)
  34. 7615 @ctx.session_new_cb = proc { |_, sess| pr.call(sess) }
  35. end
  36. else
  37. # session_new_cb not implemented under JRuby
  38. 1 def session_new_cb; end
  39. end
  40. 27 def protocol
  41. 2883 @io.alpn_protocol || super
  42. rescue StandardError
  43. 7 super
  44. end
  45. 27 if RUBY_ENGINE == "jruby"
  46. # in jruby, alpn_protocol may return ""
  47. # https://github.com/jruby/jruby-openssl/issues/287
  48. 1 def protocol
  49. 366 proto = @io.alpn_protocol
  50. 365 return super if proto.nil? || proto.empty?
  51. 364 proto
  52. rescue StandardError
  53. 1 super
  54. end
  55. end
  56. 27 def can_verify_peer?
  57. 13 @ctx.verify_mode == OpenSSL::SSL::VERIFY_PEER
  58. end
  59. 27 def verify_hostname(host)
  60. 15 return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
  61. 15 return false if !@io.respond_to?(:peer_cert) || @io.peer_cert.nil?
  62. 15 OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
  63. end
  64. 27 def connected?
  65. 12664 @state == :negotiated
  66. end
  67. 27 def expired?
  68. super || ssl_session_expired?
  69. end
  70. 27 def ssl_session_expired?
  71. 3208 @ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (@ssl_session.time.to_f + @ssl_session.timeout)
  72. end
  73. 27 def connect
  74. 12720 return if @state == :negotiated
  75. 12720 unless @state == :connected
  76. 7097 super
  77. 7065 return unless @state == :connected
  78. end
  79. 8742 unless @io.is_a?(OpenSSL::SSL::SSLSocket)
  80. 3208 if (hostname_is_ip = (@ip == @sni_hostname))
  81. # IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
  82. 32 @sni_hostname = @ip.to_string
  83. # IP addresses in SNI is not valid per RFC 6066, section 3.
  84. 32 @ctx.verify_hostname = false
  85. end
  86. 3208 @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
  87. 3208 @io.hostname = @sni_hostname unless hostname_is_ip
  88. 3208 @io.session = @ssl_session unless ssl_session_expired?
  89. 3208 @io.sync_close = true
  90. end
  91. 8742 try_ssl_connect
  92. end
  93. 27 def try_ssl_connect
  94. 8742 ret = @io.connect_nonblock(exception: false)
  95. 8762 log(level: 3, color: :cyan) { "TLS CONNECT: #{ret}..." }
  96. 7926 case ret
  97. when :wait_readable
  98. 5558 @interests = :r
  99. 5558 return
  100. when :wait_writable
  101. @interests = :w
  102. return
  103. end
  104. 3161 @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && @verify_hostname
  105. 3160 transition(:negotiated)
  106. 3160 @interests = :w
  107. end
  108. 27 private
  109. 27 def transition(nextstate)
  110. 11065 case nextstate
  111. when :negotiated
  112. 3160 return unless @state == :connected
  113. when :closed
  114. 3083 return unless @state == :negotiated ||
  115. @state == :connected
  116. end
  117. 12513 do_transition(nextstate)
  118. end
  119. 27 def log_transition_state(nextstate)
  120. 70 return super unless nextstate == :negotiated
  121. 16 server_cert = @io.peer_cert
  122. 14 "#{super}\n\n" \
  123. 2 "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
  124. 2 "ALPN, server accepted to use #{protocol}\n" \
  125. "Server certificate:\n " \
  126. 2 "subject: #{server_cert.subject}\n " \
  127. 2 "start date: #{server_cert.not_before}\n " \
  128. 2 "expire date: #{server_cert.not_after}\n " \
  129. 2 "issuer: #{server_cert.issuer}\n " \
  130. "SSL certificate verify ok."
  131. end
  132. end
  133. end

lib/httpx/io/tcp.rb

91.15% lines covered

113 relevant lines. 103 lines covered and 10 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "ipaddr"
  4. 27 module HTTPX
  5. 27 class TCP
  6. 27 include Loggable
  7. 27 using URIExtensions
  8. 27 attr_reader :ip, :port, :addresses, :state, :interests
  9. 27 alias_method :host, :ip
  10. 27 def initialize(origin, addresses, options)
  11. 7074 @state = :idle
  12. 7074 @addresses = []
  13. 7074 @hostname = origin.host
  14. 7074 @options = options
  15. 7074 @fallback_protocol = @options.fallback_protocol
  16. 7074 @port = origin.port
  17. 7074 @interests = :w
  18. 7074 if @options.io
  19. 52 @io = case @options.io
  20. when Hash
  21. 16 @options.io[origin.authority]
  22. else
  23. 36 @options.io
  24. end
  25. 52 raise Error, "Given IO objects do not match the request authority" unless @io
  26. 52 _, _, _, @ip = @io.addr
  27. 52 @addresses << @ip
  28. 52 @keep_open = true
  29. 52 @state = :connected
  30. else
  31. 7022 add_addresses(addresses)
  32. end
  33. 7074 @ip_index = @addresses.size - 1
  34. end
  35. 27 def socket
  36. 195 @io
  37. end
  38. 27 def add_addresses(addrs)
  39. 7263 return if addrs.empty?
  40. 24782 addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
  41. 7263 ip_index = @ip_index || (@addresses.size - 1)
  42. 7263 if addrs.first.ipv6?
  43. # should be the next in line
  44. 251 @addresses = [*@addresses[0, ip_index], *addrs, *@addresses[ip_index..-1]]
  45. else
  46. 7012 @addresses.unshift(*addrs)
  47. 7012 @ip_index += addrs.size if @ip_index
  48. end
  49. end
  50. 27 def to_io
  51. 23059 @io.to_io
  52. end
  53. 27 def protocol
  54. 4441 @fallback_protocol
  55. end
  56. 27 def connect
  57. 43530 return unless closed?
  58. 43403 if !@io || @io.closed?
  59. 7748 transition(:idle)
  60. 7748 @io = build_socket
  61. end
  62. 43403 try_connect
  63. rescue Errno::ECONNREFUSED,
  64. Errno::EADDRNOTAVAIL,
  65. Errno::EHOSTUNREACH,
  66. SocketError,
  67. IOError => e
  68. 897 raise e if @ip_index <= 0
  69. 835 log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
  70. 812 @ip_index -= 1
  71. 823 @io = build_socket
  72. 823 retry
  73. rescue Errno::ETIMEDOUT => e
  74. raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index <= 0
  75. log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
  76. @ip_index -= 1
  77. @io = build_socket
  78. retry
  79. end
  80. 27 def try_connect
  81. 43403 ret = @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s), exception: false)
  82. 16337 log(level: 3, color: :cyan) { "TCP CONNECT: #{ret}..." }
  83. 14466 case ret
  84. when :wait_readable
  85. @interests = :r
  86. return
  87. when :wait_writable
  88. 8559 @interests = :w
  89. 8559 return
  90. end
  91. 7674 transition(:connected)
  92. 7674 @interests = :w
  93. rescue Errno::EALREADY
  94. 26273 @interests = :w
  95. end
  96. 27 private :try_connect
  97. 27 def read(size, buffer)
  98. 51113 ret = @io.read_nonblock(size, buffer, exception: false)
  99. 51113 if ret == :wait_readable
  100. 10829 buffer.clear
  101. 9980 return 0
  102. end
  103. 40284 return if ret.nil?
  104. 40342 log { "READ: #{buffer.bytesize} bytes..." }
  105. 40265 buffer.bytesize
  106. end
  107. 27 def write(buffer)
  108. 21294 siz = @io.write_nonblock(buffer, exception: false)
  109. 21275 return 0 if siz == :wait_writable
  110. 21259 return if siz.nil?
  111. 21345 log { "WRITE: #{siz} bytes..." }
  112. 21259 buffer.shift!(siz)
  113. 21259 siz
  114. end
  115. 27 def close
  116. 8465 return if @keep_open || closed?
  117. 859 begin
  118. 7479 @io.close
  119. ensure
  120. 7479 transition(:closed)
  121. end
  122. end
  123. 27 def connected?
  124. 35582 @state == :connected
  125. end
  126. 27 def closed?
  127. 51982 @state == :idle || @state == :closed
  128. end
  129. 27 def expired?
  130. # do not mess with external sockets
  131. return false if @options.io
  132. return true unless @addresses
  133. resolver_addresses = Resolver.nolookup_resolve(@hostname)
  134. (Array(resolver_addresses) & @addresses).empty?
  135. end
  136. skipped # :nocov:
  137. skipped def inspect
  138. skipped "#<#{self.class}:#{object_id} " \
  139. skipped "#{@ip}:#{@port} " \
  140. skipped "@state=#{@state} " \
  141. skipped "@hostname=#{@hostname} " \
  142. skipped "@addresses=#{@addresses} " \
  143. skipped "@state=#{@state}>"
  144. skipped end
  145. skipped # :nocov:
  146. 27 private
  147. 27 def build_socket
  148. 8571 @ip = @addresses[@ip_index]
  149. 8571 Socket.new(@ip.family, :STREAM, 0)
  150. end
  151. 27 def transition(nextstate)
  152. 12026 case nextstate
  153. # when :idle
  154. when :connected
  155. 4569 return unless @state == :idle
  156. when :closed
  157. 4396 return unless @state == :connected
  158. end
  159. 13569 do_transition(nextstate)
  160. end
  161. 27 def do_transition(nextstate)
  162. 26230 log(level: 1) { log_transition_state(nextstate) }
  163. 26082 @state = nextstate
  164. end
  165. 27 def log_transition_state(nextstate)
  166. 131 case nextstate
  167. when :connected
  168. 40 "Connected to #{host} (##{@io.fileno})"
  169. else
  170. 96 "#{host} #{@state} -> #{nextstate}"
  171. end
  172. end
  173. end
  174. end

lib/httpx/io/udp.rb

100.0% lines covered

35 relevant lines. 35 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "ipaddr"
  3. 27 module HTTPX
  4. 27 class UDP
  5. 27 include Loggable
  6. 27 def initialize(ip, port, options)
  7. 370 @host = ip
  8. 370 @port = port
  9. 370 @io = UDPSocket.new(IPAddr.new(ip).family)
  10. 370 @options = options
  11. end
  12. 27 def to_io
  13. 1062 @io.to_io
  14. end
  15. 27 def connect; end
  16. 27 def connected?
  17. 370 true
  18. end
  19. 27 def close
  20. 370 @io.close
  21. end
  22. 27 if RUBY_ENGINE == "jruby"
  23. # In JRuby, sendmsg_nonblock is not implemented
  24. 1 def write(buffer)
  25. 59 siz = @io.send(buffer.to_s, 0, @host, @port)
  26. 59 log { "WRITE: #{siz} bytes..." }
  27. 59 buffer.shift!(siz)
  28. 59 siz
  29. end
  30. else
  31. 26 def write(buffer)
  32. 664 siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s), exception: false)
  33. 664 return 0 if siz == :wait_writable
  34. 664 return if siz.nil?
  35. 664 log { "WRITE: #{siz} bytes..." }
  36. 664 buffer.shift!(siz)
  37. 664 siz
  38. end
  39. end
  40. 27 def read(size, buffer)
  41. 1233 ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
  42. 1233 return 0 if ret == :wait_readable
  43. 648 return if ret.nil?
  44. 648 log { "READ: #{buffer.bytesize} bytes..." }
  45. 648 buffer.bytesize
  46. rescue IOError
  47. end
  48. end
  49. end

lib/httpx/io/unix.rb

94.29% lines covered

35 relevant lines. 33 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 class UNIX < TCP
  4. 27 using URIExtensions
  5. 27 attr_reader :path
  6. 27 alias_method :host, :path
  7. 27 def initialize(origin, path, options)
  8. 28 @addresses = []
  9. 28 @hostname = origin.host
  10. 28 @state = :idle
  11. 28 @options = options
  12. 28 @fallback_protocol = @options.fallback_protocol
  13. 28 if @options.io
  14. 14 @io = case @options.io
  15. when Hash
  16. 7 @options.io[origin.authority]
  17. else
  18. 7 @options.io
  19. end
  20. 14 raise Error, "Given IO objects do not match the request authority" unless @io
  21. 14 @path = @io.path
  22. 14 @keep_open = true
  23. 14 @state = :connected
  24. 14 elsif path
  25. 14 @path = path
  26. else
  27. raise Error, "No path given where to store the socket"
  28. end
  29. 28 @io ||= build_socket
  30. end
  31. 27 def connect
  32. 21 return unless closed?
  33. begin
  34. 21 if @io.closed?
  35. 7 transition(:idle)
  36. 7 @io = build_socket
  37. end
  38. 21 @io.connect_nonblock(Socket.sockaddr_un(@path))
  39. rescue Errno::EISCONN
  40. end
  41. 14 transition(:connected)
  42. rescue Errno::EINPROGRESS,
  43. Errno::EALREADY,
  44. ::IO::WaitReadable
  45. end
  46. 27 def expired?
  47. false
  48. end
  49. skipped # :nocov:
  50. skipped def inspect
  51. skipped "#<#{self.class}:#{object_id} @path=#{@path}) @state=#{@state})>"
  52. skipped end
  53. skipped # :nocov:
  54. 27 private
  55. 27 def build_socket
  56. 21 Socket.new(Socket::PF_UNIX, :STREAM, 0)
  57. end
  58. end
  59. end

lib/httpx/loggable.rb

100.0% lines covered

17 relevant lines. 17 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Loggable
  4. 27 COLORS = {
  5. black: 30,
  6. red: 31,
  7. green: 32,
  8. yellow: 33,
  9. blue: 34,
  10. magenta: 35,
  11. cyan: 36,
  12. white: 37,
  13. }.freeze
  14. 27 USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
  15. 27 def log(level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug, &msg)
  16. 443797 return unless debug_level >= level
  17. 224980 debug_stream = debug || ($stderr if USE_DEBUG_LOG)
  18. 224980 return unless debug_stream
  19. 1911 klass = self.class
  20. 3586 until (class_name = klass.name)
  21. 225 klass = klass.superclass
  22. end
  23. 1911 message = +"(pid:#{Process.pid} tid:#{Thread.current.object_id}, self:#{class_name}##{object_id}) "
  24. 1911 message << msg.call << "\n"
  25. 1911 message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
  26. 1911 debug_stream << message
  27. end
  28. 27 def log_exception(ex, level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug)
  29. 1188 log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
  30. end
  31. end
  32. end

lib/httpx/options.rb

98.82% lines covered

169 relevant lines. 167 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "socket"
  3. 27 module HTTPX
  4. # Contains a set of options which are passed and shared across from session to its requests or
  5. # responses.
  6. 27 class Options
  7. 27 BUFFER_SIZE = 1 << 14
  8. 27 WINDOW_SIZE = 1 << 14 # 16K
  9. 27 MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
  10. 27 KEEP_ALIVE_TIMEOUT = 20
  11. 27 SETTINGS_TIMEOUT = 10
  12. 27 CLOSE_HANDSHAKE_TIMEOUT = 10
  13. 27 CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
  14. 27 REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
  15. # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
  16. 2 ip_address_families = begin
  17. 27 list = Socket.ip_address_list
  18. 110 if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? && !a.ipv6_unique_local? }
  19. [Socket::AF_INET6, Socket::AF_INET]
  20. else
  21. 27 [Socket::AF_INET]
  22. end
  23. rescue NotImplementedError
  24. [Socket::AF_INET]
  25. end.freeze
  26. 27 SET_TEMPORARY_NAME = ->(mod, pl = nil) do
  27. 9470 if mod.respond_to?(:set_temporary_name) # ruby 3.4 only
  28. 3561 name = mod.name || "#{mod.superclass.name}(plugin)"
  29. 3561 name = "#{name}/#{pl}" if pl
  30. 3561 mod.set_temporary_name(name)
  31. end
  32. end
  33. 2 DEFAULT_OPTIONS = {
  34. 25 :max_requests => Float::INFINITY,
  35. :debug => nil,
  36. 27 :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
  37. :ssl => EMPTY_HASH,
  38. :http2_settings => { settings_enable_push: 0 }.freeze,
  39. :fallback_protocol => "http/1.1",
  40. :supported_compression_formats => %w[gzip deflate],
  41. :decompress_response_body => true,
  42. :compress_request_body => true,
  43. :timeout => {
  44. connect_timeout: CONNECT_TIMEOUT,
  45. settings_timeout: SETTINGS_TIMEOUT,
  46. close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
  47. operation_timeout: OPERATION_TIMEOUT,
  48. keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
  49. read_timeout: READ_TIMEOUT,
  50. write_timeout: WRITE_TIMEOUT,
  51. request_timeout: REQUEST_TIMEOUT,
  52. },
  53. :headers_class => Class.new(Headers, &SET_TEMPORARY_NAME),
  54. :headers => {},
  55. :window_size => WINDOW_SIZE,
  56. :buffer_size => BUFFER_SIZE,
  57. :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
  58. :request_class => Class.new(Request, &SET_TEMPORARY_NAME),
  59. :response_class => Class.new(Response, &SET_TEMPORARY_NAME),
  60. :request_body_class => Class.new(Request::Body, &SET_TEMPORARY_NAME),
  61. :response_body_class => Class.new(Response::Body, &SET_TEMPORARY_NAME),
  62. :pool_class => Class.new(Pool, &SET_TEMPORARY_NAME),
  63. :connection_class => Class.new(Connection, &SET_TEMPORARY_NAME),
  64. :options_class => Class.new(self, &SET_TEMPORARY_NAME),
  65. :transport => nil,
  66. :addresses => nil,
  67. :persistent => false,
  68. 27 :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
  69. :resolver_options => { cache: true }.freeze,
  70. :pool_options => EMPTY_HASH,
  71. :ip_families => ip_address_families,
  72. :close_on_fork => false,
  73. }.freeze
  74. 27 class << self
  75. 27 def new(options = {})
  76. # let enhanced options go through
  77. 11845 return options if self == Options && options.class < self
  78. 9111 return options if options.is_a?(self)
  79. 4437 super
  80. end
  81. 27 def method_added(meth)
  82. 20903 super
  83. 20903 return unless meth =~ /^option_(.+)$/
  84. 9781 optname = Regexp.last_match(1).to_sym
  85. 9781 attr_reader(optname)
  86. end
  87. end
  88. # creates a new options instance from a given hash, which optionally define the following:
  89. #
  90. # :debug :: an object which log messages are written to (must respond to <tt><<</tt>)
  91. # :debug_level :: the log level of messages (can be 1, 2, or 3).
  92. # :ssl :: a hash of options which can be set as params of OpenSSL::SSL::SSLContext (see HTTPX::IO::SSL)
  93. # :http2_settings :: a hash of options to be passed to a HTTP2::Connection (ex: <tt>{ max_concurrent_streams: 2 }</tt>)
  94. # :fallback_protocol :: version of HTTP protocol to use by default in the absence of protocol negotiation
  95. # like ALPN (defaults to <tt>"http/1.1"</tt>)
  96. # :supported_compression_formats :: list of compressions supported by the transcoder layer (defaults to <tt>%w[gzip deflate]</tt>).
  97. # :decompress_response_body :: whether to auto-decompress response body (defaults to <tt>true</tt>).
  98. # :compress_request_body :: whether to auto-decompress response body (defaults to <tt>true</tt>)
  99. # :timeout :: hash of timeout configurations (supports <tt>:connect_timeout</tt>, <tt>:settings_timeout</tt>,
  100. # <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>, <tt>:read_timeout</tt>, <tt>:write_timeout</tt>
  101. # and <tt>:request_timeout</tt>
  102. # :headers :: hash of HTTP headers (ex: <tt>{ "x-custom-foo" => "bar" }</tt>)
  103. # :window_size :: number of bytes to read from a socket
  104. # :buffer_size :: internal read and write buffer size in bytes
  105. # :body_threshold_size :: maximum size in bytes of response payload that is buffered in memory.
  106. # :request_class :: class used to instantiate a request
  107. # :response_class :: class used to instantiate a response
  108. # :headers_class :: class used to instantiate headers
  109. # :request_body_class :: class used to instantiate a request body
  110. # :response_body_class :: class used to instantiate a response body
  111. # :connection_class :: class used to instantiate connections
  112. # :pool_class :: class used to instantiate the session connection pool
  113. # :options_class :: class used to instantiate options
  114. # :transport :: type of transport to use (set to "unix" for UNIX sockets)
  115. # :addresses :: bucket of peer addresses (can be a list of IP addresses, a hash of domain to list of adddresses;
  116. # paths should be used for UNIX sockets instead)
  117. # :io :: open socket, or domain/ip-to-socket hash, which requests should be sent to
  118. # :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
  119. # :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
  120. # using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class)
  121. # :resolver_options :: hash of options passed to the resolver. Accepted keys depend on the resolver type.
  122. # :pool_options :: hash of options passed to the connection pool (See Pool#initialize).
  123. # :ip_families :: which socket families are supported (system-dependent)
  124. # :origin :: HTTP origin to set on requests with relative path (ex: "https://api.serv.com")
  125. # :base_path :: path to prefix given relative paths with (ex: "/v2")
  126. # :max_concurrent_requests :: max number of requests which can be set concurrently
  127. # :max_requests :: max number of requests which can be made on socket before it reconnects.
  128. # :close_on_fork :: whether the session automatically closes when the process is fork (defaults to <tt>false</tt>).
  129. # it only works if the session is persistent (and ruby 3.1 or higher is used).
  130. #
  131. # This list of options are enhanced with each loaded plugin, see the plugin docs for details.
  132. 27 def initialize(options = {})
  133. 4437 do_initialize(options)
  134. 4421 freeze
  135. end
  136. 27 def freeze
  137. 11787 super
  138. 11787 @origin.freeze
  139. 11787 @base_path.freeze
  140. 11787 @timeout.freeze
  141. 11787 @headers.freeze
  142. 11787 @addresses.freeze
  143. 11787 @supported_compression_formats.freeze
  144. end
  145. 27 def option_origin(value)
  146. 608 URI(value)
  147. end
  148. 27 def option_base_path(value)
  149. 32 String(value)
  150. end
  151. 27 def option_headers(value)
  152. 7916 headers_class.new(value)
  153. end
  154. 27 def option_timeout(value)
  155. 8526 Hash[value]
  156. end
  157. 27 def option_supported_compression_formats(value)
  158. 7324 Array(value).map(&:to_s)
  159. end
  160. 27 def option_max_concurrent_requests(value)
  161. 963 raise TypeError, ":max_concurrent_requests must be positive" unless value.positive?
  162. 963 value
  163. end
  164. 27 def option_max_requests(value)
  165. 7318 raise TypeError, ":max_requests must be positive" unless value.positive?
  166. 7318 value
  167. end
  168. 27 def option_window_size(value)
  169. 7316 value = Integer(value)
  170. 7316 raise TypeError, ":window_size must be positive" unless value.positive?
  171. 7316 value
  172. end
  173. 27 def option_buffer_size(value)
  174. 7316 value = Integer(value)
  175. 7316 raise TypeError, ":buffer_size must be positive" unless value.positive?
  176. 7316 value
  177. end
  178. 27 def option_body_threshold_size(value)
  179. 7300 bytes = Integer(value)
  180. 7300 raise TypeError, ":body_threshold_size must be positive" unless bytes.positive?
  181. 7300 bytes
  182. end
  183. 27 def option_transport(value)
  184. 49 transport = value.to_s
  185. 49 raise TypeError, "#{transport} is an unsupported transport type" unless %w[unix].include?(transport)
  186. 49 transport
  187. end
  188. 27 def option_addresses(value)
  189. 43 Array(value)
  190. end
  191. 27 def option_ip_families(value)
  192. 7300 Array(value)
  193. end
  194. 27 %i[
  195. ssl http2_settings
  196. request_class response_class headers_class request_body_class
  197. response_body_class connection_class options_class
  198. pool_class pool_options
  199. io fallback_protocol debug debug_level resolver_class resolver_options
  200. compress_request_body decompress_response_body
  201. persistent close_on_fork
  202. ].each do |method_name|
  203. 567 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  204. 21 # sets +v+ as the value of #{method_name}
  205. 21 def option_#{method_name}(v); v; end # def option_smth(v); v; end
  206. OUT
  207. end
  208. 27 REQUEST_BODY_IVARS = %i[@headers].freeze
  209. 27 def ==(other)
  210. 1961 super || options_equals?(other)
  211. end
  212. 27 def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
  213. # headers and other request options do not play a role, as they are
  214. # relevant only for the request.
  215. 461 ivars = instance_variables - ignore_ivars
  216. 461 other_ivars = other.instance_variables - ignore_ivars
  217. 461 return false if ivars.size != other_ivars.size
  218. 443 return false if ivars.sort != other_ivars.sort
  219. 443 ivars.all? do |ivar|
  220. 11121 instance_variable_get(ivar) == other.instance_variable_get(ivar)
  221. end
  222. end
  223. 27 def merge(other)
  224. 35691 ivar_map = nil
  225. 35691 other_ivars = case other
  226. when Hash
  227. 42559 ivar_map = other.keys.to_h { |k| [:"@#{k}", k] }
  228. 24812 ivar_map.keys
  229. else
  230. 10879 other.instance_variables
  231. end
  232. 35691 return self if other_ivars.empty?
  233. 300897 return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == access_option(other, ivar, ivar_map) }
  234. 13117 opts = dup
  235. 13117 other_ivars.each do |ivar|
  236. 95623 v = access_option(other, ivar, ivar_map)
  237. 95623 unless v
  238. 6315 opts.instance_variable_set(ivar, v)
  239. 6315 next
  240. end
  241. 89308 v = opts.__send__(:"option_#{ivar[1..-1]}", v)
  242. 89292 orig_v = instance_variable_get(ivar)
  243. 89292 v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
  244. 89292 opts.instance_variable_set(ivar, v)
  245. end
  246. 13101 opts
  247. end
  248. 27 def to_hash
  249. 3447 instance_variables.each_with_object({}) do |ivar, hs|
  250. 83629 hs[ivar[1..-1].to_sym] = instance_variable_get(ivar)
  251. end
  252. end
  253. 27 def extend_with_plugin_classes(pl)
  254. 7323 if defined?(pl::RequestMethods) || defined?(pl::RequestClassMethods)
  255. 2113 @request_class = @request_class.dup
  256. 2113 SET_TEMPORARY_NAME[@request_class, pl]
  257. 2113 @request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
  258. 2113 @request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
  259. end
  260. 7323 if defined?(pl::ResponseMethods) || defined?(pl::ResponseClassMethods)
  261. 2351 @response_class = @response_class.dup
  262. 2351 SET_TEMPORARY_NAME[@response_class, pl]
  263. 2351 @response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
  264. 2351 @response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
  265. end
  266. 7323 if defined?(pl::HeadersMethods) || defined?(pl::HeadersClassMethods)
  267. 152 @headers_class = @headers_class.dup
  268. 152 SET_TEMPORARY_NAME[@headers_class, pl]
  269. 152 @headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
  270. 152 @headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
  271. end
  272. 7323 if defined?(pl::RequestBodyMethods) || defined?(pl::RequestBodyClassMethods)
  273. 322 @request_body_class = @request_body_class.dup
  274. 322 SET_TEMPORARY_NAME[@request_body_class, pl]
  275. 322 @request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
  276. 322 @request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
  277. end
  278. 7323 if defined?(pl::ResponseBodyMethods) || defined?(pl::ResponseBodyClassMethods)
  279. 768 @response_body_class = @response_body_class.dup
  280. 768 SET_TEMPORARY_NAME[@response_body_class, pl]
  281. 768 @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
  282. 768 @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
  283. end
  284. 7323 if defined?(pl::PoolMethods)
  285. 598 @pool_class = @pool_class.dup
  286. 598 SET_TEMPORARY_NAME[@pool_class, pl]
  287. 598 @pool_class.__send__(:include, pl::PoolMethods)
  288. end
  289. 7323 if defined?(pl::ConnectionMethods)
  290. 2950 @connection_class = @connection_class.dup
  291. 2950 SET_TEMPORARY_NAME[@connection_class, pl]
  292. 2950 @connection_class.__send__(:include, pl::ConnectionMethods)
  293. end
  294. 7323 return unless defined?(pl::OptionsMethods)
  295. 2981 @options_class = @options_class.dup
  296. 2981 @options_class.__send__(:include, pl::OptionsMethods)
  297. end
  298. 27 private
  299. 27 def do_initialize(options = {})
  300. 4437 defaults = DEFAULT_OPTIONS.merge(options)
  301. 4437 defaults.each do |k, v|
  302. 134000 next if v.nil?
  303. 120689 option_method_name = :"option_#{k}"
  304. 120689 raise Error, "unknown option: #{k}" unless respond_to?(option_method_name)
  305. 120681 value = __send__(option_method_name, v)
  306. 120673 instance_variable_set(:"@#{k}", value)
  307. end
  308. end
  309. 27 def access_option(obj, k, ivar_map)
  310. 336331 case obj
  311. when Hash
  312. 28773 obj[ivar_map[k]]
  313. else
  314. 345892 obj.instance_variable_get(k)
  315. end
  316. end
  317. end
  318. end

lib/httpx/parser/http1.rb

100.0% lines covered

109 relevant lines. 109 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Parser
  4. 27 class Error < Error; end
  5. 27 class HTTP1
  6. 27 VERSIONS = %w[1.0 1.1].freeze
  7. 27 attr_reader :status_code, :http_version, :headers
  8. 27 def initialize(observer)
  9. 4599 @observer = observer
  10. 4599 @state = :idle
  11. 4599 @buffer = "".b
  12. 4599 @headers = {}
  13. end
  14. 27 def <<(chunk)
  15. 7349 @buffer << chunk
  16. 7349 parse
  17. end
  18. 27 def reset!
  19. 9403 @state = :idle
  20. 9403 @headers = {}
  21. 9403 @content_length = nil
  22. 9403 @_has_trailers = nil
  23. end
  24. 27 def upgrade?
  25. 4613 @upgrade
  26. end
  27. 27 def upgrade_data
  28. 32 @buffer
  29. end
  30. 27 private
  31. 27 def parse
  32. 7349 loop do
  33. 15598 state = @state
  34. 13960 case @state
  35. when :idle
  36. 4966 parse_headline
  37. when :headers, :trailers
  38. 5054 parse_headers
  39. when :data
  40. 5576 parse_data
  41. end
  42. 11408 return if @buffer.empty? || state == @state
  43. end
  44. end
  45. 27 def parse_headline
  46. 4966 idx = @buffer.index("\n")
  47. 4966 return unless idx
  48. 4966 (m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
  49. raise(Error, "wrong head line format")
  50. 4958 version, code, _ = m.captures
  51. 4958 raise(Error, "unsupported HTTP version (HTTP/#{version})") unless version && VERSIONS.include?(version)
  52. 4950 @http_version = version.split(".").map(&:to_i)
  53. 4950 @status_code = code.to_i
  54. 4950 raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
  55. 4942 @buffer = @buffer.byteslice((idx + 1)..-1)
  56. 4942 nextstate(:headers)
  57. end
  58. 27 def parse_headers
  59. 5056 headers = @headers
  60. 5056 buffer = @buffer
  61. 39165 while (idx = buffer.index("\n"))
  62. # @type var line: String
  63. 38581 line = buffer.byteslice(0..idx)
  64. 38581 raise Error, "wrong header format" if line.start_with?("\s", "\t")
  65. 38573 line.lstrip!
  66. 38573 buffer = @buffer = buffer.byteslice((idx + 1)..-1)
  67. 38573 if line.empty?
  68. 4430 case @state
  69. when :headers
  70. 4926 prepare_data(headers)
  71. 4926 @observer.on_headers(headers)
  72. 4299 return unless @state == :headers
  73. # state might have been reset
  74. # in the :headers callback
  75. 4227 nextstate(:data)
  76. 4227 headers.clear
  77. when :trailers
  78. 16 @observer.on_trailers(headers)
  79. 16 headers.clear
  80. 16 nextstate(:complete)
  81. end
  82. 4235 return
  83. end
  84. 33631 separator_index = line.index(":")
  85. 33631 raise Error, "wrong header format" unless separator_index
  86. # @type var key: String
  87. 33623 key = line.byteslice(0..(separator_index - 1))
  88. 33623 key.rstrip! # was lstripped previously!
  89. # @type var value: String
  90. 33623 value = line.byteslice((separator_index + 1)..-1)
  91. 33623 value.strip!
  92. 33623 raise Error, "wrong header format" if value.nil?
  93. 33623 (headers[key.downcase] ||= []) << value
  94. end
  95. end
  96. 27 def parse_data
  97. 5576 if @buffer.respond_to?(:each)
  98. 201 @buffer.each do |chunk|
  99. 234 @observer.on_data(chunk)
  100. end
  101. 5374 elsif @content_length
  102. # @type var data: String
  103. 5335 data = @buffer.byteslice(0, @content_length)
  104. 5335 @buffer = @buffer.byteslice(@content_length..-1) || "".b
  105. 4767 @content_length -= data.bytesize
  106. 5335 @observer.on_data(data)
  107. 5313 data.clear
  108. else
  109. 40 @observer.on_data(@buffer)
  110. 40 @buffer.clear
  111. end
  112. 5546 return unless no_more_data?
  113. 4077 @buffer = @buffer.to_s
  114. 4077 if @_has_trailers
  115. 16 nextstate(:trailers)
  116. else
  117. 4061 nextstate(:complete)
  118. end
  119. end
  120. 27 def prepare_data(headers)
  121. 4926 @upgrade = headers.key?("upgrade")
  122. 4926 @_has_trailers = headers.key?("trailer")
  123. 4926 if (tr_encodings = headers["transfer-encoding"])
  124. 114 tr_encodings.reverse_each do |tr_encoding|
  125. 114 tr_encoding.split(/ *, */).each do |encoding|
  126. 100 case encoding
  127. when "chunked"
  128. 114 @buffer = Transcoder::Chunker::Decoder.new(@buffer, @_has_trailers)
  129. end
  130. end
  131. end
  132. else
  133. 4812 @content_length = headers["content-length"][0].to_i if headers.key?("content-length")
  134. end
  135. end
  136. 27 def no_more_data?
  137. 5546 if @content_length
  138. 5313 @content_length <= 0
  139. 232 elsif @buffer.respond_to?(:finished?)
  140. 193 @buffer.finished?
  141. else
  142. 40 false
  143. end
  144. end
  145. 27 def nextstate(state)
  146. 13262 @state = state
  147. 11898 case state
  148. when :headers
  149. 4942 @observer.on_start
  150. when :complete
  151. 4077 @observer.on_complete
  152. 584 reset!
  153. 584 nextstate(:idle) unless @buffer.empty?
  154. end
  155. end
  156. end
  157. end
  158. end

lib/httpx/plugins/auth.rb

100.0% lines covered

9 relevant lines. 9 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds a shim +authorization+ method to the session, which will fill
  6. # the HTTP Authorization header, and another, +bearer_auth+, which fill the "Bearer " prefix
  7. # in its value.
  8. #
  9. # https://gitlab.com/os85/httpx/wikis/Auth#auth
  10. #
  11. 8 module Auth
  12. 8 module InstanceMethods
  13. 8 def authorization(token)
  14. 144 with(headers: { "authorization" => token })
  15. end
  16. 8 def bearer_auth(token)
  17. 16 authorization("Bearer #{token}")
  18. end
  19. end
  20. end
  21. 8 register_plugin :auth, Auth
  22. end
  23. end

lib/httpx/plugins/auth/basic.rb

100.0% lines covered

10 relevant lines. 10 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 9 require "httpx/base64"
  3. 9 module HTTPX
  4. 9 module Plugins
  5. 9 module Authentication
  6. 9 class Basic
  7. 9 def initialize(user, password, **)
  8. 274 @user = user
  9. 274 @password = password
  10. end
  11. 9 def authenticate(*)
  12. 255 "Basic #{Base64.strict_encode64("#{@user}:#{@password}")}"
  13. end
  14. end
  15. end
  16. end
  17. end

lib/httpx/plugins/auth/digest.rb

100.0% lines covered

61 relevant lines. 61 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "time"
  3. 8 require "securerandom"
  4. 8 require "digest"
  5. 8 module HTTPX
  6. 8 module Plugins
  7. 8 module Authentication
  8. 8 class Digest
  9. 8 def initialize(user, password, hashed: false, **)
  10. 176 @user = user
  11. 176 @password = password
  12. 176 @nonce = 0
  13. 176 @hashed = hashed
  14. end
  15. 8 def can_authenticate?(authenticate)
  16. 160 authenticate && /Digest .*/.match?(authenticate)
  17. end
  18. 8 def authenticate(request, authenticate)
  19. 160 "Digest #{generate_header(request.verb, request.path, authenticate)}"
  20. end
  21. 8 private
  22. 8 def generate_header(meth, uri, authenticate)
  23. # discard first token, it's Digest
  24. 160 auth_info = authenticate[/^(\w+) (.*)/, 2]
  25. 160 params = auth_info.split(/ *, */)
  26. 832 .to_h { |val| val.split("=", 2) }
  27. 832 .transform_values { |v| v.delete("\"") }
  28. 160 nonce = params["nonce"]
  29. 160 nc = next_nonce
  30. # verify qop
  31. 160 qop = params["qop"]
  32. 160 if params["algorithm"] =~ /(.*?)(-sess)?$/
  33. 144 alg = Regexp.last_match(1)
  34. 144 algorithm = ::Digest.const_get(alg)
  35. 144 raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
  36. 144 sess = Regexp.last_match(2)
  37. else
  38. 16 algorithm = ::Digest::MD5
  39. end
  40. 160 if qop || sess
  41. 160 cnonce = make_cnonce
  42. 160 nc = format("%<nonce>08x", nonce: nc)
  43. end
  44. 160 a1 = if sess
  45. 4 [
  46. 32 (@hashed ? @password : algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}")),
  47. nonce,
  48. cnonce,
  49. 3 ].join ":"
  50. else
  51. 128 @hashed ? @password : "#{@user}:#{params["realm"]}:#{@password}"
  52. end
  53. 160 ha1 = algorithm.hexdigest(a1)
  54. 160 ha2 = algorithm.hexdigest("#{meth}:#{uri}")
  55. 160 request_digest = [ha1, nonce]
  56. 160 request_digest.push(nc, cnonce, qop) if qop
  57. 160 request_digest << ha2
  58. 160 request_digest = request_digest.join(":")
  59. 40 header = [
  60. 140 %(username="#{@user}"),
  61. 20 %(nonce="#{nonce}"),
  62. 20 %(uri="#{uri}"),
  63. 20 %(response="#{algorithm.hexdigest(request_digest)}"),
  64. ]
  65. 160 header << %(realm="#{params["realm"]}") if params.key?("realm")
  66. 160 header << %(algorithm=#{params["algorithm"]}) if params.key?("algorithm")
  67. 160 header << %(cnonce="#{cnonce}") if cnonce
  68. 160 header << %(nc=#{nc})
  69. 160 header << %(qop=#{qop}) if qop
  70. 160 header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
  71. 160 header.join ", "
  72. end
  73. 8 def make_cnonce
  74. 180 ::Digest::MD5.hexdigest [
  75. Time.now.to_i,
  76. Process.pid,
  77. SecureRandom.random_number(2**32),
  78. ].join ":"
  79. end
  80. 8 def next_nonce
  81. 140 @nonce += 1
  82. end
  83. end
  84. end
  85. end
  86. end

lib/httpx/plugins/auth/ntlm.rb

100.0% lines covered

19 relevant lines. 19 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 3 require "httpx/base64"
  3. 3 require "ntlm"
  4. 3 module HTTPX
  5. 3 module Plugins
  6. 3 module Authentication
  7. 3 class Ntlm
  8. 3 def initialize(user, password, domain: nil)
  9. 4 @user = user
  10. 4 @password = password
  11. 4 @domain = domain
  12. end
  13. 3 def can_authenticate?(authenticate)
  14. 2 authenticate && /NTLM .*/.match?(authenticate)
  15. end
  16. 3 def negotiate
  17. 4 "NTLM #{NTLM.negotiate(domain: @domain).to_base64}"
  18. end
  19. 3 def authenticate(_req, www)
  20. 2 challenge = www[/NTLM (.*)/, 1]
  21. 2 challenge = Base64.decode64(challenge)
  22. 2 ntlm_challenge = NTLM.authenticate(challenge, @user, @domain, @password).to_base64
  23. 2 "NTLM #{ntlm_challenge}"
  24. end
  25. end
  26. end
  27. end
  28. end

lib/httpx/plugins/auth/socks5.rb

100.0% lines covered

11 relevant lines. 11 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 module Plugins
  4. 10 module Authentication
  5. 10 class Socks5
  6. 10 def initialize(user, password, **)
  7. 48 @user = user
  8. 48 @password = password
  9. end
  10. 10 def can_authenticate?(*)
  11. 48 @user && @password
  12. end
  13. 10 def authenticate(*)
  14. 48 [0x01, @user.bytesize, @user, @password.bytesize, @password].pack("CCA*CA*")
  15. end
  16. end
  17. end
  18. end
  19. end

lib/httpx/plugins/aws_sdk_authentication.rb

100.0% lines covered

43 relevant lines. 43 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin applies AWS Sigv4 to requests, using the AWS SDK credentials and configuration.
  6. #
  7. # It requires the "aws-sdk-core" gem.
  8. #
  9. 8 module AwsSdkAuthentication
  10. # Mock configuration, to be used only when resolving credentials
  11. 8 class Configuration
  12. 8 attr_reader :profile
  13. 8 def initialize(profile)
  14. 32 @profile = profile
  15. end
  16. 8 def respond_to_missing?(*)
  17. 16 true
  18. end
  19. 8 def method_missing(*); end
  20. end
  21. #
  22. # encapsulates access to an AWS SDK credentials store.
  23. #
  24. 8 class Credentials
  25. 8 def initialize(aws_credentials)
  26. 16 @aws_credentials = aws_credentials
  27. end
  28. 8 def username
  29. 16 @aws_credentials.access_key_id
  30. end
  31. 8 def password
  32. 16 @aws_credentials.secret_access_key
  33. end
  34. 8 def security_token
  35. 16 @aws_credentials.session_token
  36. end
  37. end
  38. 8 class << self
  39. 8 def load_dependencies(_klass)
  40. 16 require "aws-sdk-core"
  41. end
  42. 8 def configure(klass)
  43. 16 klass.plugin(:aws_sigv4)
  44. end
  45. 8 def extra_options(options)
  46. 16 options.merge(max_concurrent_requests: 1)
  47. end
  48. 8 def credentials(profile)
  49. 16 mock_configuration = Configuration.new(profile)
  50. 16 Credentials.new(Aws::CredentialProviderChain.new(mock_configuration).resolve)
  51. end
  52. 8 def region(profile)
  53. # https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb#L62
  54. 16 keys = %w[AWS_REGION AMAZON_REGION AWS_DEFAULT_REGION]
  55. 16 env_region = ENV.values_at(*keys).compact.first
  56. 16 env_region = nil if env_region == ""
  57. 16 cfg_region = Aws.shared_config.region(profile: profile)
  58. 16 env_region || cfg_region
  59. end
  60. end
  61. # adds support for the following options:
  62. #
  63. # :aws_profile :: AWS account profile to retrieve credentials from.
  64. 8 module OptionsMethods
  65. 8 def option_aws_profile(value)
  66. 80 String(value)
  67. end
  68. end
  69. 8 module InstanceMethods
  70. #
  71. # aws_authentication
  72. # aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
  73. # aws_authentication()
  74. #
  75. 8 def aws_sdk_authentication(
  76. credentials: AwsSdkAuthentication.credentials(@options.aws_profile),
  77. region: AwsSdkAuthentication.region(@options.aws_profile),
  78. **options
  79. )
  80. 16 aws_sigv4_authentication(
  81. credentials: credentials,
  82. region: region,
  83. provider_prefix: "aws",
  84. header_provider_field: "amz",
  85. **options
  86. )
  87. end
  88. 8 alias_method :aws_auth, :aws_sdk_authentication
  89. end
  90. end
  91. 8 register_plugin :aws_sdk_authentication, AwsSdkAuthentication
  92. end
  93. end

lib/httpx/plugins/aws_sigv4.rb

100.0% lines covered

120 relevant lines. 120 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds AWS Sigv4 authentication.
  6. #
  7. # https://docs.aws.amazon.com/IAM/latest/UserGuide/signing-elements.html
  8. #
  9. # https://gitlab.com/os85/httpx/wikis/AWS-SigV4
  10. #
  11. 8 module AWSSigV4
  12. 8 Credentials = Struct.new(:username, :password, :security_token)
  13. # Signs requests using the AWS sigv4 signing.
  14. 8 class Signer
  15. 8 def initialize(
  16. service:,
  17. region:,
  18. credentials: nil,
  19. username: nil,
  20. password: nil,
  21. security_token: nil,
  22. provider_prefix: "aws",
  23. header_provider_field: "amz",
  24. unsigned_headers: [],
  25. apply_checksum_header: true,
  26. algorithm: "SHA256"
  27. )
  28. 152 @credentials = credentials || Credentials.new(username, password, security_token)
  29. 152 @service = service
  30. 152 @region = region
  31. 152 @unsigned_headers = Set.new(unsigned_headers.map(&:downcase))
  32. 152 @unsigned_headers << "authorization"
  33. 152 @unsigned_headers << "x-amzn-trace-id"
  34. 152 @unsigned_headers << "expect"
  35. 152 @apply_checksum_header = apply_checksum_header
  36. 152 @provider_prefix = provider_prefix
  37. 152 @header_provider_field = header_provider_field
  38. 152 @algorithm = algorithm
  39. end
  40. 8 def sign!(request)
  41. 152 lower_provider_prefix = "#{@provider_prefix}4"
  42. 152 upper_provider_prefix = lower_provider_prefix.upcase
  43. 152 downcased_algorithm = @algorithm.downcase
  44. 152 datetime = (request.headers["x-#{@header_provider_field}-date"] ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ"))
  45. 152 date = datetime[0, 8]
  46. 152 content_hashed = request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] || hexdigest(request.body)
  47. 144 request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] ||= content_hashed if @apply_checksum_header
  48. 144 request.headers["x-#{@header_provider_field}-security-token"] ||= @credentials.security_token if @credentials.security_token
  49. 144 signature_headers = request.headers.each.reject do |k, _|
  50. 984 @unsigned_headers.include?(k)
  51. end
  52. # aws sigv4 needs to declare the host, regardless of protocol version
  53. 144 signature_headers << ["host", request.authority] unless request.headers.key?("host")
  54. 144 signature_headers.sort_by!(&:first)
  55. 144 signed_headers = signature_headers.map(&:first).join(";")
  56. 144 canonical_headers = signature_headers.map do |k, v|
  57. # eliminate whitespace between value fields, unless it's a quoted value
  58. 847 "#{k}:#{v.start_with?("\"") && v.end_with?("\"") ? v : v.gsub(/\s+/, " ").strip}\n"
  59. end.join
  60. # canonical request
  61. 144 creq = "#{request.verb}" \
  62. 18 "\n#{request.canonical_path}" \
  63. 18 "\n#{request.canonical_query}" \
  64. 18 "\n#{canonical_headers}" \
  65. 18 "\n#{signed_headers}" \
  66. 18 "\n#{content_hashed}"
  67. 144 credential_scope = "#{date}" \
  68. 18 "/#{@region}" \
  69. 18 "/#{@service}" \
  70. 18 "/#{lower_provider_prefix}_request"
  71. 144 algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
  72. # string to sign
  73. 144 sts = "#{algo_line}" \
  74. 18 "\n#{datetime}" \
  75. 18 "\n#{credential_scope}" \
  76. 18 "\n#{OpenSSL::Digest.new(@algorithm).hexdigest(creq)}"
  77. # signature
  78. 144 k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
  79. 144 k_region = hmac(k_date, @region)
  80. 144 k_service = hmac(k_region, @service)
  81. 144 k_credentials = hmac(k_service, "#{lower_provider_prefix}_request")
  82. 144 sig = hexhmac(k_credentials, sts)
  83. 144 credential = "#{@credentials.username}/#{credential_scope}"
  84. # apply signature
  85. 126 request.headers["authorization"] =
  86. 18 "#{algo_line} " \
  87. 18 "Credential=#{credential}, " \
  88. 18 "SignedHeaders=#{signed_headers}, " \
  89. 18 "Signature=#{sig}"
  90. end
  91. 8 private
  92. 8 def hexdigest(value)
  93. 144 digest = OpenSSL::Digest.new(@algorithm)
  94. 144 if value.respond_to?(:read)
  95. 32 if value.respond_to?(:to_path)
  96. # files, pathnames
  97. 8 digest.file(value.to_path).hexdigest
  98. else
  99. # gzipped request bodies
  100. 24 raise Error, "request body must be rewindable" unless value.respond_to?(:rewind)
  101. 24 buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  102. 2 begin
  103. 24 IO.copy_stream(value, buffer)
  104. 24 buffer.flush
  105. 24 digest.file(buffer.to_path).hexdigest
  106. ensure
  107. 24 value.rewind
  108. 24 buffer.close
  109. 24 buffer.unlink
  110. end
  111. end
  112. else
  113. # error on endless generators
  114. 112 raise Error, "hexdigest for endless enumerators is not supported" if value.unbounded_body?
  115. 104 mb_buffer = value.each.with_object("".b) do |chunk, b|
  116. 56 b << chunk
  117. 56 break if b.bytesize >= 1024 * 1024
  118. end
  119. 104 digest.hexdigest(mb_buffer)
  120. end
  121. end
  122. 8 def hmac(key, value)
  123. 576 OpenSSL::HMAC.digest(OpenSSL::Digest.new(@algorithm), key, value)
  124. end
  125. 8 def hexhmac(key, value)
  126. 144 OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new(@algorithm), key, value)
  127. end
  128. end
  129. 8 class << self
  130. 8 def load_dependencies(*)
  131. 152 require "set"
  132. 152 require "digest/sha2"
  133. end
  134. 8 def configure(klass)
  135. 152 klass.plugin(:expect)
  136. end
  137. end
  138. # adds support for the following options:
  139. #
  140. # :sigv4_signer :: instance of HTTPX::Plugins::AWSSigV4 used to sign requests.
  141. 8 module OptionsMethods
  142. 8 def option_sigv4_signer(value)
  143. 320 value.is_a?(Signer) ? value : Signer.new(value)
  144. end
  145. end
  146. 8 module InstanceMethods
  147. 8 def aws_sigv4_authentication(**options)
  148. 152 with(sigv4_signer: Signer.new(**options))
  149. end
  150. 8 def build_request(*)
  151. 152 request = super
  152. 152 return request if request.headers.key?("authorization")
  153. 152 signer = request.options.sigv4_signer
  154. 152 return request unless signer
  155. 152 signer.sign!(request)
  156. 144 request
  157. end
  158. end
  159. 8 module RequestMethods
  160. 8 def canonical_path
  161. 144 path = uri.path.dup
  162. 144 path << "/" if path.empty?
  163. 176 path.gsub(%r{[^/]+}) { |part| CGI.escape(part.encode("UTF-8")).gsub("+", "%20").gsub("%7E", "~") }
  164. end
  165. 8 def canonical_query
  166. 176 params = query.split("&")
  167. # params = params.map { |p| p.match(/=/) ? p : p + '=' }
  168. # From: https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html#create-canonical-request
  169. # Sort the parameter names by character code point in ascending order.
  170. # Parameters with duplicate names should be sorted by value.
  171. #
  172. # Default sort <=> in JRuby will swap members
  173. # occasionally when <=> is 0 (considered still sorted), but this
  174. # causes our normalized query string to not match the sent querystring.
  175. # When names match, we then sort by their values. When values also
  176. # match then we sort by their original order
  177. 176 params.each.with_index.sort do |a, b|
  178. 64 a, a_offset = a
  179. 64 b, b_offset = b
  180. 64 a_name, a_value = a.split("=", 2)
  181. 64 b_name, b_value = b.split("=", 2)
  182. 64 if a_name == b_name
  183. 32 if a_value == b_value
  184. 16 a_offset <=> b_offset
  185. else
  186. 16 a_value <=> b_value
  187. end
  188. else
  189. 32 a_name <=> b_name
  190. end
  191. end.map(&:first).join("&")
  192. end
  193. end
  194. end
  195. 8 register_plugin :aws_sigv4, AWSSigV4
  196. end
  197. end

lib/httpx/plugins/basic_auth.rb

100.0% lines covered

12 relevant lines. 12 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds helper methods to implement HTTP Basic Auth (https://datatracker.ietf.org/doc/html/rfc7617)
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Auth#basic-auth
  8. #
  9. 8 module BasicAuth
  10. 8 class << self
  11. 8 def load_dependencies(_klass)
  12. 112 require_relative "auth/basic"
  13. end
  14. 8 def configure(klass)
  15. 112 klass.plugin(:auth)
  16. end
  17. end
  18. 8 module InstanceMethods
  19. 8 def basic_auth(user, password)
  20. 128 authorization(Authentication::Basic.new(user, password).authenticate)
  21. end
  22. end
  23. end
  24. 8 register_plugin :basic_auth, BasicAuth
  25. end
  26. end

lib/httpx/plugins/brotli.rb

100.0% lines covered

25 relevant lines. 25 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. 6 module Brotli
  5. 6 class Deflater < Transcoder::Deflater
  6. 6 def deflate(chunk)
  7. 24 return unless chunk
  8. 12 ::Brotli.deflate(chunk)
  9. end
  10. end
  11. 6 module RequestBodyClassMethods
  12. 6 def initialize_deflater_body(body, encoding)
  13. 24 return Brotli.encode(body) if encoding == "br"
  14. 12 super
  15. end
  16. end
  17. 6 module ResponseBodyClassMethods
  18. 6 def initialize_inflater_by_encoding(encoding, response, **kwargs)
  19. 24 return Brotli.decode(response, **kwargs) if encoding == "br"
  20. 12 super
  21. end
  22. end
  23. 6 module_function
  24. 6 def load_dependencies(*)
  25. 24 require "brotli"
  26. end
  27. 6 def self.extra_options(options)
  28. 24 options.merge(supported_compression_formats: %w[br] + options.supported_compression_formats)
  29. end
  30. 6 def encode(body)
  31. 12 Deflater.new(body)
  32. end
  33. 6 def decode(_response, **)
  34. 12 ::Brotli.method(:inflate)
  35. end
  36. end
  37. 6 register_plugin :brotli, Brotli
  38. end
  39. end

lib/httpx/plugins/callbacks.rb

100.0% lines covered

48 relevant lines. 48 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Plugins
  4. #
  5. # This plugin adds suppoort for callbacks around the request/response lifecycle.
  6. #
  7. # https://gitlab.com/os85/httpx/-/wikis/Events
  8. #
  9. 27 module Callbacks
  10. # connection closed user-space errors happen after errors can be surfaced to requests,
  11. # so they need to pierce through the scheduler, which is only possible by simulating an
  12. # interrupt.
  13. 27 class CallbackError < Exception; end # rubocop:disable Lint/InheritException
  14. 27 module InstanceMethods
  15. 27 include HTTPX::Callbacks
  16. 27 %i[
  17. connection_opened connection_closed
  18. request_error
  19. request_started request_body_chunk request_completed
  20. response_started response_body_chunk response_completed
  21. ].each do |meth|
  22. 243 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  23. 9 def on_#{meth}(&blk) # def on_connection_opened(&blk)
  24. 9 on(:#{meth}, &blk) # on(:connection_opened, &blk)
  25. self # self
  26. end # end
  27. MOD
  28. end
  29. 27 private
  30. 27 def do_init_connection(connection, selector)
  31. 209 super
  32. 209 connection.on(:open) do
  33. 195 next unless connection.current_session == self
  34. 195 emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
  35. end
  36. 209 connection.on(:close) do
  37. 211 next unless connection.current_session == self
  38. 211 emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
  39. end
  40. 209 connection
  41. end
  42. 27 def set_request_callbacks(request)
  43. 211 super
  44. 211 request.on(:headers) do
  45. 179 emit_or_callback_error(:request_started, request)
  46. end
  47. 211 request.on(:body_chunk) do |chunk|
  48. 16 emit_or_callback_error(:request_body_chunk, request, chunk)
  49. end
  50. 211 request.on(:done) do
  51. 163 emit_or_callback_error(:request_completed, request)
  52. end
  53. 211 request.on(:response_started) do |res|
  54. 163 if res.is_a?(Response)
  55. 147 emit_or_callback_error(:response_started, request, res)
  56. 131 res.on(:chunk_received) do |chunk|
  57. 161 emit_or_callback_error(:response_body_chunk, request, res, chunk)
  58. end
  59. else
  60. 16 emit_or_callback_error(:request_error, request, res.error)
  61. end
  62. end
  63. 211 request.on(:response) do |res|
  64. 131 emit_or_callback_error(:response_completed, request, res)
  65. end
  66. end
  67. 27 def emit_or_callback_error(*args)
  68. 1203 emit(*args)
  69. rescue StandardError => e
  70. 112 ex = CallbackError.new(e.message)
  71. 112 ex.set_backtrace(e.backtrace)
  72. 112 raise ex
  73. end
  74. 27 def receive_requests(*)
  75. 211 super
  76. rescue CallbackError => e
  77. 104 raise e.cause
  78. end
  79. 27 def close(*)
  80. 209 super
  81. rescue CallbackError => e
  82. 8 raise e.cause
  83. end
  84. end
  85. end
  86. 27 register_plugin :callbacks, Callbacks
  87. end
  88. end

lib/httpx/plugins/circuit_breaker.rb

100.0% lines covered

66 relevant lines. 66 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin implements a circuit breaker around connection errors.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Circuit-Breaker
  8. #
  9. 8 module CircuitBreaker
  10. 8 using URIExtensions
  11. 8 def self.load_dependencies(*)
  12. 56 require_relative "circuit_breaker/circuit"
  13. 56 require_relative "circuit_breaker/circuit_store"
  14. end
  15. 8 def self.extra_options(options)
  16. 56 options.merge(
  17. circuit_breaker_max_attempts: 3,
  18. circuit_breaker_reset_attempts_in: 60,
  19. circuit_breaker_break_in: 60,
  20. circuit_breaker_half_open_drip_rate: 1
  21. )
  22. end
  23. 8 module InstanceMethods
  24. 8 include HTTPX::Callbacks
  25. 8 def initialize(*)
  26. 56 super
  27. 56 @circuit_store = CircuitStore.new(@options)
  28. end
  29. 8 %i[circuit_open].each do |meth|
  30. 8 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  31. 1 def on_#{meth}(&blk) # def on_circuit_open(&blk)
  32. 1 on(:#{meth}, &blk) # on(:circuit_open, &blk)
  33. self # self
  34. end # end
  35. MOD
  36. end
  37. 8 private
  38. 8 def send_requests(*requests)
  39. # @type var short_circuit_responses: Array[response]
  40. 224 short_circuit_responses = []
  41. # run all requests through the circuit breaker, see if the circuit is
  42. # open for any of them.
  43. 224 real_requests = requests.each_with_index.with_object([]) do |(req, idx), real_reqs|
  44. 224 short_circuit_response = @circuit_store.try_respond(req)
  45. 224 if short_circuit_response.nil?
  46. 176 real_reqs << req
  47. 176 next
  48. end
  49. 42 short_circuit_responses[idx] = short_circuit_response
  50. end
  51. # run requests for the remainder
  52. 224 unless real_requests.empty?
  53. 176 responses = super(*real_requests)
  54. 176 real_requests.each_with_index do |request, idx|
  55. 154 short_circuit_responses[requests.index(request)] = responses[idx]
  56. end
  57. end
  58. 224 short_circuit_responses
  59. end
  60. 8 def set_request_callbacks(request)
  61. 224 super
  62. 224 request.on(:response) do |response|
  63. 176 emit(:circuit_open, request) if try_circuit_open(request, response)
  64. end
  65. end
  66. 8 def try_circuit_open(request, response)
  67. 176 if response.is_a?(ErrorResponse)
  68. 112 case response.error
  69. when RequestTimeoutError
  70. 80 @circuit_store.try_open(request.uri, response)
  71. else
  72. 48 @circuit_store.try_open(request.origin, response)
  73. end
  74. 48 elsif (break_on = request.options.circuit_breaker_break_on) && break_on.call(response)
  75. 16 @circuit_store.try_open(request.uri, response)
  76. else
  77. 32 @circuit_store.try_close(request.uri)
  78. 12 nil
  79. end
  80. end
  81. end
  82. # adds support for the following options:
  83. #
  84. # :circuit_breaker_max_attempts :: the number of attempts the circuit allows, before it is opened (defaults to <tt>3</tt>).
  85. # :circuit_breaker_reset_attempts_in :: the time a circuit stays open at most, before it resets (defaults to <tt>60</tt>).
  86. # :circuit_breaker_break_on :: callable defining an alternative rule for a response to break
  87. # (i.e. <tt>->(res) { res.status == 429 } </tt>)
  88. # :circuit_breaker_break_in :: the time that must elapse before an open circuit can transit to the half-open state
  89. # (defaults to <tt><60</tt>).
  90. # :circuit_breaker_half_open_drip_rate :: the rate of requests a circuit allows to be performed when in an half-open state
  91. # (defaults to <tt>1</tt>).
  92. 8 module OptionsMethods
  93. 8 def option_circuit_breaker_max_attempts(value)
  94. 112 attempts = Integer(value)
  95. 112 raise TypeError, ":circuit_breaker_max_attempts must be positive" unless attempts.positive?
  96. 112 attempts
  97. end
  98. 8 def option_circuit_breaker_reset_attempts_in(value)
  99. 64 timeout = Float(value)
  100. 64 raise TypeError, ":circuit_breaker_reset_attempts_in must be positive" unless timeout.positive?
  101. 64 timeout
  102. end
  103. 8 def option_circuit_breaker_break_in(value)
  104. 88 timeout = Float(value)
  105. 88 raise TypeError, ":circuit_breaker_break_in must be positive" unless timeout.positive?
  106. 88 timeout
  107. end
  108. 8 def option_circuit_breaker_half_open_drip_rate(value)
  109. 88 ratio = Float(value)
  110. 88 raise TypeError, ":circuit_breaker_half_open_drip_rate must be a number between 0 and 1" unless (0..1).cover?(ratio)
  111. 88 ratio
  112. end
  113. 8 def option_circuit_breaker_break_on(value)
  114. 16 raise TypeError, ":circuit_breaker_break_on must be called with the response" unless value.respond_to?(:call)
  115. 16 value
  116. end
  117. end
  118. end
  119. 8 register_plugin :circuit_breaker, CircuitBreaker
  120. end
  121. end

lib/httpx/plugins/circuit_breaker/circuit.rb

100.0% lines covered

47 relevant lines. 47 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins::CircuitBreaker
  4. #
  5. # A circuit is assigned to a given absoolute url or origin.
  6. #
  7. # It sets +max_attempts+, the number of attempts the circuit allows, before it is opened.
  8. # It sets +reset_attempts_in+, the time a circuit stays open at most, before it resets.
  9. # It sets +break_in+, the time that must elapse before an open circuit can transit to the half-open state.
  10. # It sets +circuit_breaker_half_open_drip_rate+, the rate of requests a circuit allows to be performed when in an half-open state.
  11. #
  12. 8 class Circuit
  13. 8 def initialize(max_attempts, reset_attempts_in, break_in, circuit_breaker_half_open_drip_rate)
  14. 56 @max_attempts = max_attempts
  15. 56 @reset_attempts_in = reset_attempts_in
  16. 56 @break_in = break_in
  17. 56 @circuit_breaker_half_open_drip_rate = circuit_breaker_half_open_drip_rate
  18. 56 @attempts = 0
  19. 56 total_real_attempts = @max_attempts * @circuit_breaker_half_open_drip_rate
  20. 56 @drip_factor = (@max_attempts / total_real_attempts).round
  21. 56 @state = :closed
  22. end
  23. 8 def respond
  24. 224 try_close
  25. 196 case @state
  26. when :closed
  27. 51 nil
  28. when :half_open
  29. 49 @attempts += 1
  30. # do real requests while drip rate valid
  31. 56 if (@real_attempts % @drip_factor).zero?
  32. 35 @real_attempts += 1
  33. 35 return
  34. end
  35. 16 @response
  36. when :open
  37. 32 @response
  38. end
  39. end
  40. 8 def try_open(response)
  41. 126 case @state
  42. when :closed
  43. 120 now = Utils.now
  44. 120 if @attempts.positive?
  45. # reset if error happened long ago
  46. 48 @attempts = 0 if now - @attempted_at > @reset_attempts_in
  47. else
  48. 72 @attempted_at = now
  49. end
  50. 105 @attempts += 1
  51. 120 return unless @attempts >= @max_attempts
  52. 64 @state = :open
  53. 64 @opened_at = now
  54. 64 @response = response
  55. when :half_open
  56. # open immediately
  57. 24 @state = :open
  58. 24 @attempted_at = @opened_at = Utils.now
  59. 24 @response = response
  60. end
  61. end
  62. 8 def try_close
  63. 224 case @state
  64. when :closed
  65. 51 nil
  66. when :half_open
  67. # do not close circuit unless attempts exhausted
  68. 48 return unless @attempts >= @max_attempts
  69. # reset!
  70. 16 @attempts = 0
  71. 16 @opened_at = @attempted_at = @response = nil
  72. 16 @state = :closed
  73. when :open
  74. 72 if Utils.elapsed_time(@opened_at) > @break_in
  75. 40 @state = :half_open
  76. 40 @attempts = 0
  77. 40 @real_attempts = 0
  78. end
  79. end
  80. end
  81. end
  82. end
  83. end

lib/httpx/plugins/circuit_breaker/circuit_store.rb

100.0% lines covered

23 relevant lines. 23 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX::Plugins::CircuitBreaker
  3. 8 using HTTPX::URIExtensions
  4. 8 class CircuitStore
  5. 8 def initialize(options)
  6. 56 @circuits = Hash.new do |h, k|
  7. 49 h[k] = Circuit.new(
  8. options.circuit_breaker_max_attempts,
  9. options.circuit_breaker_reset_attempts_in,
  10. options.circuit_breaker_break_in,
  11. options.circuit_breaker_half_open_drip_rate
  12. )
  13. end
  14. 56 @circuits_mutex = Thread::Mutex.new
  15. end
  16. 8 def try_open(uri, response)
  17. 288 circuit = @circuits_mutex.synchronize { get_circuit_for_uri(uri) }
  18. 144 circuit.try_open(response)
  19. end
  20. 8 def try_close(uri)
  21. 32 circuit = @circuits_mutex.synchronize do
  22. 32 return unless @circuits.key?(uri.origin) || @circuits.key?(uri.to_s)
  23. 32 get_circuit_for_uri(uri)
  24. end
  25. 32 circuit.try_close
  26. end
  27. # if circuit is open, it'll respond with the stored response.
  28. # if not, nil.
  29. 8 def try_respond(request)
  30. 448 circuit = @circuits_mutex.synchronize { get_circuit_for_uri(request.uri) }
  31. 224 circuit.respond
  32. end
  33. 8 private
  34. 8 def get_circuit_for_uri(uri)
  35. 400 if uri.respond_to?(:origin) && @circuits.key?(uri.origin)
  36. 288 @circuits[uri.origin]
  37. else
  38. 112 @circuits[uri.to_s]
  39. end
  40. end
  41. end
  42. end

lib/httpx/plugins/content_digest.rb

100.0% lines covered

102 relevant lines. 102 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds `Content-Digest` headers to requests
  6. # and can validate these headers on responses
  7. #
  8. # https://datatracker.ietf.org/doc/html/rfc9530
  9. #
  10. 8 module ContentDigest
  11. 8 class Error < HTTPX::Error; end
  12. # Error raised on response "content-digest" header validation.
  13. 8 class ValidationError < Error
  14. 8 attr_reader :response
  15. 8 def initialize(message, response)
  16. 48 super(message)
  17. 48 @response = response
  18. end
  19. end
  20. 8 class MissingContentDigestError < ValidationError; end
  21. 8 class InvalidContentDigestError < ValidationError; end
  22. 2 SUPPORTED_ALGORITHMS = {
  23. 6 "sha-256" => OpenSSL::Digest::SHA256,
  24. "sha-512" => OpenSSL::Digest::SHA512,
  25. }.freeze
  26. 8 class << self
  27. 8 def extra_options(options)
  28. 208 options.merge(encode_content_digest: true, validate_content_digest: false, content_digest_algorithm: "sha-256")
  29. end
  30. end
  31. # add support for the following options:
  32. #
  33. # :content_digest_algorithm :: the digest algorithm to use. Currently supports `sha-256` and `sha-512`. (defaults to `sha-256`)
  34. # :encode_content_digest :: whether a <tt>Content-Digest</tt> header should be computed for the request;
  35. # can also be a callable object (i.e. <tt>->(req) { ... }</tt>, defaults to <tt>true</tt>)
  36. # :validate_content_digest :: whether a <tt>Content-Digest</tt> header in the response should be validated;
  37. # can also be a callable object (i.e. <tt>->(res) { ... }</tt>, defaults to <tt>false</tt>)
  38. 8 module OptionsMethods
  39. 8 def option_content_digest_algorithm(value)
  40. 216 raise TypeError, ":content_digest_algorithm must be one of 'sha-256', 'sha-512'" unless SUPPORTED_ALGORITHMS.key?(value)
  41. 216 value
  42. end
  43. 8 def option_encode_content_digest(value)
  44. 208 value
  45. end
  46. 8 def option_validate_content_digest(value)
  47. 144 value
  48. end
  49. end
  50. 8 module ResponseBodyMethods
  51. 8 attr_reader :content_digest_buffer
  52. 8 def initialize(response, options)
  53. 176 super
  54. 176 return unless response.headers.key?("content-digest")
  55. 128 should_validate = options.validate_content_digest
  56. 128 should_validate = should_validate.call(response) if should_validate.respond_to?(:call)
  57. 128 return unless should_validate
  58. 112 @content_digest_buffer = Response::Buffer.new(
  59. threshold_size: @options.body_threshold_size,
  60. bytesize: @length,
  61. encoding: @encoding
  62. )
  63. end
  64. 8 def write(chunk)
  65. 291 @content_digest_buffer.write(chunk) if @content_digest_buffer
  66. 291 super
  67. end
  68. 8 def close
  69. 112 if @content_digest_buffer
  70. 112 @content_digest_buffer.close
  71. 112 @content_digest_buffer = nil
  72. end
  73. 112 super
  74. end
  75. end
  76. 8 module InstanceMethods
  77. 8 def build_request(*)
  78. 224 request = super
  79. 224 return request if request.empty?
  80. 48 return request if request.headers.key?("content-digest")
  81. 48 perform_encoding = @options.encode_content_digest
  82. 48 perform_encoding = perform_encoding.call(request) if perform_encoding.respond_to?(:call)
  83. 48 return request unless perform_encoding
  84. 40 digest = base64digest(request.body)
  85. 40 request.headers.add("content-digest", "#{@options.content_digest_algorithm}=:#{digest}:")
  86. 40 request
  87. end
  88. 8 private
  89. 8 def fetch_response(request, _, _)
  90. 603 response = super
  91. 603 return response unless response.is_a?(Response)
  92. 176 perform_validation = @options.validate_content_digest
  93. 176 perform_validation = perform_validation.call(response) if perform_validation.respond_to?(:call)
  94. 176 validate_content_digest(response) if perform_validation
  95. 128 response
  96. rescue ValidationError => e
  97. 48 ErrorResponse.new(request, e)
  98. end
  99. 8 def validate_content_digest(response)
  100. 128 content_digest_header = response.headers["content-digest"]
  101. 128 raise MissingContentDigestError.new("response is missing a `content-digest` header", response) unless content_digest_header
  102. 112 digests = extract_content_digests(content_digest_header)
  103. 112 included_algorithms = SUPPORTED_ALGORITHMS.keys & digests.keys
  104. 112 raise MissingContentDigestError.new("unsupported algorithms: #{digests.keys.join(", ")}", response) if included_algorithms.empty?
  105. 112 content_buffer = response.body.content_digest_buffer
  106. 112 included_algorithms.each do |algorithm|
  107. 112 digest = SUPPORTED_ALGORITHMS.fetch(algorithm).new
  108. 112 digest_received = digests[algorithm]
  109. 14 digest_computed =
  110. 111 if content_buffer.respond_to?(:to_path)
  111. 16 content_buffer.flush
  112. 16 digest.file(content_buffer.to_path).base64digest
  113. else
  114. 96 digest.base64digest(content_buffer.to_s)
  115. end
  116. 4 raise InvalidContentDigestError.new("#{algorithm} digest does not match content",
  117. 111 response) unless digest_received == digest_computed
  118. end
  119. end
  120. 8 def extract_content_digests(header)
  121. 112 header.split(",").to_h do |entry|
  122. 128 algorithm, digest = entry.split("=", 2)
  123. 128 raise Error, "#{entry} is an invalid digest format" unless algorithm && digest
  124. 128 [algorithm, digest.byteslice(1..-2)]
  125. end
  126. end
  127. 8 def base64digest(body)
  128. 40 digest = SUPPORTED_ALGORITHMS.fetch(@options.content_digest_algorithm).new
  129. 40 if body.respond_to?(:read)
  130. 32 if body.respond_to?(:to_path)
  131. 8 digest.file(body.to_path).base64digest
  132. else
  133. 24 raise ContentDigestError, "request body must be rewindable" unless body.respond_to?(:rewind)
  134. 24 buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  135. 2 begin
  136. 24 IO.copy_stream(body, buffer)
  137. 24 buffer.flush
  138. 24 digest.file(buffer.to_path).base64digest
  139. ensure
  140. 24 body.rewind
  141. 24 buffer.close
  142. 24 buffer.unlink
  143. end
  144. end
  145. else
  146. 8 raise ContentDigestError, "base64digest for endless enumerators is not supported" if body.unbounded_body?
  147. 8 buffer = "".b
  148. 16 body.each { |chunk| buffer << chunk }
  149. 8 digest.base64digest(buffer)
  150. end
  151. end
  152. end
  153. end
  154. 8 register_plugin :content_digest, ContentDigest
  155. end
  156. end

lib/httpx/plugins/cookies.rb

100.0% lines covered

52 relevant lines. 52 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "forwardable"
  3. 8 module HTTPX
  4. 8 module Plugins
  5. #
  6. # This plugin implements a persistent cookie jar for the duration of a session.
  7. #
  8. # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
  9. #
  10. # https://gitlab.com/os85/httpx/wikis/Cookies
  11. #
  12. 8 module Cookies
  13. 8 def self.load_dependencies(*)
  14. 144 require "httpx/plugins/cookies/jar"
  15. 144 require "httpx/plugins/cookies/cookie"
  16. 144 require "httpx/plugins/cookies/set_cookie_parser"
  17. end
  18. 8 module InstanceMethods
  19. 8 extend Forwardable
  20. 8 def_delegator :@options, :cookies
  21. 8 def initialize(options = {}, &blk)
  22. 288 super({ cookies: Jar.new }.merge(options), &blk)
  23. end
  24. 8 def wrap
  25. 16 return super unless block_given?
  26. 16 super do |session|
  27. 16 old_cookies_jar = @options.cookies.dup
  28. 1 begin
  29. 16 yield session
  30. ensure
  31. 16 @options = @options.merge(cookies: old_cookies_jar)
  32. end
  33. end
  34. end
  35. 8 def build_request(*)
  36. 320 request = super
  37. 320 request.headers.set_cookie(request.options.cookies[request.uri])
  38. 320 request
  39. end
  40. 8 private
  41. 8 def set_request_callbacks(request)
  42. 320 super
  43. 320 request.on(:response) do |response|
  44. 320 next unless response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
  45. 64 log { "cookies: set-cookie is over #{Cookie::MAX_LENGTH}" } if set_cookie.bytesize > Cookie::MAX_LENGTH
  46. 64 @options.cookies.parse(set_cookie)
  47. end
  48. end
  49. end
  50. 8 module HeadersMethods
  51. 8 def set_cookie(cookies)
  52. 320 return if cookies.empty?
  53. 272 header_value = cookies.sort.join("; ")
  54. 272 add("cookie", header_value)
  55. end
  56. end
  57. # adds support for the following options:
  58. #
  59. # :cookies :: cookie jar for the session (can be a Hash, an Array, an instance of HTTPX::Plugins::Cookies::CookieJar)
  60. 8 module OptionsMethods
  61. 8 def option_headers(*)
  62. 320 value = super
  63. 320 merge_cookie_in_jar(value.delete("cookie"), @cookies) if defined?(@cookies) && value.key?("cookie")
  64. 320 value
  65. end
  66. 8 def option_cookies(value)
  67. 480 jar = value.is_a?(Jar) ? value : Jar.new(value)
  68. 480 merge_cookie_in_jar(@headers.delete("cookie"), jar) if defined?(@headers) && @headers.key?("cookie")
  69. 480 jar
  70. end
  71. 8 private
  72. 8 def merge_cookie_in_jar(cookies, jar)
  73. 16 cookies.each do |ck|
  74. 16 ck.split(/ *; */).each do |cookie|
  75. 32 name, value = cookie.split("=", 2)
  76. 32 jar.add(Cookie.new(name, value))
  77. end
  78. end
  79. end
  80. end
  81. end
  82. 8 register_plugin :cookies, Cookies
  83. end
  84. end

lib/httpx/plugins/cookies/cookie.rb

100.0% lines covered

77 relevant lines. 77 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins::Cookies
  4. # The HTTP Cookie.
  5. #
  6. # Contains the single cookie info: name, value and attributes.
  7. 8 class Cookie
  8. 8 include Comparable
  9. # Maximum number of bytes per cookie (RFC 6265 6.1 requires 4096 at
  10. # least)
  11. 8 MAX_LENGTH = 4096
  12. 8 attr_reader :domain, :path, :name, :value, :created_at
  13. 8 def path=(path)
  14. 184 path = String(path)
  15. 184 @path = path.start_with?("/") ? path : "/"
  16. end
  17. # See #domain.
  18. 8 def domain=(domain)
  19. 40 domain = String(domain)
  20. 40 if domain.start_with?(".")
  21. 16 @for_domain = true
  22. 16 domain = domain[1..-1]
  23. end
  24. 40 return if domain.empty?
  25. 40 @domain_name = DomainName.new(domain)
  26. # RFC 6265 5.3 5.
  27. 40 @for_domain = false if @domain_name.domain.nil? # a public suffix or IP address
  28. 40 @domain = @domain_name.hostname
  29. end
  30. # Compares the cookie with another. When there are many cookies with
  31. # the same name for a URL, the value of the smallest must be used.
  32. 8 def <=>(other)
  33. # RFC 6265 5.4
  34. # Precedence: 1. longer path 2. older creation
  35. 699 (@name <=> other.name).nonzero? ||
  36. 60 (other.path.length <=> @path.length).nonzero? ||
  37. 35 (@created_at <=> other.created_at).nonzero? || 0
  38. end
  39. 8 class << self
  40. 8 def new(cookie, *args)
  41. 504 return cookie if cookie.is_a?(self)
  42. 504 super
  43. end
  44. # Tests if +target_path+ is under +base_path+ as described in RFC
  45. # 6265 5.1.4. +base_path+ must be an absolute path.
  46. # +target_path+ may be empty, in which case it is treated as the
  47. # root path.
  48. #
  49. # e.g.
  50. #
  51. # path_match?('/admin/', '/admin/index') == true
  52. # path_match?('/admin/', '/Admin/index') == false
  53. # path_match?('/admin/', '/admin/') == true
  54. # path_match?('/admin/', '/admin') == false
  55. #
  56. # path_match?('/admin', '/admin') == true
  57. # path_match?('/admin', '/Admin') == false
  58. # path_match?('/admin', '/admins') == false
  59. # path_match?('/admin', '/admin/') == true
  60. # path_match?('/admin', '/admin/index') == true
  61. 8 def path_match?(base_path, target_path)
  62. 1352 base_path.start_with?("/") || (return false)
  63. # RFC 6265 5.1.4
  64. 1352 bsize = base_path.size
  65. 1352 tsize = target_path.size
  66. 1352 return bsize == 1 if tsize.zero? # treat empty target_path as "/"
  67. 1352 return false unless target_path.start_with?(base_path)
  68. 1344 return true if bsize == tsize || base_path.end_with?("/")
  69. 16 target_path[bsize] == "/"
  70. end
  71. end
  72. 8 def initialize(arg, *attrs)
  73. 504 @created_at = Time.now
  74. 504 if attrs.empty?
  75. 24 attr_hash = Hash.try_convert(arg)
  76. else
  77. 480 @name = arg
  78. 480 @value, attr_hash = attrs
  79. 480 attr_hash = Hash.try_convert(attr_hash)
  80. end
  81. 33 attr_hash.each do |key, val|
  82. 312 key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)
  83. 273 case key
  84. when :domain, :path
  85. 201 __send__(:"#{key}=", val)
  86. else
  87. 88 instance_variable_set(:"@#{key}", val)
  88. end
  89. 503 end if attr_hash
  90. 504 @path ||= "/"
  91. 504 raise ArgumentError, "name must be specified" if @name.nil?
  92. 504 @name = @name.to_s
  93. end
  94. 8 def expires
  95. 760 @expires || (@created_at && @max_age ? @created_at + @max_age : nil)
  96. end
  97. 8 def expired?(time = Time.now)
  98. 728 return false unless expires
  99. 32 expires <= time
  100. end
  101. # Returns a string for use in the Cookie header, i.e. `name=value`
  102. # or `name="value"`.
  103. 8 def cookie_value
  104. 483 "#{@name}=#{Scanner.quote(@value.to_s)}"
  105. end
  106. 8 alias_method :to_s, :cookie_value
  107. # Tests if it is OK to send this cookie to a given `uri`. A
  108. # RuntimeError is raised if the cookie's domain is unknown.
  109. 8 def valid_for_uri?(uri)
  110. 712 uri = URI(uri)
  111. # RFC 6265 5.4
  112. 712 return false if @secure && uri.scheme != "https"
  113. 704 acceptable_from_uri?(uri) && Cookie.path_match?(@path, uri.path)
  114. end
  115. 8 private
  116. # Tests if it is OK to accept this cookie if it is sent from a given
  117. # URI/URL, `uri`.
  118. 8 def acceptable_from_uri?(uri)
  119. 736 uri = URI(uri)
  120. 736 host = DomainName.new(uri.host)
  121. # RFC 6265 5.3
  122. 736 if host.hostname == @domain
  123. 16 true
  124. 719 elsif @for_domain # !host-only-flag
  125. 32 host.cookie_domain?(@domain_name)
  126. else
  127. 688 @domain.nil?
  128. end
  129. end
  130. 8 module Scanner
  131. 8 RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
  132. 8 module_function
  133. 8 def quote(s)
  134. 552 return s unless s.match(RE_BAD_CHAR)
  135. 8 "\"#{s.gsub(/([\\"])/, "\\\\\\1")}\""
  136. end
  137. end
  138. end
  139. end
  140. end

lib/httpx/plugins/cookies/jar.rb

100.0% lines covered

47 relevant lines. 47 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins::Cookies
  4. # The Cookie Jar
  5. #
  6. # It holds a bunch of cookies.
  7. 8 class Jar
  8. 8 using URIExtensions
  9. 8 include Enumerable
  10. 8 def initialize_dup(orig)
  11. 216 super
  12. 216 @cookies = orig.instance_variable_get(:@cookies).dup
  13. end
  14. 8 def initialize(cookies = nil)
  15. 536 @cookies = []
  16. 120 cookies.each do |elem|
  17. 176 cookie = case elem
  18. when Cookie
  19. 16 elem
  20. when Array
  21. 144 Cookie.new(*elem)
  22. else
  23. 16 Cookie.new(elem)
  24. end
  25. 176 @cookies << cookie
  26. 535 end if cookies
  27. end
  28. 8 def parse(set_cookie)
  29. 144 SetCookieParser.call(set_cookie) do |name, value, attrs|
  30. 208 add(Cookie.new(name, value, attrs))
  31. end
  32. end
  33. 8 def add(cookie, path = nil)
  34. 456 c = cookie.dup
  35. 456 c.path = path if path && c.path == "/"
  36. # If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
  37. # as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
  38. 864 @cookies.delete_if { |ck| ck.name == c.name && ck.domain == c.domain && ck.path == c.path }
  39. 456 @cookies << c
  40. end
  41. 8 def [](uri)
  42. 472 each(uri).sort
  43. end
  44. 8 def each(uri = nil, &blk)
  45. 1184 return enum_for(__method__, uri) unless blk
  46. 680 return @cookies.each(&blk) unless uri
  47. 472 uri = URI(uri)
  48. 472 now = Time.now
  49. 472 tpath = uri.path
  50. 472 @cookies.delete_if do |cookie|
  51. 728 if cookie.expired?(now)
  52. 16 true
  53. else
  54. 712 yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
  55. 712 false
  56. end
  57. end
  58. end
  59. 8 def merge(other)
  60. 200 cookies_dup = dup
  61. 200 other.each do |elem|
  62. 216 cookie = case elem
  63. when Cookie
  64. 200 elem
  65. when Array
  66. 8 Cookie.new(*elem)
  67. else
  68. 8 Cookie.new(elem)
  69. end
  70. 216 cookies_dup.add(cookie)
  71. end
  72. 200 cookies_dup
  73. end
  74. end
  75. end
  76. end

lib/httpx/plugins/cookies/set_cookie_parser.rb

100.0% lines covered

70 relevant lines. 70 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "strscan"
  3. 8 require "time"
  4. 8 module HTTPX
  5. 8 module Plugins::Cookies
  6. 8 module SetCookieParser
  7. # Whitespace.
  8. 8 RE_WSP = /[ \t]+/.freeze
  9. # A pattern that matches a cookie name or attribute name which may
  10. # be empty, capturing trailing whitespace.
  11. 8 RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
  12. 8 RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
  13. # A pattern that matches the comma in a (typically date) value.
  14. 8 RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
  15. 8 module_function
  16. 8 def scan_dquoted(scanner)
  17. 16 s = +""
  18. 22 until scanner.eos?
  19. 64 break if scanner.skip(/"/)
  20. 48 if scanner.skip(/\\/)
  21. 16 s << scanner.getch
  22. 31 elsif scanner.scan(/[^"\\]+/)
  23. 32 s << scanner.matched
  24. end
  25. end
  26. 16 s
  27. end
  28. 8 def scan_value(scanner, comma_as_separator = false)
  29. 440 value = +""
  30. 498 until scanner.eos?
  31. 760 if scanner.scan(/[^,;"]+/)
  32. 432 value << scanner.matched
  33. 327 elsif scanner.skip(/"/)
  34. # RFC 6265 2.2
  35. # A cookie-value may be DQUOTE'd.
  36. 16 value << scan_dquoted(scanner)
  37. 311 elsif scanner.check(/;/)
  38. 232 break
  39. 79 elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
  40. 64 break
  41. else
  42. 16 value << scanner.getch
  43. end
  44. end
  45. 440 value.rstrip!
  46. 440 value
  47. end
  48. 8 def scan_name_value(scanner, comma_as_separator = false)
  49. 440 name = scanner.scan(RE_NAME)
  50. 440 name.rstrip! if name
  51. 440 if scanner.skip(/=/)
  52. 432 value = scan_value(scanner, comma_as_separator)
  53. else
  54. 8 scan_value(scanner, comma_as_separator)
  55. 8 value = nil
  56. end
  57. 440 [name, value]
  58. end
  59. 8 def call(set_cookie)
  60. 144 scanner = StringScanner.new(set_cookie)
  61. # RFC 6265 4.1.1 & 5.2
  62. 170 until scanner.eos?
  63. 208 start = scanner.pos
  64. 208 len = nil
  65. 208 scanner.skip(RE_WSP)
  66. 208 name, value = scan_name_value(scanner, true)
  67. 208 value = nil if name.empty?
  68. 208 attrs = {}
  69. 237 until scanner.eos?
  70. 296 if scanner.skip(/,/)
  71. # The comma is used as separator for concatenating multiple
  72. # values of a header.
  73. 64 len = (scanner.pos - 1) - start
  74. 64 break
  75. 231 elsif scanner.skip(/;/)
  76. 232 scanner.skip(RE_WSP)
  77. 232 aname, avalue = scan_name_value(scanner, true)
  78. 232 next if aname.empty? || value.nil?
  79. 232 aname.downcase!
  80. 203 case aname
  81. when "expires"
  82. # RFC 6265 5.2.1
  83. 16 (avalue &&= Time.parse(avalue)) || next
  84. when "max-age"
  85. # RFC 6265 5.2.2
  86. 8 next unless /\A-?\d+\z/.match?(avalue)
  87. 8 avalue = Integer(avalue)
  88. when "domain"
  89. # RFC 6265 5.2.3
  90. # An empty value SHOULD be ignored.
  91. 24 next if avalue.nil? || avalue.empty?
  92. when "path"
  93. # RFC 6265 5.2.4
  94. # A relative path must be ignored rather than normalizing it
  95. # to "/".
  96. 176 next unless avalue.start_with?("/")
  97. when "secure", "httponly"
  98. # RFC 6265 5.2.5, 5.2.6
  99. 7 avalue = true
  100. end
  101. 203 attrs[aname] = avalue
  102. end
  103. end
  104. 208 len ||= scanner.pos - start
  105. 208 next if len > Cookie::MAX_LENGTH
  106. 208 yield(name, value, attrs) if name && !name.empty? && value
  107. end
  108. end
  109. end
  110. end
  111. end

lib/httpx/plugins/digest_auth.rb

100.0% lines covered

29 relevant lines. 29 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds helper methods to implement HTTP Digest Auth (https://datatracker.ietf.org/doc/html/rfc7616)
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
  8. #
  9. 8 module DigestAuth
  10. 8 DigestError = Class.new(Error)
  11. 8 class << self
  12. 8 def extra_options(options)
  13. 160 options.merge(max_concurrent_requests: 1)
  14. end
  15. 8 def load_dependencies(*)
  16. 160 require_relative "auth/digest"
  17. end
  18. end
  19. # adds support for the following options:
  20. #
  21. # :digest :: instance of HTTPX::Plugins::Authentication::Digest, used to authenticate requests in the session.
  22. 8 module OptionsMethods
  23. 8 def option_digest(value)
  24. 320 raise TypeError, ":digest must be a #{Authentication::Digest}" unless value.is_a?(Authentication::Digest)
  25. 320 value
  26. end
  27. end
  28. 8 module InstanceMethods
  29. 8 def digest_auth(user, password, hashed: false)
  30. 160 with(digest: Authentication::Digest.new(user, password, hashed: hashed))
  31. end
  32. 8 private
  33. 8 def send_requests(*requests)
  34. 192 requests.flat_map do |request|
  35. 192 digest = request.options.digest
  36. 192 next super(request) unless digest
  37. 320 probe_response = wrap { super(request).first }
  38. 160 return probe_response unless probe_response.is_a?(Response)
  39. 160 if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
  40. 144 request.transition(:idle)
  41. 126 request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
  42. 144 super(request)
  43. else
  44. 16 probe_response
  45. end
  46. end
  47. end
  48. end
  49. end
  50. 8 register_plugin :digest_auth, DigestAuth
  51. end
  52. end

lib/httpx/plugins/expect.rb

100.0% lines covered

56 relevant lines. 56 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin makes all HTTP/1.1 requests with a body send the "Expect: 100-continue".
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Expect#expect
  8. #
  9. 8 module Expect
  10. 8 EXPECT_TIMEOUT = 2
  11. 8 class << self
  12. 8 def no_expect_store
  13. 176 @no_expect_store ||= []
  14. end
  15. 8 def extra_options(options)
  16. 216 options.merge(expect_timeout: EXPECT_TIMEOUT)
  17. end
  18. end
  19. # adds support for the following options:
  20. #
  21. # :expect_timeout :: time (in seconds) to wait for a 100-expect response,
  22. # before retrying without the Expect header (defaults to <tt>2</tt>).
  23. # :expect_threshold_size :: min threshold (in bytes) of the request payload to enable the 100-continue negotiation on.
  24. 8 module OptionsMethods
  25. 8 def option_expect_timeout(value)
  26. 384 seconds = Float(value)
  27. 384 raise TypeError, ":expect_timeout must be positive" unless seconds.positive?
  28. 384 seconds
  29. end
  30. 8 def option_expect_threshold_size(value)
  31. 16 bytes = Integer(value)
  32. 16 raise TypeError, ":expect_threshold_size must be positive" unless bytes.positive?
  33. 16 bytes
  34. end
  35. end
  36. 8 module RequestMethods
  37. 8 def initialize(*)
  38. 248 super
  39. 248 return if @body.empty?
  40. 168 threshold = @options.expect_threshold_size
  41. 168 return if threshold && !@body.unbounded_body? && @body.bytesize < threshold
  42. 152 return if Expect.no_expect_store.include?(origin)
  43. 126 @headers["expect"] = "100-continue"
  44. end
  45. 8 def response=(response)
  46. 184 if response.is_a?(Response) &&
  47. response.status == 100 &&
  48. !@headers.key?("expect") &&
  49. 3 (@state == :body || @state == :done)
  50. # if we're past this point, this means that we just received a 100-Continue response,
  51. # but the request doesn't have the expect flag, and is already flushing (or flushed) the body.
  52. #
  53. # this means that expect was deactivated for this request too soon, i.e. response took longer.
  54. #
  55. # so we have to reactivate it again.
  56. 7 @headers["expect"] = "100-continue"
  57. 8 @informational_status = 100
  58. 8 Expect.no_expect_store.delete(origin)
  59. end
  60. 184 super
  61. end
  62. end
  63. 8 module ConnectionMethods
  64. 8 def send_request_to_parser(request)
  65. 112 super
  66. 112 return unless request.headers["expect"] == "100-continue"
  67. 80 expect_timeout = request.options.expect_timeout
  68. 80 return if expect_timeout.nil? || expect_timeout.infinite?
  69. 80 set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
  70. # expect timeout expired
  71. 16 if request.state == :expect && !request.expects?
  72. 16 Expect.no_expect_store << request.origin
  73. 16 request.headers.delete("expect")
  74. 16 consume
  75. end
  76. end
  77. end
  78. end
  79. 8 module InstanceMethods
  80. 8 def fetch_response(request, selector, options)
  81. 428 response = super
  82. 428 return unless response
  83. 112 if response.is_a?(Response) && response.status == 417 && request.headers.key?("expect")
  84. 16 response.close
  85. 16 request.headers.delete("expect")
  86. 16 request.transition(:idle)
  87. 16 send_request(request, selector, options)
  88. 14 return
  89. end
  90. 96 response
  91. end
  92. end
  93. end
  94. 8 register_plugin :expect, Expect
  95. end
  96. end

lib/httpx/plugins/follow_redirects.rb

100.0% lines covered

108 relevant lines. 108 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 15 module HTTPX
  3. 15 InsecureRedirectError = Class.new(Error)
  4. 15 module Plugins
  5. #
  6. # This plugin adds support for automatically following redirect (status 30X) responses.
  7. #
  8. # It has a default upper bound of followed redirects (see *MAX_REDIRECTS* and the *max_redirects* option),
  9. # after which it will return the last redirect response. It will **not** raise an exception.
  10. #
  11. # It doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
  12. #
  13. # It doesn't propagate authorization related headers to requests redirecting to different origins
  14. # (see *allow_auth_to_other_origins*) to override.
  15. #
  16. # It allows customization of when to redirect via the *redirect_on* callback option).
  17. #
  18. # https://gitlab.com/os85/httpx/wikis/Follow-Redirects
  19. #
  20. 15 module FollowRedirects
  21. 15 MAX_REDIRECTS = 3
  22. 15 REDIRECT_STATUS = (300..399).freeze
  23. 15 REQUEST_BODY_HEADERS = %w[transfer-encoding content-encoding content-type content-length content-language content-md5 trailer].freeze
  24. 15 using URIExtensions
  25. # adds support for the following options:
  26. #
  27. # :max_redirects :: max number of times a request will be redirected (defaults to <tt>3</tt>).
  28. # :follow_insecure_redirects :: whether redirects to an "http://" URI, when coming from an "https//", are allowed
  29. # (defaults to <tt>false</tt>).
  30. # :allow_auth_to_other_origins :: whether auth-related headers, such as "Authorization", are propagated on redirection
  31. # (defaults to <tt>false</tt>).
  32. # :redirect_on :: optional callback which receives the redirect location and can halt the redirect chain if it returns <tt>false</tt>.
  33. 15 module OptionsMethods
  34. 15 def option_max_redirects(value)
  35. 462 num = Integer(value)
  36. 462 raise TypeError, ":max_redirects must be positive" if num.negative?
  37. 462 num
  38. end
  39. 15 def option_follow_insecure_redirects(value)
  40. 24 value
  41. end
  42. 15 def option_allow_auth_to_other_origins(value)
  43. 24 value
  44. end
  45. 15 def option_redirect_on(value)
  46. 48 raise TypeError, ":redirect_on must be callable" unless value.respond_to?(:call)
  47. 48 value
  48. end
  49. end
  50. 15 module InstanceMethods
  51. # returns a session with the *max_redirects* option set to +n+
  52. 15 def max_redirects(n)
  53. 48 with(max_redirects: n.to_i)
  54. end
  55. 15 private
  56. 15 def fetch_response(request, selector, options)
  57. 4291265 redirect_request = request.redirect_request
  58. 4291265 response = super(redirect_request, selector, options)
  59. 4291265 return unless response
  60. 568 max_redirects = redirect_request.max_redirects
  61. 568 return response unless response.is_a?(Response)
  62. 552 return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
  63. 361 return response unless max_redirects.positive?
  64. 329 redirect_uri = __get_location_from_response(response)
  65. 329 if options.redirect_on
  66. 32 redirect_allowed = options.redirect_on.call(redirect_uri)
  67. 32 return response unless redirect_allowed
  68. end
  69. # build redirect request
  70. 313 request_body = redirect_request.body
  71. 313 redirect_method = "GET"
  72. 313 redirect_params = {}
  73. 313 if response.status == 305 && options.respond_to?(:proxy)
  74. 8 request_body.rewind
  75. # The requested resource MUST be accessed through the proxy given by
  76. # the Location field. The Location field gives the URI of the proxy.
  77. 8 redirect_options = options.merge(headers: redirect_request.headers,
  78. proxy: { uri: redirect_uri },
  79. max_redirects: max_redirects - 1)
  80. 7 redirect_params[:body] = request_body
  81. 8 redirect_uri = redirect_request.uri
  82. 8 options = redirect_options
  83. else
  84. 305 redirect_headers = redirect_request_headers(redirect_request.uri, redirect_uri, request.headers, options)
  85. 305 redirect_opts = Hash[options]
  86. 268 redirect_params[:max_redirects] = max_redirects - 1
  87. 305 unless request_body.empty?
  88. 24 if response.status == 307
  89. # The method and the body of the original request are reused to perform the redirected request.
  90. 8 redirect_method = redirect_request.verb
  91. 8 request_body.rewind
  92. 7 redirect_params[:body] = request_body
  93. else
  94. # redirects are **ALWAYS** GET, so remove body-related headers
  95. 16 REQUEST_BODY_HEADERS.each do |h|
  96. 112 redirect_headers.delete(h)
  97. end
  98. 14 redirect_params[:body] = nil
  99. end
  100. end
  101. 305 options = options.class.new(redirect_opts.merge(headers: redirect_headers.to_h))
  102. end
  103. 313 redirect_uri = Utils.to_uri(redirect_uri)
  104. 313 if !options.follow_insecure_redirects &&
  105. response.uri.scheme == "https" &&
  106. redirect_uri.scheme == "http"
  107. 8 error = InsecureRedirectError.new(redirect_uri.to_s)
  108. 8 error.set_backtrace(caller)
  109. 7 return ErrorResponse.new(request, error)
  110. end
  111. 305 retry_request = build_request(redirect_method, redirect_uri, redirect_params, options)
  112. 305 request.redirect_request = retry_request
  113. 305 redirect_after = response.headers["retry-after"]
  114. 305 if redirect_after
  115. # Servers send the "Retry-After" header field to indicate how long the
  116. # user agent ought to wait before making a follow-up request.
  117. # When sent with any 3xx (Redirection) response, Retry-After indicates
  118. # the minimum time that the user agent is asked to wait before issuing
  119. # the redirected request.
  120. #
  121. 31 redirect_after = Utils.parse_retry_after(redirect_after)
  122. 31 retry_start = Utils.now
  123. 31 log { "redirecting after #{redirect_after} secs..." }
  124. 31 selector.after(redirect_after) do
  125. 31 if (response = request.response)
  126. 15 response.finish!
  127. 15 retry_request.response = response
  128. # request has terminated abruptly meanwhile
  129. 15 retry_request.emit(:response, response)
  130. else
  131. 16 log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
  132. 16 send_request(retry_request, selector, options)
  133. end
  134. end
  135. else
  136. 274 send_request(retry_request, selector, options)
  137. end
  138. 116 nil
  139. end
  140. # :nodoc:
  141. 15 def redirect_request_headers(original_uri, redirect_uri, headers, options)
  142. 305 headers = headers.dup
  143. 305 return headers if options.allow_auth_to_other_origins
  144. 297 return headers unless headers.key?("authorization")
  145. 8 return headers if original_uri.origin == redirect_uri.origin
  146. 8 headers.delete("authorization")
  147. 8 headers
  148. end
  149. # :nodoc:
  150. 15 def __get_location_from_response(response)
  151. # @type var location_uri: http_uri
  152. 329 location_uri = URI(response.headers["location"])
  153. 329 location_uri = response.uri.merge(location_uri) if location_uri.relative?
  154. 329 location_uri
  155. end
  156. end
  157. 15 module RequestMethods
  158. # returns the top-most original HTTPX::Request from the redirect chain
  159. 15 attr_accessor :root_request
  160. # returns the follow-up redirect request, or itself
  161. 15 def redirect_request
  162. 4291265 @redirect_request || self
  163. end
  164. # sets the follow-up redirect request
  165. 15 def redirect_request=(req)
  166. 305 @redirect_request = req
  167. 305 req.root_request = @root_request || self
  168. 305 @response = nil
  169. end
  170. 15 def response
  171. 4293077 return super unless @redirect_request && @response.nil?
  172. 73 @redirect_request.response
  173. end
  174. 15 def max_redirects
  175. 568 @options.max_redirects || MAX_REDIRECTS
  176. end
  177. end
  178. 15 module ConnectionMethods
  179. 15 private
  180. 15 def set_request_request_timeout(request)
  181. 541 return unless request.root_request.nil?
  182. 257 super
  183. end
  184. end
  185. end
  186. 15 register_plugin :follow_redirects, FollowRedirects
  187. end
  188. end

lib/httpx/plugins/grpc.rb

100.0% lines covered

133 relevant lines. 133 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 GRPCError = Class.new(Error) do
  4. 6 attr_reader :status, :details, :metadata
  5. 6 def initialize(status, details, metadata)
  6. 24 @status = status
  7. 24 @details = details
  8. 24 @metadata = metadata
  9. 24 super("GRPC error, code=#{status}, details=#{details}, metadata=#{metadata}")
  10. end
  11. end
  12. 6 module Plugins
  13. #
  14. # This plugin adds DSL to build GRPC interfaces.
  15. #
  16. # https://gitlab.com/os85/httpx/wikis/GRPC
  17. #
  18. 6 module GRPC
  19. 6 unless String.method_defined?(:underscore)
  20. 6 module StringExtensions
  21. 6 refine String do
  22. 6 def underscore
  23. 312 s = dup # Avoid mutating the argument, as it might be frozen.
  24. 312 s.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
  25. 312 s.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
  26. 312 s.tr!("-", "_")
  27. 312 s.downcase!
  28. 312 s
  29. end
  30. end
  31. end
  32. 6 using StringExtensions
  33. end
  34. 6 DEADLINE = 60
  35. 6 MARSHAL_METHOD = :encode
  36. 6 UNMARSHAL_METHOD = :decode
  37. 6 HEADERS = {
  38. "content-type" => "application/grpc",
  39. "te" => "trailers",
  40. "accept" => "application/grpc",
  41. # metadata fits here
  42. # ex "foo-bin" => base64("bar")
  43. }.freeze
  44. 6 class << self
  45. 6 def load_dependencies(*)
  46. 138 require "stringio"
  47. 138 require "httpx/plugins/grpc/grpc_encoding"
  48. 138 require "httpx/plugins/grpc/message"
  49. 138 require "httpx/plugins/grpc/call"
  50. end
  51. 6 def configure(klass)
  52. 138 klass.plugin(:persistent)
  53. 138 klass.plugin(:stream)
  54. end
  55. 6 def extra_options(options)
  56. 138 options.merge(
  57. fallback_protocol: "h2",
  58. grpc_rpcs: {}.freeze,
  59. grpc_compression: false,
  60. grpc_deadline: DEADLINE
  61. )
  62. end
  63. end
  64. 6 module OptionsMethods
  65. 6 def option_grpc_service(value)
  66. 120 String(value)
  67. end
  68. 6 def option_grpc_compression(value)
  69. 162 case value
  70. when true, false
  71. 138 value
  72. else
  73. 24 value.to_s
  74. end
  75. end
  76. 6 def option_grpc_rpcs(value)
  77. 1116 Hash[value]
  78. end
  79. 6 def option_grpc_deadline(value)
  80. 804 raise TypeError, ":grpc_deadline must be positive" unless value.positive?
  81. 804 value
  82. end
  83. 6 def option_call_credentials(value)
  84. 18 raise TypeError, ":call_credentials must respond to #call" unless value.respond_to?(:call)
  85. 18 value
  86. end
  87. end
  88. 6 module ResponseMethods
  89. 6 attr_reader :trailing_metadata
  90. 6 def merge_headers(trailers)
  91. 114 @trailing_metadata = Hash[trailers]
  92. 114 super
  93. end
  94. end
  95. 6 module RequestBodyMethods
  96. 6 def initialize(*, **)
  97. 126 super
  98. 126 if (compression = @headers["grpc-encoding"])
  99. 12 deflater_body = self.class.initialize_deflater_body(@body, compression)
  100. 12 @body = Transcoder::GRPCEncoding.encode(deflater_body || @body, compressed: !deflater_body.nil?)
  101. else
  102. 114 @body = Transcoder::GRPCEncoding.encode(@body, compressed: false)
  103. end
  104. end
  105. end
  106. 6 module InstanceMethods
  107. 6 def with_channel_credentials(ca_path, key = nil, cert = nil, **ssl_opts)
  108. # @type var ssl_params: ::Hash[::Symbol, untyped]
  109. 72 ssl_params = {
  110. **ssl_opts,
  111. ca_file: ca_path,
  112. }
  113. 72 if key
  114. 72 key = File.read(key) if File.file?(key)
  115. 72 ssl_params[:key] = OpenSSL::PKey.read(key)
  116. end
  117. 72 if cert
  118. 72 cert = File.read(cert) if File.file?(cert)
  119. 72 ssl_params[:cert] = OpenSSL::X509::Certificate.new(cert)
  120. end
  121. 72 with(ssl: ssl_params)
  122. end
  123. 6 def rpc(rpc_name, input, output, **opts)
  124. 312 rpc_name = rpc_name.to_s
  125. 312 raise Error, "rpc #{rpc_name} already defined" if @options.grpc_rpcs.key?(rpc_name)
  126. rpc_opts = {
  127. 312 deadline: @options.grpc_deadline,
  128. }.merge(opts)
  129. 312 local_rpc_name = rpc_name.underscore
  130. 312 session_class = Class.new(self.class) do
  131. # define rpc method with ruby style name
  132. 312 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  133. def #{local_rpc_name}(input, **opts) # def grpc_action(input, **opts)
  134. rpc_execute("#{local_rpc_name}", input, **opts) # rpc_execute("grpc_action", input, **opts)
  135. end # end
  136. OUT
  137. # define rpc method with original name
  138. 312 unless local_rpc_name == rpc_name
  139. 12 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  140. def #{rpc_name}(input, **opts) # def grpcAction(input, **opts)
  141. rpc_execute("#{local_rpc_name}", input, **opts) # rpc_execute("grpc_action", input, **opts)
  142. end # end
  143. OUT
  144. end
  145. end
  146. 312 session_class.new(@options.merge(
  147. grpc_rpcs: @options.grpc_rpcs.merge(
  148. local_rpc_name => [rpc_name, input, output, rpc_opts]
  149. ).freeze
  150. ))
  151. end
  152. 6 def build_stub(origin, service: nil, compression: false)
  153. 138 scheme = @options.ssl.empty? ? "http" : "https"
  154. 138 origin = URI.parse("#{scheme}://#{origin}")
  155. 138 session = self
  156. 138 if service && service.respond_to?(:rpc_descs)
  157. # it's a grpc generic service
  158. 60 service.rpc_descs.each do |rpc_name, rpc_desc|
  159. rpc_opts = {
  160. 300 marshal_method: rpc_desc.marshal_method,
  161. unmarshal_method: rpc_desc.unmarshal_method,
  162. }
  163. 300 input = rpc_desc.input
  164. 300 input = input.type if input.respond_to?(:type)
  165. 300 output = rpc_desc.output
  166. 300 if output.respond_to?(:type)
  167. 120 rpc_opts[:stream] = true
  168. 120 output = output.type
  169. end
  170. 300 session = session.rpc(rpc_name, input, output, **rpc_opts)
  171. end
  172. 60 service = service.service_name
  173. end
  174. 138 session.with(origin: origin, grpc_service: service, grpc_compression: compression)
  175. end
  176. 6 def execute(rpc_method, input,
  177. deadline: DEADLINE,
  178. metadata: nil,
  179. **opts)
  180. 126 grpc_request = build_grpc_request(rpc_method, input, deadline: deadline, metadata: metadata, **opts)
  181. 126 response = request(grpc_request, **opts)
  182. 126 response.raise_for_status unless opts[:stream]
  183. 114 GRPC::Call.new(response)
  184. end
  185. 6 private
  186. 6 def rpc_execute(rpc_name, input, **opts)
  187. 60 rpc_name, input_enc, output_enc, rpc_opts = @options.grpc_rpcs[rpc_name]
  188. 60 exec_opts = rpc_opts.merge(opts)
  189. 60 marshal_method ||= exec_opts.delete(:marshal_method) || MARSHAL_METHOD
  190. 60 unmarshal_method ||= exec_opts.delete(:unmarshal_method) || UNMARSHAL_METHOD
  191. 60 messages = if input.respond_to?(:each)
  192. 24 Enumerator.new do |y|
  193. 24 input.each do |message|
  194. 48 y << input_enc.__send__(marshal_method, message)
  195. end
  196. end
  197. else
  198. 36 input_enc.__send__(marshal_method, input)
  199. end
  200. 60 call = execute(rpc_name, messages, **exec_opts)
  201. 60 call.decoder = output_enc.method(unmarshal_method)
  202. 60 call
  203. end
  204. 6 def build_grpc_request(rpc_method, input, deadline:, metadata: nil, **)
  205. 126 uri = @options.origin.dup
  206. 126 rpc_method = "/#{rpc_method}" unless rpc_method.start_with?("/")
  207. 126 rpc_method = "/#{@options.grpc_service}#{rpc_method}" if @options.grpc_service
  208. 126 uri.path = rpc_method
  209. 126 headers = HEADERS.merge(
  210. "grpc-accept-encoding" => ["identity", *@options.supported_compression_formats]
  211. )
  212. 126 unless deadline == Float::INFINITY
  213. # convert to milliseconds
  214. 126 deadline = (deadline * 1000.0).to_i
  215. 126 headers["grpc-timeout"] = "#{deadline}m"
  216. end
  217. 126 headers = headers.merge(metadata.transform_keys(&:to_s)) if metadata
  218. # prepare compressor
  219. 126 compression = @options.grpc_compression == true ? "gzip" : @options.grpc_compression
  220. 126 headers["grpc-encoding"] = compression if compression
  221. 126 headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
  222. 126 build_request("POST", uri, headers: headers, body: input)
  223. end
  224. end
  225. end
  226. 6 register_plugin :grpc, GRPC
  227. end
  228. end

lib/httpx/plugins/grpc/call.rb

90.91% lines covered

33 relevant lines. 30 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. 6 module GRPC
  5. # Encapsulates call information
  6. 6 class Call
  7. 6 attr_writer :decoder
  8. 6 def initialize(response)
  9. 114 @response = response
  10. 156 @decoder = ->(z) { z }
  11. 114 @consumed = false
  12. 114 @grpc_response = nil
  13. end
  14. 6 def inspect
  15. "#{self.class}(#{grpc_response})"
  16. end
  17. 6 def to_s
  18. 66 grpc_response.to_s
  19. end
  20. 6 def metadata
  21. response.headers
  22. end
  23. 6 def trailing_metadata
  24. 72 return unless @consumed
  25. 48 @response.trailing_metadata
  26. end
  27. 6 private
  28. 6 def grpc_response
  29. 186 @grpc_response ||= if @response.respond_to?(:each)
  30. 24 Enumerator.new do |y|
  31. 24 Message.stream(@response).each do |message|
  32. 48 y << @decoder.call(message)
  33. end
  34. 24 @consumed = true
  35. end
  36. else
  37. 90 @consumed = true
  38. 90 @decoder.call(Message.unary(@response))
  39. end
  40. end
  41. 6 def respond_to_missing?(meth, *args, &blk)
  42. 24 grpc_response.respond_to?(meth, *args) || super
  43. end
  44. 6 def method_missing(meth, *args, &blk)
  45. 48 return grpc_response.__send__(meth, *args, &blk) if grpc_response.respond_to?(meth)
  46. super
  47. end
  48. end
  49. end
  50. end
  51. end

lib/httpx/plugins/grpc/grpc_encoding.rb

97.87% lines covered

47 relevant lines. 46 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Transcoder
  4. 6 module GRPCEncoding
  5. 6 class Deflater
  6. 6 extend Forwardable
  7. 6 attr_reader :content_type
  8. 6 def initialize(body, compressed:)
  9. 126 @content_type = body.content_type
  10. 126 @body = BodyReader.new(body)
  11. 126 @compressed = compressed
  12. end
  13. 6 def bytesize
  14. 402 return @body.bytesize if @body.respond_to?(:bytesize)
  15. Float::INFINITY
  16. end
  17. 6 def read(length = nil, outbuf = nil)
  18. 264 buf = @body.read(length, outbuf)
  19. 252 return unless buf
  20. 138 compressed_flag = @compressed ? 1 : 0
  21. 138 buf = outbuf if outbuf
  22. 138 buf = buf.b if buf.frozen?
  23. 138 buf.prepend([compressed_flag, buf.bytesize].pack("CL>"))
  24. 138 buf
  25. end
  26. end
  27. 6 class Inflater
  28. 6 def initialize(response)
  29. 90 @response = response
  30. 90 @grpc_encodings = nil
  31. end
  32. 6 def call(message, &blk)
  33. 114 data = "".b
  34. 114 until message.empty?
  35. 114 compressed, size = message.unpack("CL>")
  36. 114 encoded_data = message.byteslice(5..size + 5 - 1)
  37. 114 if compressed == 1
  38. 12 grpc_encodings.reverse_each do |encoding|
  39. 12 decoder = @response.body.class.initialize_inflater_by_encoding(encoding, @response, bytesize: encoded_data.bytesize)
  40. 12 encoded_data = decoder.call(encoded_data)
  41. 12 blk.call(encoded_data) if blk
  42. 12 data << encoded_data
  43. end
  44. else
  45. 102 blk.call(encoded_data) if blk
  46. 102 data << encoded_data
  47. end
  48. 114 message = message.byteslice((size + 5)..-1)
  49. end
  50. 114 data
  51. end
  52. 6 private
  53. 6 def grpc_encodings
  54. 12 @grpc_encodings ||= @response.headers.get("grpc-encoding")
  55. end
  56. end
  57. 6 def self.encode(*args, **kwargs)
  58. 126 Deflater.new(*args, **kwargs)
  59. end
  60. 6 def self.decode(response)
  61. 90 Inflater.new(response)
  62. end
  63. end
  64. end
  65. end

lib/httpx/plugins/grpc/message.rb

95.83% lines covered

24 relevant lines. 23 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. 6 module GRPC
  5. # Encoding module for GRPC responses
  6. #
  7. # Can encode and decode grpc messages.
  8. 6 module Message
  9. 6 module_function
  10. # decodes a unary grpc response
  11. 6 def unary(response)
  12. 90 verify_status(response)
  13. 66 decoder = Transcoder::GRPCEncoding.decode(response)
  14. 66 decoder.call(response.to_s)
  15. end
  16. # lazy decodes a grpc stream response
  17. 6 def stream(response, &block)
  18. 48 return enum_for(__method__, response) unless block
  19. 24 decoder = Transcoder::GRPCEncoding.decode(response)
  20. 24 response.each do |frame|
  21. 48 decoder.call(frame, &block)
  22. end
  23. 24 verify_status(response)
  24. end
  25. 6 def cancel(request)
  26. request.emit(:refuse, :client_cancellation)
  27. end
  28. # interprets the grpc call trailing metadata, and raises an
  29. # exception in case of error code
  30. 6 def verify_status(response)
  31. # return standard errors if need be
  32. 114 response.raise_for_status
  33. 114 status = Integer(response.headers["grpc-status"])
  34. 114 message = response.headers["grpc-message"]
  35. 114 return if status.zero?
  36. 24 response.close
  37. 24 raise GRPCError.new(status, message, response.trailing_metadata)
  38. end
  39. end
  40. end
  41. end
  42. end

lib/httpx/plugins/h2c.rb

94.92% lines covered

59 relevant lines. 56 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
  6. # (https://datatracker.ietf.org/doc/html/rfc7540#section-3.2)
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2c
  9. #
  10. 8 module H2C
  11. 8 VALID_H2C_VERBS = %w[GET OPTIONS HEAD].freeze
  12. 8 class << self
  13. 8 def load_dependencies(klass)
  14. 16 klass.plugin(:upgrade)
  15. end
  16. 8 def call(connection, request, response)
  17. 16 connection.upgrade_to_h2c(request, response)
  18. end
  19. 8 def extra_options(options)
  20. 16 options.merge(max_concurrent_requests: 1, upgrade_handlers: options.upgrade_handlers.merge("h2c" => self))
  21. end
  22. end
  23. 8 class H2CParser < Connection::HTTP2
  24. 8 def upgrade(request, response)
  25. # skip checks, it is assumed that this is the first
  26. # request in the connection
  27. 16 stream = @connection.upgrade
  28. # on_settings
  29. 16 handle_stream(stream, request)
  30. 14 @streams[request] = stream
  31. # clean up data left behind in the buffer, if the server started
  32. # sending frames
  33. 16 data = response.read
  34. 16 @connection << data
  35. end
  36. end
  37. 8 module RequestMethods
  38. 8 def valid_h2c_verb?
  39. 16 VALID_H2C_VERBS.include?(@verb)
  40. end
  41. end
  42. 8 module ConnectionMethods
  43. 8 using URIExtensions
  44. 8 def initialize(*)
  45. 16 super
  46. 16 @h2c_handshake = false
  47. end
  48. 8 def send(request)
  49. 56 return super if @h2c_handshake
  50. 16 return super unless request.valid_h2c_verb? && request.scheme == "http"
  51. 16 return super if @upgrade_protocol == "h2c"
  52. 16 @h2c_handshake = true
  53. # build upgrade request
  54. 16 request.headers.add("connection", "upgrade")
  55. 16 request.headers.add("connection", "http2-settings")
  56. 14 request.headers["upgrade"] = "h2c"
  57. 14 request.headers["http2-settings"] = ::HTTP2::Client.settings_header(request.options.http2_settings)
  58. 16 super
  59. end
  60. 8 def upgrade_to_h2c(request, response)
  61. 16 prev_parser = @parser
  62. 16 if prev_parser
  63. 16 prev_parser.reset
  64. 14 @inflight -= prev_parser.requests.size
  65. end
  66. 16 @parser = H2CParser.new(@write_buffer, @options)
  67. 16 set_parser_callbacks(@parser)
  68. 14 @inflight += 1
  69. 16 @parser.upgrade(request, response)
  70. 16 @upgrade_protocol = "h2c"
  71. 16 prev_parser.requests.each do |req|
  72. 16 req.transition(:idle)
  73. 16 send(req)
  74. end
  75. end
  76. 8 private
  77. 8 def send_request_to_parser(request)
  78. 56 super
  79. 56 return unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
  80. 16 max_concurrent_requests = parser.max_concurrent_requests
  81. 16 return if max_concurrent_requests == 1
  82. parser.max_concurrent_requests = 1
  83. request.once(:response) do
  84. parser.max_concurrent_requests = max_concurrent_requests
  85. end
  86. end
  87. end
  88. end
  89. 8 register_plugin(:h2c, H2C)
  90. end
  91. end

lib/httpx/plugins/ntlm_auth.rb

100.0% lines covered

30 relevant lines. 30 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 3 module HTTPX
  3. 3 module Plugins
  4. #
  5. # https://gitlab.com/os85/httpx/wikis/Auth#ntlm-auth
  6. #
  7. 3 module NTLMAuth
  8. 3 class << self
  9. 3 def load_dependencies(_klass)
  10. 2 require_relative "auth/ntlm"
  11. end
  12. 3 def extra_options(options)
  13. 2 options.merge(max_concurrent_requests: 1)
  14. end
  15. end
  16. 3 module OptionsMethods
  17. 3 def option_ntlm(value)
  18. 8 raise TypeError, ":ntlm must be a #{Authentication::Ntlm}" unless value.is_a?(Authentication::Ntlm)
  19. 8 value
  20. end
  21. end
  22. 3 module InstanceMethods
  23. 3 def ntlm_auth(user, password, domain = nil)
  24. 4 with(ntlm: Authentication::Ntlm.new(user, password, domain: domain))
  25. end
  26. 3 private
  27. 3 def send_requests(*requests)
  28. 8 requests.flat_map do |request|
  29. 8 ntlm = request.options.ntlm
  30. 8 if ntlm
  31. 4 request.headers["authorization"] = ntlm.negotiate
  32. 8 probe_response = wrap { super(request).first }
  33. 4 return probe_response unless probe_response.is_a?(Response)
  34. 4 if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
  35. 2 request.transition(:idle)
  36. 2 request.headers["authorization"] = ntlm.authenticate(request, probe_response.headers["www-authenticate"])
  37. 2 super(request)
  38. else
  39. 2 probe_response
  40. end
  41. else
  42. 4 super(request)
  43. end
  44. end
  45. end
  46. end
  47. end
  48. 3 register_plugin :ntlm_auth, NTLMAuth
  49. end
  50. end

lib/httpx/plugins/oauth.rb

100.0% lines covered

87 relevant lines. 87 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # https://gitlab.com/os85/httpx/wikis/OAuth
  6. #
  7. 8 module OAuth
  8. 8 class << self
  9. 8 def load_dependencies(_klass)
  10. 144 require_relative "auth/basic"
  11. end
  12. end
  13. 8 SUPPORTED_GRANT_TYPES = %w[client_credentials refresh_token].freeze
  14. 8 SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
  15. 8 class OAuthSession
  16. 8 attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope
  17. 8 def initialize(
  18. issuer:,
  19. client_id:,
  20. client_secret:,
  21. access_token: nil,
  22. refresh_token: nil,
  23. scope: nil,
  24. token_endpoint: nil,
  25. response_type: nil,
  26. grant_type: nil,
  27. token_endpoint_auth_method: nil
  28. )
  29. 128 @issuer = URI(issuer)
  30. 128 @client_id = client_id
  31. 128 @client_secret = client_secret
  32. 128 @token_endpoint = URI(token_endpoint) if token_endpoint
  33. 128 @response_type = response_type
  34. 128 @scope = case scope
  35. when String
  36. 48 scope.split
  37. when Array
  38. 32 scope
  39. end
  40. 128 @access_token = access_token
  41. 128 @refresh_token = refresh_token
  42. 128 @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
  43. 128 @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
  44. 128 unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
  45. 16 raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
  46. end
  47. 112 return if SUPPORTED_GRANT_TYPES.include?(@grant_type)
  48. 16 raise Error, "#{@grant_type} is not a supported grant type"
  49. end
  50. 8 def token_endpoint
  51. 112 @token_endpoint || "#{@issuer}/token"
  52. end
  53. 8 def token_endpoint_auth_method
  54. 160 @token_endpoint_auth_method || "client_secret_basic"
  55. end
  56. 8 def load(http)
  57. 48 return if @grant_type && @scope
  58. 16 metadata = http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json
  59. 16 @token_endpoint = metadata["token_endpoint"]
  60. 16 @scope = metadata["scopes_supported"]
  61. 64 @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
  62. 16 @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
  63. 16 SUPPORTED_AUTH_METHODS.include?(am)
  64. end
  65. 6 nil
  66. end
  67. 8 def merge(other)
  68. 96 obj = dup
  69. 84 case other
  70. when OAuthSession
  71. 48 other.instance_variables.each do |ivar|
  72. 432 val = other.instance_variable_get(ivar)
  73. 432 next unless val
  74. 336 obj.instance_variable_set(ivar, val)
  75. end
  76. when Hash
  77. 48 other.each do |k, v|
  78. 96 obj.instance_variable_set(:"@#{k}", v) if obj.instance_variable_defined?(:"@#{k}")
  79. end
  80. end
  81. 96 obj
  82. end
  83. end
  84. 8 module OptionsMethods
  85. 8 def option_oauth_session(value)
  86. 266 case value
  87. when Hash
  88. 16 OAuthSession.new(**value)
  89. when OAuthSession
  90. 272 value
  91. else
  92. 16 raise TypeError, ":oauth_session must be a #{OAuthSession}"
  93. end
  94. end
  95. end
  96. 8 module InstanceMethods
  97. 8 def oauth_auth(**args)
  98. 112 with(oauth_session: OAuthSession.new(**args))
  99. end
  100. 8 def with_access_token
  101. 48 oauth_session = @options.oauth_session
  102. 48 oauth_session.load(self)
  103. 48 grant_type = oauth_session.grant_type
  104. 48 headers = {}
  105. 48 form_post = { "grant_type" => grant_type, "scope" => Array(oauth_session.scope).join(" ") }.compact
  106. # auth
  107. 42 case oauth_session.token_endpoint_auth_method
  108. when "client_secret_post"
  109. 14 form_post["client_id"] = oauth_session.client_id
  110. 14 form_post["client_secret"] = oauth_session.client_secret
  111. when "client_secret_basic"
  112. 28 headers["authorization"] = Authentication::Basic.new(oauth_session.client_id, oauth_session.client_secret).authenticate
  113. end
  114. 42 case grant_type
  115. when "client_credentials"
  116. # do nothing
  117. when "refresh_token"
  118. 14 form_post["refresh_token"] = oauth_session.refresh_token
  119. end
  120. 48 token_request = build_request("POST", oauth_session.token_endpoint, headers: headers, form: form_post)
  121. 48 token_request.headers.delete("authorization") unless oauth_session.token_endpoint_auth_method == "client_secret_basic"
  122. 48 token_response = request(token_request)
  123. 48 token_response.raise_for_status
  124. 48 payload = token_response.json
  125. 48 access_token = payload["access_token"]
  126. 48 refresh_token = payload["refresh_token"]
  127. 48 with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
  128. end
  129. 8 def build_request(*)
  130. 128 request = super
  131. 128 return request if request.headers.key?("authorization")
  132. 96 oauth_session = @options.oauth_session
  133. 96 return request unless oauth_session && oauth_session.access_token
  134. 56 request.headers["authorization"] = "Bearer #{oauth_session.access_token}"
  135. 64 request
  136. end
  137. end
  138. end
  139. 8 register_plugin :oauth, OAuth
  140. end
  141. end

lib/httpx/plugins/persistent.rb

100.0% lines covered

28 relevant lines. 28 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 17 module HTTPX
  3. 17 module Plugins
  4. # This plugin implements a session that persists connections over the duration of the process.
  5. #
  6. # This will improve connection reuse in a long-running process.
  7. #
  8. # One important caveat to note is, although this session might not close connections,
  9. # other sessions from the same process that don't have this plugin turned on might.
  10. #
  11. # This session will still be able to work with it, as if, when expecting a connection
  12. # terminated by a different session, it will just retry on a new one and keep it open.
  13. #
  14. # This plugin is also not recommendable when connecting to >9000 (like, a lot) different origins.
  15. # So when you use this, make sure that you don't fall into this trap.
  16. #
  17. # https://gitlab.com/os85/httpx/wikis/Persistent
  18. #
  19. 17 module Persistent
  20. # subset of retryable errors which are safe to retry when reconnecting
  21. 2 RECONNECTABLE_ERRORS = [
  22. 15 IOError,
  23. EOFError,
  24. Errno::ECONNRESET,
  25. Errno::ECONNABORTED,
  26. Errno::EPIPE,
  27. Errno::EINVAL,
  28. Errno::ETIMEDOUT,
  29. ConnectionError,
  30. ].freeze
  31. 17 def self.load_dependencies(klass)
  32. 491 max_retries = if klass.default_options.respond_to?(:max_retries)
  33. 8 [klass.default_options.max_retries, 1].max
  34. else
  35. 483 1
  36. end
  37. 491 klass.plugin(:retries, max_retries: max_retries)
  38. end
  39. 17 def self.extra_options(options)
  40. 491 options.merge(persistent: true)
  41. end
  42. 17 module InstanceMethods
  43. 17 private
  44. 17 def repeatable_request?(request, _)
  45. 511 super || begin
  46. 196 return false unless request.ping?
  47. 8 response = request.response
  48. 8 return false unless response && response.is_a?(ErrorResponse)
  49. 8 error = response.error
  50. 72 RECONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
  51. end
  52. end
  53. 17 def get_current_selector
  54. 483 super(&nil) || begin
  55. 467 return unless block_given?
  56. 467 default = yield
  57. 467 set_current_selector(default)
  58. 467 default
  59. end
  60. end
  61. end
  62. end
  63. 17 register_plugin :persistent, Persistent
  64. end
  65. end

lib/httpx/plugins/proxy.rb

98.01% lines covered

151 relevant lines. 148 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 class HTTPProxyError < ConnectionError; end
  4. 10 module Plugins
  5. #
  6. # This plugin adds support for proxies. It ships with support for:
  7. #
  8. # * HTTP proxies
  9. # * HTTPS proxies
  10. # * Socks4/4a proxies
  11. # * Socks5 proxies
  12. #
  13. # https://gitlab.com/os85/httpx/wikis/Proxy
  14. #
  15. 10 module Proxy
  16. 10 Error = HTTPProxyError
  17. 10 PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
  18. 10 class << self
  19. 10 def configure(klass)
  20. 337 klass.plugin(:"proxy/http")
  21. 337 klass.plugin(:"proxy/socks4")
  22. 337 klass.plugin(:"proxy/socks5")
  23. end
  24. 10 def extra_options(options)
  25. 337 options.merge(supported_proxy_protocols: [])
  26. end
  27. end
  28. 10 class Parameters
  29. 10 attr_reader :uri, :username, :password, :scheme, :no_proxy
  30. 10 def initialize(uri: nil, scheme: nil, username: nil, password: nil, no_proxy: nil, **extra)
  31. 371 @no_proxy = Array(no_proxy) if no_proxy
  32. 371 @uris = Array(uri)
  33. 371 uri = @uris.first
  34. 371 @username = username
  35. 371 @password = password
  36. 371 @ns = 0
  37. 371 if uri
  38. 331 @uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
  39. 331 @username ||= @uri.user
  40. 331 @password ||= @uri.password
  41. end
  42. 371 @scheme = scheme
  43. 371 return unless @uri && @username && @password
  44. 210 @authenticator = nil
  45. 210 @scheme ||= infer_default_auth_scheme(@uri)
  46. 210 return unless @scheme
  47. 162 @authenticator = load_authenticator(@scheme, @username, @password, **extra)
  48. end
  49. 10 def shift
  50. # TODO: this operation must be synchronized
  51. 105 @ns += 1
  52. 120 @uri = @uris[@ns]
  53. 120 return unless @uri
  54. 16 @uri = URI(@uri) unless @uri.is_a?(URI::Generic)
  55. 16 scheme = infer_default_auth_scheme(@uri)
  56. 16 return unless scheme != @scheme
  57. 16 @scheme = scheme
  58. 16 @username = username || @uri.user
  59. 16 @password = password || @uri.password
  60. 16 @authenticator = load_authenticator(scheme, @username, @password)
  61. end
  62. 10 def can_authenticate?(*args)
  63. 184 return false unless @authenticator
  64. 64 @authenticator.can_authenticate?(*args)
  65. end
  66. 10 def authenticate(*args)
  67. 159 return unless @authenticator
  68. 159 @authenticator.authenticate(*args)
  69. end
  70. 10 def ==(other)
  71. 384 case other
  72. when Parameters
  73. 396 @uri == other.uri &&
  74. @username == other.username &&
  75. @password == other.password &&
  76. @scheme == other.scheme
  77. when URI::Generic, String
  78. 24 proxy_uri = @uri.dup
  79. 24 proxy_uri.user = @username
  80. 24 proxy_uri.password = @password
  81. 24 other_uri = other.is_a?(URI::Generic) ? other : URI.parse(other)
  82. 24 proxy_uri == other_uri
  83. else
  84. 16 super
  85. end
  86. end
  87. 10 private
  88. 10 def infer_default_auth_scheme(uri)
  89. 185 case uri.scheme
  90. when "socks5"
  91. 48 uri.scheme
  92. when "http", "https"
  93. 101 "basic"
  94. end
  95. end
  96. 10 def load_authenticator(scheme, username, password, **extra)
  97. 178 auth_scheme = scheme.to_s.capitalize
  98. 178 require_relative "auth/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
  99. 178 Authentication.const_get(auth_scheme).new(username, password, **extra)
  100. end
  101. end
  102. # adds support for the following options:
  103. #
  104. # :proxy :: proxy options defining *:uri*, *:username*, *:password* or
  105. # *:scheme* (i.e. <tt>{ uri: "http://proxy" }</tt>)
  106. 10 module OptionsMethods
  107. 10 def option_proxy(value)
  108. 672 value.is_a?(Parameters) ? value : Parameters.new(**Hash[value])
  109. end
  110. 10 def option_supported_proxy_protocols(value)
  111. 1699 raise TypeError, ":supported_proxy_protocols must be an Array" unless value.is_a?(Array)
  112. 1699 value.map(&:to_s)
  113. end
  114. end
  115. 10 module InstanceMethods
  116. 10 def find_connection(request_uri, selector, options)
  117. 417 return super unless options.respond_to?(:proxy)
  118. 417 if (next_proxy = request_uri.find_proxy)
  119. 4 return super(request_uri, selector, options.merge(proxy: Parameters.new(uri: next_proxy)))
  120. end
  121. 413 proxy = options.proxy
  122. 413 return super unless proxy
  123. 403 next_proxy = proxy.uri
  124. 403 raise Error, "Failed to connect to proxy" unless next_proxy
  125. 1 raise Error,
  126. 387 "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
  127. 379 if (no_proxy = proxy.no_proxy)
  128. 16 no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
  129. # TODO: setting proxy to nil leaks the connection object in the pool
  130. 16 return super(request_uri, selector, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(request_uri.host, next_proxy.host,
  131. next_proxy.port, no_proxy)
  132. end
  133. 371 super(request_uri, selector, options.merge(proxy: proxy))
  134. end
  135. 10 private
  136. 10 def fetch_response(request, selector, options)
  137. 1581 response = super
  138. 1581 if response.is_a?(ErrorResponse) && proxy_error?(request, response, options)
  139. 120 options.proxy.shift
  140. # return last error response if no more proxies to try
  141. 120 return response if options.proxy.uri.nil?
  142. 16 log { "failed connecting to proxy, trying next..." }
  143. 16 request.transition(:idle)
  144. 16 send_request(request, selector, options)
  145. 14 return
  146. end
  147. 1461 response
  148. end
  149. 10 def proxy_error?(_request, response, options)
  150. 169 return false unless options.proxy
  151. 168 error = response.error
  152. 147 case error
  153. when NativeResolveError
  154. 16 proxy_uri = URI(options.proxy.uri)
  155. 16 peer = error.connection.peer
  156. # failed resolving proxy domain
  157. 16 peer.host == proxy_uri.host && peer.port == proxy_uri.port
  158. when ResolveError
  159. proxy_uri = URI(options.proxy.uri)
  160. error.message.end_with?(proxy_uri.to_s)
  161. when *PROXY_ERRORS
  162. # timeout errors connecting to proxy
  163. 152 true
  164. else
  165. false
  166. end
  167. end
  168. end
  169. 10 module ConnectionMethods
  170. 10 using URIExtensions
  171. 10 def initialize(*)
  172. 384 super
  173. 384 return unless @options.proxy
  174. # redefining the connection origin as the proxy's URI,
  175. # as this will be used as the tcp peer ip.
  176. 366 @proxy_uri = URI(@options.proxy.uri)
  177. end
  178. 10 def peer
  179. 946 @proxy_uri || super
  180. end
  181. 10 def connecting?
  182. 4580 return super unless @options.proxy
  183. 4411 super || @state == :connecting || @state == :connected
  184. end
  185. 10 def call
  186. 1135 super
  187. 1135 return unless @options.proxy
  188. 995 case @state
  189. when :connecting
  190. 309 consume
  191. end
  192. end
  193. 10 def reset
  194. 384 return super unless @options.proxy
  195. 367 @state = :open
  196. 367 super
  197. # emit(:close)
  198. end
  199. 10 private
  200. 10 def initialize_type(uri, options)
  201. 384 return super unless options.proxy
  202. 366 "tcp"
  203. end
  204. 10 def connect
  205. 1042 return super unless @options.proxy
  206. 891 case @state
  207. when :idle
  208. 712 transition(:connecting)
  209. when :connected
  210. 296 transition(:open)
  211. end
  212. end
  213. 10 def handle_transition(nextstate)
  214. 2117 return super unless @options.proxy
  215. 1791 case nextstate
  216. when :closing
  217. # this is a hack so that we can use the super method
  218. # and it'll think that the current state is open
  219. 375 @state = :open if @state == :connecting
  220. end
  221. 2030 super
  222. end
  223. end
  224. end
  225. 10 register_plugin :proxy, Proxy
  226. end
  227. 10 class ProxySSL < SSL
  228. 10 def initialize(tcp, request_uri, options)
  229. 89 @io = tcp.to_io
  230. 89 super(request_uri, tcp.addresses, options)
  231. 89 @hostname = request_uri.host
  232. 89 @state = :connected
  233. end
  234. end
  235. end

lib/httpx/plugins/proxy/http.rb

100.0% lines covered

104 relevant lines. 104 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 module Plugins
  4. 10 module Proxy
  5. 10 module HTTP
  6. 10 class << self
  7. 10 def extra_options(options)
  8. 337 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[http])
  9. end
  10. end
  11. 10 module InstanceMethods
  12. 10 def with_proxy_basic_auth(opts)
  13. 8 with(proxy: opts.merge(scheme: "basic"))
  14. end
  15. 10 def with_proxy_digest_auth(opts)
  16. 24 with(proxy: opts.merge(scheme: "digest"))
  17. end
  18. 10 def with_proxy_ntlm_auth(opts)
  19. 8 with(proxy: opts.merge(scheme: "ntlm"))
  20. end
  21. 10 def fetch_response(request, selector, options)
  22. 1581 response = super
  23. 1581 if response &&
  24. response.is_a?(Response) &&
  25. response.status == 407 &&
  26. !request.headers.key?("proxy-authorization") &&
  27. response.headers.key?("proxy-authenticate") && options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
  28. 8 request.transition(:idle)
  29. 7 request.headers["proxy-authorization"] =
  30. options.proxy.authenticate(request, response.headers["proxy-authenticate"])
  31. 8 send_request(request, selector, options)
  32. 7 return
  33. end
  34. 1573 response
  35. end
  36. end
  37. 10 module ConnectionMethods
  38. 10 def connecting?
  39. 4580 super || @state == :connecting || @state == :connected
  40. end
  41. 10 private
  42. 10 def handle_transition(nextstate)
  43. 2356 return super unless @options.proxy && @options.proxy.uri.scheme == "http"
  44. 1020 case nextstate
  45. when :connecting
  46. 304 return unless @state == :idle
  47. 304 @io.connect
  48. 304 return unless @io.connected?
  49. 152 @parser || begin
  50. 144 @parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
  51. 144 parser = @parser
  52. 144 parser.extend(ProxyParser)
  53. 144 parser.on(:response, &method(:__http_on_connect))
  54. 144 parser.on(:close) do |force|
  55. 57 next unless @parser
  56. 8 if force
  57. 8 reset
  58. 8 emit(:terminate)
  59. end
  60. end
  61. 144 parser.on(:reset) do
  62. 16 if parser.empty?
  63. 8 reset
  64. else
  65. 8 transition(:closing)
  66. 8 transition(:closed)
  67. 8 parser.reset if @parser
  68. 8 transition(:idle)
  69. 8 transition(:connecting)
  70. end
  71. end
  72. 144 __http_proxy_connect(parser)
  73. end
  74. 152 return if @state == :connected
  75. when :connected
  76. 136 return unless @state == :idle || @state == :connecting
  77. 121 case @state
  78. when :connecting
  79. 49 parser = @parser
  80. 49 @parser = nil
  81. 49 parser.close
  82. when :idle
  83. 87 @parser.callbacks.clear
  84. 87 set_parser_callbacks(@parser)
  85. end
  86. end
  87. 908 super
  88. end
  89. 10 def __http_proxy_connect(parser)
  90. 144 req = @pending.first
  91. 144 if req && req.uri.scheme == "https"
  92. # if the first request after CONNECT is to an https address, it is assumed that
  93. # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
  94. # and therefore, will share the connection.
  95. #
  96. 57 connect_request = ConnectRequest.new(req.uri, @options)
  97. 51 @inflight += 1
  98. 57 parser.send(connect_request)
  99. else
  100. 87 handle_transition(:connected)
  101. end
  102. end
  103. 10 def __http_on_connect(request, response)
  104. 58 @inflight -= 1
  105. 65 if response.is_a?(Response) && response.status == 200
  106. 49 req = @pending.first
  107. 49 request_uri = req.uri
  108. 49 @io = ProxySSL.new(@io, request_uri, @options)
  109. 49 transition(:connected)
  110. 49 throw(:called)
  111. 15 elsif response.is_a?(Response) &&
  112. response.status == 407 &&
  113. !request.headers.key?("proxy-authorization") &&
  114. @options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
  115. 8 request.transition(:idle)
  116. 7 request.headers["proxy-authorization"] = @options.proxy.authenticate(request, response.headers["proxy-authenticate"])
  117. 8 @parser.send(request)
  118. 7 @inflight += 1
  119. else
  120. 8 pending = @pending + @parser.pending
  121. 21 while (req = pending.shift)
  122. 8 response.finish!
  123. 8 req.response = response
  124. 8 req.emit(:response, response)
  125. end
  126. 8 reset
  127. end
  128. end
  129. end
  130. 10 module ProxyParser
  131. 10 def join_headline(request)
  132. 144 return super if request.verb == "CONNECT"
  133. 70 "#{request.verb} #{request.uri} HTTP/#{@version.join(".")}"
  134. end
  135. 10 def set_protocol_headers(request)
  136. 152 extra_headers = super
  137. 152 proxy_params = @options.proxy
  138. 152 if proxy_params.scheme == "basic"
  139. # opt for basic auth
  140. 85 extra_headers["proxy-authorization"] = proxy_params.authenticate(extra_headers)
  141. end
  142. 152 extra_headers["proxy-connection"] = extra_headers.delete("connection") if extra_headers.key?("connection")
  143. 152 extra_headers
  144. end
  145. end
  146. 10 class ConnectRequest < Request
  147. 10 def initialize(uri, options)
  148. 57 super("CONNECT", uri, options)
  149. 57 @headers.delete("accept")
  150. end
  151. 10 def path
  152. 65 "#{@uri.hostname}:#{@uri.port}"
  153. end
  154. end
  155. end
  156. end
  157. 10 register_plugin :"proxy/http", Proxy::HTTP
  158. end
  159. end

lib/httpx/plugins/proxy/socks4.rb

97.47% lines covered

79 relevant lines. 77 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 require "resolv"
  3. 10 require "ipaddr"
  4. 10 module HTTPX
  5. 10 class Socks4Error < HTTPProxyError; end
  6. 10 module Plugins
  7. 10 module Proxy
  8. 10 module Socks4
  9. 10 VERSION = 4
  10. 10 CONNECT = 1
  11. 10 GRANTED = 0x5A
  12. 10 PROTOCOLS = %w[socks4 socks4a].freeze
  13. 10 Error = Socks4Error
  14. 10 class << self
  15. 10 def extra_options(options)
  16. 337 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + PROTOCOLS)
  17. end
  18. end
  19. 10 module ConnectionMethods
  20. 10 def interests
  21. 3891 if @state == :connecting
  22. return @write_buffer.empty? ? :r : :w
  23. end
  24. 3891 super
  25. end
  26. 10 private
  27. 10 def handle_transition(nextstate)
  28. 2420 return super unless @options.proxy && PROTOCOLS.include?(@options.proxy.uri.scheme)
  29. 382 case nextstate
  30. when :connecting
  31. 128 return unless @state == :idle
  32. 128 @io.connect
  33. 128 return unless @io.connected?
  34. 64 req = @pending.first
  35. 64 return unless req
  36. 64 request_uri = req.uri
  37. 64 @write_buffer << Packet.connect(@options.proxy, request_uri)
  38. 64 __socks4_proxy_connect
  39. when :connected
  40. 48 return unless @state == :connecting
  41. 48 @parser = nil
  42. end
  43. 371 log(level: 1) { "SOCKS4: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
  44. 371 super
  45. end
  46. 10 def __socks4_proxy_connect
  47. 64 @parser = SocksParser.new(@write_buffer, @options)
  48. 64 @parser.once(:packet, &method(:__socks4_on_packet))
  49. end
  50. 10 def __socks4_on_packet(packet)
  51. 64 _version, status, _port, _ip = packet.unpack("CCnN")
  52. 64 if status == GRANTED
  53. 48 req = @pending.first
  54. 48 request_uri = req.uri
  55. 48 @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
  56. 48 transition(:connected)
  57. 48 throw(:called)
  58. else
  59. 16 on_socks4_error("socks error: #{status}")
  60. end
  61. end
  62. 10 def on_socks4_error(message)
  63. 16 ex = Error.new(message)
  64. 16 ex.set_backtrace(caller)
  65. 16 on_error(ex)
  66. 16 throw(:called)
  67. end
  68. end
  69. 10 class SocksParser
  70. 10 include HTTPX::Callbacks
  71. 10 def initialize(buffer, options)
  72. 64 @buffer = buffer
  73. 64 @options = options
  74. end
  75. 10 def close; end
  76. 10 def consume(*); end
  77. 10 def empty?
  78. true
  79. end
  80. 10 def <<(packet)
  81. 64 emit(:packet, packet)
  82. end
  83. end
  84. 10 module Packet
  85. 10 module_function
  86. 10 def connect(parameters, uri)
  87. 64 packet = [VERSION, CONNECT, uri.port].pack("CCn")
  88. 56 case parameters.uri.scheme
  89. when "socks4"
  90. 48 socks_host = uri.host
  91. 5 begin
  92. 96 ip = IPAddr.new(socks_host)
  93. 48 packet << ip.hton
  94. rescue IPAddr::InvalidAddressError
  95. 48 socks_host = Resolv.getaddress(socks_host)
  96. 48 retry
  97. end
  98. 48 packet << [parameters.username].pack("Z*")
  99. when "socks4a"
  100. 16 packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
  101. end
  102. 64 packet
  103. end
  104. end
  105. end
  106. end
  107. 10 register_plugin :"proxy/socks4", Proxy::Socks4
  108. end
  109. end

lib/httpx/plugins/proxy/socks5.rb

99.12% lines covered

113 relevant lines. 112 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 class Socks5Error < HTTPProxyError; end
  4. 10 module Plugins
  5. 10 module Proxy
  6. 10 module Socks5
  7. 10 VERSION = 5
  8. 10 NOAUTH = 0
  9. 10 PASSWD = 2
  10. 10 NONE = 0xff
  11. 10 CONNECT = 1
  12. 10 IPV4 = 1
  13. 10 DOMAIN = 3
  14. 10 IPV6 = 4
  15. 10 SUCCESS = 0
  16. 10 Error = Socks5Error
  17. 10 class << self
  18. 10 def load_dependencies(*)
  19. 337 require_relative "../auth/socks5"
  20. end
  21. 10 def extra_options(options)
  22. 337 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[socks5])
  23. end
  24. end
  25. 10 module ConnectionMethods
  26. 10 def call
  27. 1135 super
  28. 1135 return unless @options.proxy && @options.proxy.uri.scheme == "socks5"
  29. 316 case @state
  30. when :connecting,
  31. :negotiating,
  32. :authenticating
  33. 157 consume
  34. end
  35. end
  36. 10 def connecting?
  37. 4580 super || @state == :authenticating || @state == :negotiating
  38. end
  39. 10 def interests
  40. 6487 if @state == :connecting || @state == :authenticating || @state == :negotiating
  41. 2366 return @write_buffer.empty? ? :r : :w
  42. end
  43. 3891 super
  44. end
  45. 10 private
  46. 10 def handle_transition(nextstate)
  47. 2708 return super unless @options.proxy && @options.proxy.uri.scheme == "socks5"
  48. 909 case nextstate
  49. when :connecting
  50. 288 return unless @state == :idle
  51. 288 @io.connect
  52. 288 return unless @io.connected?
  53. 144 @write_buffer << Packet.negotiate(@options.proxy)
  54. 144 __socks5_proxy_connect
  55. when :authenticating
  56. 48 return unless @state == :connecting
  57. 48 @write_buffer << Packet.authenticate(@options.proxy)
  58. when :negotiating
  59. 192 return unless @state == :connecting || @state == :authenticating
  60. 48 req = @pending.first
  61. 48 request_uri = req.uri
  62. 48 @write_buffer << Packet.connect(request_uri)
  63. when :connected
  64. 32 return unless @state == :negotiating
  65. 32 @parser = nil
  66. end
  67. 751 log(level: 1) { "SOCKS5: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
  68. 751 super
  69. end
  70. 10 def __socks5_proxy_connect
  71. 144 @parser = SocksParser.new(@write_buffer, @options)
  72. 144 @parser.on(:packet, &method(:__socks5_on_packet))
  73. 144 transition(:negotiating)
  74. end
  75. 10 def __socks5_on_packet(packet)
  76. 210 case @state
  77. when :connecting
  78. 144 version, method = packet.unpack("CC")
  79. 144 __socks5_check_version(version)
  80. 126 case method
  81. when PASSWD
  82. 48 transition(:authenticating)
  83. 18 nil
  84. when NONE
  85. 80 __on_socks5_error("no supported authorization methods")
  86. else
  87. 16 transition(:negotiating)
  88. end
  89. when :authenticating
  90. 48 _, status = packet.unpack("CC")
  91. 48 return transition(:negotiating) if status == SUCCESS
  92. 16 __on_socks5_error("socks authentication error: #{status}")
  93. when :negotiating
  94. 48 version, reply, = packet.unpack("CC")
  95. 48 __socks5_check_version(version)
  96. 48 __on_socks5_error("socks5 negotiation error: #{reply}") unless reply == SUCCESS
  97. 32 req = @pending.first
  98. 32 request_uri = req.uri
  99. 32 @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
  100. 32 transition(:connected)
  101. 32 throw(:called)
  102. end
  103. end
  104. 10 def __socks5_check_version(version)
  105. 192 __on_socks5_error("invalid SOCKS version (#{version})") if version != 5
  106. end
  107. 10 def __on_socks5_error(message)
  108. 112 ex = Error.new(message)
  109. 112 ex.set_backtrace(caller)
  110. 112 on_error(ex)
  111. 112 throw(:called)
  112. end
  113. end
  114. 10 class SocksParser
  115. 10 include HTTPX::Callbacks
  116. 10 def initialize(buffer, options)
  117. 144 @buffer = buffer
  118. 144 @options = options
  119. end
  120. 10 def close; end
  121. 10 def consume(*); end
  122. 10 def empty?
  123. true
  124. end
  125. 10 def <<(packet)
  126. 240 emit(:packet, packet)
  127. end
  128. end
  129. 10 module Packet
  130. 10 module_function
  131. 10 def negotiate(parameters)
  132. 144 methods = [NOAUTH]
  133. 144 methods << PASSWD if parameters.can_authenticate?
  134. 144 methods.unshift(methods.size)
  135. 144 methods.unshift(VERSION)
  136. 144 methods.pack("C*")
  137. end
  138. 10 def authenticate(parameters)
  139. 48 parameters.authenticate
  140. end
  141. 10 def connect(uri)
  142. 48 packet = [VERSION, CONNECT, 0].pack("C*")
  143. 5 begin
  144. 48 ip = IPAddr.new(uri.host)
  145. 16 ipcode = ip.ipv6? ? IPV6 : IPV4
  146. 16 packet << [ipcode].pack("C") << ip.hton
  147. rescue IPAddr::InvalidAddressError
  148. 32 packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
  149. end
  150. 48 packet << [uri.port].pack("n")
  151. 48 packet
  152. end
  153. end
  154. end
  155. end
  156. 10 register_plugin :"proxy/socks5", Proxy::Socks5
  157. end
  158. end

lib/httpx/plugins/proxy/ssh.rb

92.31% lines covered

52 relevant lines. 48 lines covered and 4 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "httpx/plugins/proxy"
  3. 6 module HTTPX
  4. 6 module Plugins
  5. 6 module Proxy
  6. 6 module SSH
  7. 6 class << self
  8. 6 def load_dependencies(*)
  9. 12 require "net/ssh/gateway"
  10. end
  11. end
  12. 6 module OptionsMethods
  13. 6 def option_proxy(value)
  14. 24 Hash[value]
  15. end
  16. end
  17. 6 module InstanceMethods
  18. 6 def request(*args, **options)
  19. 12 raise ArgumentError, "must perform at least one request" if args.empty?
  20. 12 requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
  21. 12 request = requests.first or return super
  22. 12 request_options = request.options
  23. 12 return super unless request_options.proxy
  24. 12 ssh_options = request_options.proxy
  25. 12 ssh_uris = ssh_options.delete(:uri)
  26. 12 ssh_uri = URI.parse(ssh_uris.shift)
  27. 12 return super unless ssh_uri.scheme == "ssh"
  28. 12 ssh_username = ssh_options.delete(:username)
  29. 12 ssh_options[:port] ||= ssh_uri.port || 22
  30. 12 if request_options.debug
  31. ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
  32. end
  33. 12 request_uri = URI(requests.first.uri)
  34. 12 @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
  35. begin
  36. 12 @_gateway.open(request_uri.host, request_uri.port) do |local_port|
  37. 12 io = build_gateway_socket(local_port, request_uri, request_options)
  38. 12 super(*args, **options.merge(io: io))
  39. end
  40. ensure
  41. 12 @_gateway.shutdown!
  42. end
  43. end
  44. 6 private
  45. 6 def build_gateway_socket(port, request_uri, options)
  46. 12 case request_uri.scheme
  47. when "https"
  48. 6 ctx = OpenSSL::SSL::SSLContext.new
  49. 6 ctx_options = SSL::TLS_OPTIONS.merge(options.ssl)
  50. 6 ctx.set_params(ctx_options) unless ctx_options.empty?
  51. 6 sock = TCPSocket.open("localhost", port)
  52. 6 io = OpenSSL::SSL::SSLSocket.new(sock, ctx)
  53. 6 io.hostname = request_uri.host
  54. 6 io.sync_close = true
  55. 6 io.connect
  56. 6 io.post_connection_check(request_uri.host) if ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
  57. 6 io
  58. when "http"
  59. 6 TCPSocket.open("localhost", port)
  60. else
  61. raise TypeError, "unexpected scheme: #{request_uri.scheme}"
  62. end
  63. end
  64. end
  65. 6 module ConnectionMethods
  66. # should not coalesce connections here, as the IP is the IP of the proxy
  67. 6 def coalescable?(*)
  68. return super unless @options.proxy
  69. false
  70. end
  71. end
  72. end
  73. end
  74. 6 register_plugin :"proxy/ssh", Proxy::SSH
  75. end
  76. end

lib/httpx/plugins/push_promise.rb

100.0% lines covered

41 relevant lines. 41 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for HTTP/2 Push responses.
  6. #
  7. # In order to benefit from this, requests are sent one at a time, so that
  8. # no push responses are received after corresponding request has been sent.
  9. #
  10. # https://gitlab.com/os85/httpx/wikis/Server-Push
  11. #
  12. 8 module PushPromise
  13. 8 def self.extra_options(options)
  14. 16 options.merge(http2_settings: { settings_enable_push: 1 },
  15. max_concurrent_requests: 1)
  16. end
  17. 8 module ResponseMethods
  18. 8 def pushed?
  19. 16 @__pushed
  20. end
  21. 8 def mark_as_pushed!
  22. 8 @__pushed = true
  23. end
  24. end
  25. 8 module InstanceMethods
  26. 8 private
  27. 8 def promise_headers
  28. 16 @promise_headers ||= {}
  29. end
  30. 8 def on_promise(parser, stream)
  31. 16 stream.on(:promise_headers) do |h|
  32. 16 __on_promise_request(parser, stream, h)
  33. end
  34. 16 stream.on(:headers) do |h|
  35. 8 __on_promise_response(parser, stream, h)
  36. end
  37. end
  38. 8 def __on_promise_request(parser, stream, h)
  39. 16 log(level: 1, color: :yellow) do
  40. skipped # :nocov:
  41. skipped h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
  42. skipped # :nocov:
  43. end
  44. 16 headers = @options.headers_class.new(h)
  45. 16 path = headers[":path"]
  46. 16 authority = headers[":authority"]
  47. 24 request = parser.pending.find { |r| r.authority == authority && r.path == path }
  48. 16 if request
  49. 8 request.merge_headers(headers)
  50. 7 promise_headers[stream] = request
  51. 8 parser.pending.delete(request)
  52. 7 parser.streams[request] = stream
  53. 8 request.transition(:done)
  54. else
  55. 8 stream.refuse
  56. end
  57. end
  58. 8 def __on_promise_response(parser, stream, h)
  59. 8 request = promise_headers.delete(stream)
  60. 8 return unless request
  61. 8 parser.__send__(:on_stream_headers, stream, request, h)
  62. 8 response = request.response
  63. 8 response.mark_as_pushed!
  64. 8 stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
  65. 8 stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
  66. end
  67. end
  68. end
  69. 8 register_plugin(:push_promise, PushPromise)
  70. end
  71. end

lib/httpx/plugins/query.rb

100.0% lines covered

15 relevant lines. 15 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for using the experimental QUERY HTTP method
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Query
  8. 8 module Query
  9. 8 def self.subplugins
  10. 3 {
  11. 20 retries: QueryRetries,
  12. }
  13. end
  14. 8 module InstanceMethods
  15. 8 def query(*uri, **options)
  16. 16 request("QUERY", uri, **options)
  17. end
  18. end
  19. 8 module QueryRetries
  20. 8 module InstanceMethods
  21. 8 private
  22. 8 def repeatable_request?(request, options)
  23. 24 super || request.verb == "QUERY"
  24. end
  25. end
  26. end
  27. end
  28. 8 register_plugin :query, Query
  29. end
  30. end

lib/httpx/plugins/rate_limiter.rb

100.0% lines covered

18 relevant lines. 18 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when the request:
  6. #
  7. # * is rate limited;
  8. # * when the server is unavailable (503);
  9. # * when a 3xx request comes with a "retry-after" value
  10. #
  11. # https://gitlab.com/os85/httpx/wikis/Rate-Limiter
  12. #
  13. 8 module RateLimiter
  14. 8 class << self
  15. 8 RATE_LIMIT_CODES = [429, 503].freeze
  16. 8 def configure(klass)
  17. 64 klass.plugin(:retries,
  18. retry_change_requests: true,
  19. 7 retry_on: method(:retry_on_rate_limited_response),
  20. retry_after: method(:retry_after_rate_limit))
  21. end
  22. 8 def retry_on_rate_limited_response(response)
  23. 128 return false unless response.is_a?(Response)
  24. 128 status = response.status
  25. 128 RATE_LIMIT_CODES.include?(status)
  26. end
  27. # Servers send the "Retry-After" header field to indicate how long the
  28. # user agent ought to wait before making a follow-up request. When
  29. # sent with a 503 (Service Unavailable) response, Retry-After indicates
  30. # how long the service is expected to be unavailable to the client.
  31. # When sent with any 3xx (Redirection) response, Retry-After indicates
  32. # the minimum time that the user agent is asked to wait before issuing
  33. # the redirected request.
  34. #
  35. 8 def retry_after_rate_limit(_, response)
  36. 64 return unless response.is_a?(Response)
  37. 64 retry_after = response.headers["retry-after"]
  38. 64 return unless retry_after
  39. 32 Utils.parse_retry_after(retry_after)
  40. end
  41. end
  42. end
  43. 8 register_plugin :rate_limiter, RateLimiter
  44. end
  45. end

lib/httpx/plugins/response_cache.rb

100.0% lines covered

143 relevant lines. 143 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when certain errors happen.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Response-Cache
  8. #
  9. 8 module ResponseCache
  10. 8 CACHEABLE_VERBS = %w[GET HEAD].freeze
  11. 8 CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
  12. 8 SUPPORTED_VARY_HEADERS = %w[accept accept-encoding accept-language cookie origin].sort.freeze
  13. 8 private_constant :CACHEABLE_VERBS
  14. 8 private_constant :CACHEABLE_STATUS_CODES
  15. 8 class << self
  16. 8 def load_dependencies(*)
  17. 216 require_relative "response_cache/store"
  18. 216 require_relative "response_cache/file_store"
  19. end
  20. # whether the +response+ can be stored in the response cache.
  21. # (i.e. has a cacheable body, does not contain directives prohibiting storage, etc...)
  22. 8 def cacheable_response?(response)
  23. 136 response.is_a?(Response) &&
  24. (
  25. 136 response.cache_control.nil? ||
  26. # TODO: !response.cache_control.include?("private") && is shared cache
  27. !response.cache_control.include?("no-store")
  28. ) &&
  29. CACHEABLE_STATUS_CODES.include?(response.status) &&
  30. # RFC 2616 13.4 - A response received with a status code of 200, 203, 206, 300, 301 or
  31. # 410 MAY be stored by a cache and used in reply to a subsequent
  32. # request, subject to the expiration mechanism, unless a cache-control
  33. # directive prohibits caching. However, a cache that does not support
  34. # the Range and Content-Range headers MUST NOT cache 206 (Partial
  35. # Content) responses.
  36. response.status != 206
  37. end
  38. # whether the +response+
  39. 8 def not_modified?(response)
  40. 168 response.is_a?(Response) && response.status == 304
  41. end
  42. 8 def extra_options(options)
  43. 216 options.merge(
  44. supported_vary_headers: SUPPORTED_VARY_HEADERS,
  45. response_cache_store: :store,
  46. )
  47. end
  48. end
  49. # adds support for the following options:
  50. #
  51. # :supported_vary_headers :: array of header values that will be considered for a "vary" header based cache validation
  52. # (defaults to {SUPPORTED_VARY_HEADERS}).
  53. # :response_cache_store :: object where cached responses are fetch from or stored in; defaults to <tt>:store</tt> (in-memory
  54. # cache), can be set to <tt>:file_store</tt> (file system cache store) as well, or any object which
  55. # abides by the Cache Store Interface
  56. #
  57. # The Cache Store Interface requires implementation of the following methods:
  58. #
  59. # * +#get(request) -> response or nil+
  60. # * +#set(request, response) -> void+
  61. # * +#clear() -> void+)
  62. #
  63. 8 module OptionsMethods
  64. 8 def option_response_cache_store(value)
  65. 315 case value
  66. when :store
  67. 232 Store.new
  68. when :file_store
  69. 16 FileStore.new
  70. else
  71. 112 value
  72. end
  73. end
  74. 8 def option_supported_vary_headers(value)
  75. 216 Array(value).sort
  76. end
  77. end
  78. 8 module InstanceMethods
  79. # wipes out all cached responses from the cache store.
  80. 8 def clear_response_cache
  81. 128 @options.response_cache_store.clear
  82. end
  83. 8 def build_request(*)
  84. 456 request = super
  85. 456 return request unless cacheable_request?(request)
  86. 440 prepare_cache(request)
  87. 440 request
  88. end
  89. 8 private
  90. 8 def send_request(request, *)
  91. 168 return request if request.response
  92. 152 super
  93. end
  94. 8 def fetch_response(request, *)
  95. 539 response = super
  96. 539 return unless response
  97. 168 if ResponseCache.not_modified?(response)
  98. 32 log { "returning cached response for #{request.uri}" }
  99. 32 response.copy_from_cached!
  100. 135 elsif request.cacheable_verb? && ResponseCache.cacheable_response?(response)
  101. 112 request.options.response_cache_store.set(request, response) unless response.cached?
  102. end
  103. 168 response
  104. end
  105. # will either assign a still-fresh cached response to +request+, or set up its HTTP
  106. # cache invalidation headers in case it's not fresh anymore.
  107. 8 def prepare_cache(request)
  108. 664 cached_response = request.options.response_cache_store.get(request)
  109. 664 return unless cached_response && match_by_vary?(request, cached_response)
  110. 304 cached_response.body.rewind
  111. 304 if cached_response.fresh?
  112. 64 cached_response = cached_response.dup
  113. 64 cached_response.mark_as_cached!
  114. 64 request.response = cached_response
  115. 64 request.emit(:response, cached_response)
  116. 56 return
  117. end
  118. 240 request.cached_response = cached_response
  119. 240 if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
  120. 32 request.headers.add("if-modified-since", last_modified)
  121. end
  122. 240 if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
  123. 176 request.headers.add("if-none-match", etag)
  124. end
  125. end
  126. 8 def cacheable_request?(request)
  127. 456 request.cacheable_verb? &&
  128. (
  129. 440 !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
  130. )
  131. end
  132. # whether the +response+ complies with the directives set by the +request+ "vary" header
  133. # (true when none is available).
  134. 8 def match_by_vary?(request, response)
  135. 304 vary = response.vary
  136. 304 return true unless vary
  137. 96 original_request = response.original_request
  138. 96 if vary == %w[*]
  139. 32 request.options.supported_vary_headers.each do |field|
  140. 160 return false unless request.headers[field] == original_request.headers[field]
  141. end
  142. 28 return true
  143. end
  144. 64 vary.all? do |field|
  145. 64 !original_request.headers.key?(field) || request.headers[field] == original_request.headers[field]
  146. end
  147. end
  148. end
  149. 8 module RequestMethods
  150. # points to a previously cached Response corresponding to this request.
  151. 8 attr_accessor :cached_response
  152. 8 def initialize(*)
  153. 608 super
  154. 608 @cached_response = nil
  155. end
  156. 8 def merge_headers(*)
  157. 288 super
  158. 288 @response_cache_key = nil
  159. end
  160. # returns whether this request is cacheable as per HTTP caching rules.
  161. 8 def cacheable_verb?
  162. 592 CACHEABLE_VERBS.include?(@verb)
  163. end
  164. # returns a unique cache key as a String identifying this request
  165. 8 def response_cache_key
  166. 1272 @response_cache_key ||= begin
  167. 464 keys = [@verb, @uri]
  168. 464 @options.supported_vary_headers.each do |field|
  169. 2320 value = @headers[field]
  170. 2320 keys << value if value
  171. end
  172. 464 Digest::SHA1.hexdigest("httpx-response-cache-#{keys.join("-")}")
  173. end
  174. end
  175. end
  176. 8 module ResponseMethods
  177. 8 attr_writer :original_request
  178. 8 def initialize(*)
  179. 488 super
  180. 488 @cached = false
  181. end
  182. # a copy of the request this response was originally cached from
  183. 8 def original_request
  184. 96 @original_request || @request
  185. end
  186. # whether this Response was duplicated from a previously {RequestMethods#cached_response}.
  187. 8 def cached?
  188. 112 @cached
  189. end
  190. # sets this Response as being duplicated from a previously cached response.
  191. 8 def mark_as_cached!
  192. 64 @cached = true
  193. end
  194. # eager-copies the response headers and body from {RequestMethods#cached_response}.
  195. 8 def copy_from_cached!
  196. 32 cached_response = @request.cached_response
  197. 32 return unless cached_response
  198. # 304 responses do not have content-type, which are needed for decoding.
  199. 32 @headers = @headers.class.new(cached_response.headers.merge(@headers))
  200. 32 @body = cached_response.body.dup
  201. 32 @body.rewind
  202. end
  203. # A response is fresh if its age has not yet exceeded its freshness lifetime.
  204. # other (#cache_control} directives may influence the outcome, as per the rules
  205. # from the {rfc}[https://www.rfc-editor.org/rfc/rfc7234]
  206. 8 def fresh?
  207. 304 if cache_control
  208. 112 return false if cache_control.include?("no-cache")
  209. 80 return true if cache_control.include?("immutable")
  210. # check age: max-age
  211. 192 max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
  212. 192 max_age ||= cache_control.find { |directive| directive.start_with?("max-age") }
  213. 80 max_age = max_age[/age=(\d+)/, 1] if max_age
  214. 80 max_age = max_age.to_i if max_age
  215. 80 return max_age > age if max_age
  216. end
  217. # check age: expires
  218. 192 if @headers.key?("expires")
  219. 5 begin
  220. 48 expires = Time.httpdate(@headers["expires"])
  221. rescue ArgumentError
  222. 16 return false
  223. end
  224. 28 return (expires - Time.now).to_i.positive?
  225. end
  226. 144 false
  227. end
  228. # returns the "cache-control" directives as an Array of String(s).
  229. 8 def cache_control
  230. 848 return @cache_control if defined?(@cache_control)
  231. 48 @cache_control = begin
  232. 384 return unless @headers.key?("cache-control")
  233. 112 @headers["cache-control"].split(/ *, */)
  234. end
  235. end
  236. # returns the "vary" header value as an Array of (String) headers.
  237. 8 def vary
  238. 304 return @vary if defined?(@vary)
  239. 34 @vary = begin
  240. 272 return unless @headers.key?("vary")
  241. 64 @headers["vary"].split(/ *, */).map(&:downcase)
  242. end
  243. end
  244. 8 private
  245. # returns the value of the "age" header as an Integer (time since epoch).
  246. # if no "age" of header exists, it returns the number of seconds since {#date}.
  247. 8 def age
  248. 80 return @headers["age"].to_i if @headers.key?("age")
  249. 80 (Time.now - date).to_i
  250. end
  251. # returns the value of the "date" header as a Time object
  252. 8 def date
  253. 80 @date ||= Time.httpdate(@headers["date"])
  254. rescue NoMethodError, ArgumentError
  255. 16 Time.now
  256. end
  257. end
  258. end
  259. 8 register_plugin :response_cache, ResponseCache
  260. end
  261. end

lib/httpx/plugins/response_cache/file_store.rb

100.0% lines covered

71 relevant lines. 71 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "pathname"
  3. 8 module HTTPX::Plugins
  4. 8 module ResponseCache
  5. # Implementation of a file system based cache store.
  6. #
  7. # It stores cached responses in a file under a directory pointed by the +dir+
  8. # variable (defaults to the default temp directory from the OS), in a custom
  9. # format (similar but different from HTTP/1.1 request/response framing).
  10. 8 class FileStore
  11. 8 CRLF = HTTPX::Connection::HTTP1::CRLF
  12. 8 attr_reader :dir
  13. 8 def initialize(dir = Dir.tmpdir)
  14. 72 @dir = Pathname.new(dir).join("httpx-response-cache")
  15. 72 FileUtils.mkdir_p(@dir)
  16. end
  17. 8 def clear
  18. 56 FileUtils.rm_rf(@dir)
  19. end
  20. 8 def get(request)
  21. 280 path = file_path(request)
  22. 280 return unless File.exist?(path)
  23. 144 File.open(path, mode: File::RDONLY | File::BINARY) do |f|
  24. 144 f.flock(File::Constants::LOCK_SH)
  25. 144 read_from_file(request, f)
  26. end
  27. end
  28. 8 def set(request, response)
  29. 88 path = file_path(request)
  30. 88 file_exists = File.exist?(path)
  31. 88 mode = file_exists ? File::RDWR : File::CREAT | File::Constants::WRONLY
  32. 88 File.open(path, mode: mode | File::BINARY) do |f|
  33. 88 f.flock(File::Constants::LOCK_EX)
  34. 88 if file_exists
  35. 8 cached_response = read_from_file(request, f)
  36. 8 if cached_response
  37. 8 next if cached_response == request.cached_response
  38. 8 cached_response.close
  39. 8 f.truncate(0)
  40. 8 f.rewind
  41. end
  42. end
  43. # cache the request headers
  44. 88 f << request.verb << CRLF
  45. 88 f << request.uri << CRLF
  46. 88 request.headers.each do |field, value|
  47. 264 f << field << ":" << value << CRLF
  48. end
  49. 88 f << CRLF
  50. # cache the response
  51. 88 f << response.status << CRLF
  52. 88 f << response.version << CRLF
  53. 88 response.headers.each do |field, value|
  54. 248 f << field << ":" << value << CRLF
  55. end
  56. 88 f << CRLF
  57. 88 response.body.rewind
  58. 88 ::IO.copy_stream(response.body, f)
  59. end
  60. end
  61. 8 private
  62. 8 def file_path(request)
  63. 368 @dir.join(request.response_cache_key)
  64. end
  65. 8 def read_from_file(request, f)
  66. # if it's an empty file
  67. 152 return if f.eof?
  68. # read request data
  69. 152 verb = f.readline.delete_suffix!(CRLF)
  70. 152 uri = f.readline.delete_suffix!(CRLF)
  71. 152 request_headers = {}
  72. 665 while (line = f.readline) != CRLF
  73. 456 line.delete_suffix!(CRLF)
  74. 456 sep_index = line.index(":")
  75. 456 field = line.byteslice(0..(sep_index - 1))
  76. 456 value = line.byteslice((sep_index + 1)..-1)
  77. 399 request_headers[field] = value
  78. end
  79. 152 status = f.readline.delete_suffix!(CRLF)
  80. 152 version = f.readline.delete_suffix!(CRLF)
  81. 152 response_headers = {}
  82. 637 while (line = f.readline) != CRLF
  83. 424 line.delete_suffix!(CRLF)
  84. 424 sep_index = line.index(":")
  85. 424 field = line.byteslice(0..(sep_index - 1))
  86. 424 value = line.byteslice((sep_index + 1)..-1)
  87. 371 response_headers[field] = value
  88. end
  89. 152 original_request = request.options.request_class.new(verb, uri, request.options)
  90. 152 original_request.merge_headers(request_headers)
  91. 152 response = request.options.response_class.new(request, status, version, response_headers)
  92. 152 response.original_request = original_request
  93. 152 ::IO.copy_stream(f, response.body)
  94. 152 response
  95. end
  96. end
  97. end
  98. end

lib/httpx/plugins/response_cache/store.rb

100.0% lines covered

16 relevant lines. 16 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX::Plugins
  3. 8 module ResponseCache
  4. # Implementation of a thread-safe in-memory cache store.
  5. 8 class Store
  6. 8 def initialize
  7. 288 @store = {}
  8. 288 @store_mutex = Thread::Mutex.new
  9. end
  10. 8 def clear
  11. 144 @store_mutex.synchronize { @store.clear }
  12. end
  13. 8 def get(request)
  14. 448 @store_mutex.synchronize do
  15. 448 @store[request.response_cache_key]
  16. end
  17. end
  18. 8 def set(request, response)
  19. 200 @store_mutex.synchronize do
  20. 200 cached_response = @store[request.response_cache_key]
  21. 200 cached_response.close if cached_response
  22. 175 @store[request.response_cache_key] = response
  23. end
  24. end
  25. end
  26. end
  27. end

lib/httpx/plugins/retries.rb

96.88% lines covered

96 relevant lines. 93 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 17 module HTTPX
  3. 17 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when errors happen.
  6. #
  7. # It has a default max number of retries (see *MAX_RETRIES* and the *max_retries* option),
  8. # after which it will return the last response, error or not. It will **not** raise an exception.
  9. #
  10. # It does not retry which are not considered idempotent (see *retry_change_requests* to override).
  11. #
  12. # https://gitlab.com/os85/httpx/wikis/Retries
  13. #
  14. 17 module Retries
  15. 17 MAX_RETRIES = 3
  16. # TODO: pass max_retries in a configure/load block
  17. 17 IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
  18. 2 RETRYABLE_ERRORS = [
  19. 15 IOError,
  20. EOFError,
  21. Errno::ECONNRESET,
  22. Errno::ECONNABORTED,
  23. Errno::EPIPE,
  24. Errno::EINVAL,
  25. Errno::ETIMEDOUT,
  26. Parser::Error,
  27. TLSError,
  28. TimeoutError,
  29. ConnectionError,
  30. Connection::HTTP2::GoawayError,
  31. ].freeze
  32. 17 DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
  33. 17 if ENV.key?("HTTPX_NO_JITTER")
  34. 16 def self.extra_options(options)
  35. 765 options.merge(max_retries: MAX_RETRIES)
  36. end
  37. else
  38. 1 def self.extra_options(options)
  39. 2 options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
  40. end
  41. end
  42. # adds support for the following options:
  43. #
  44. # :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
  45. # :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
  46. # :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
  47. # :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
  48. # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
  49. # (i.e. <tt>->(res) { ... }</tt>).
  50. 17 module OptionsMethods
  51. 17 def option_retry_after(value)
  52. # return early if callable
  53. 208 unless value.respond_to?(:call)
  54. 96 value = Float(value)
  55. 96 raise TypeError, ":retry_after must be positive" unless value.positive?
  56. end
  57. 208 value
  58. end
  59. 17 def option_retry_jitter(value)
  60. # return early if callable
  61. 54 raise TypeError, ":retry_jitter must be callable" unless value.respond_to?(:call)
  62. 54 value
  63. end
  64. 17 def option_max_retries(value)
  65. 2307 num = Integer(value)
  66. 2307 raise TypeError, ":max_retries must be positive" unless num >= 0
  67. 2307 num
  68. end
  69. 17 def option_retry_change_requests(v)
  70. 128 v
  71. end
  72. 17 def option_retry_on(value)
  73. 286 raise TypeError, ":retry_on must be called with the response" unless value.respond_to?(:call)
  74. 286 value
  75. end
  76. end
  77. 17 module InstanceMethods
  78. # returns a `:retries` plugin enabled session with +n+ maximum retries per request setting.
  79. 17 def max_retries(n)
  80. 96 with(max_retries: n)
  81. end
  82. 17 private
  83. 17 def fetch_response(request, selector, options)
  84. 7183760 response = super
  85. 7183760 if response &&
  86. request.retries.positive? &&
  87. repeatable_request?(request, options) &&
  88. (
  89. 95 (
  90. 324 response.is_a?(ErrorResponse) && retryable_error?(response.error)
  91. ) ||
  92. (
  93. 227 options.retry_on && options.retry_on.call(response)
  94. )
  95. )
  96. 483 try_partial_retry(request, response)
  97. 483 log { "failed to get response, #{request.retries} tries to go..." }
  98. 483 request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
  99. 483 request.transition(:idle)
  100. 483 retry_after = options.retry_after
  101. 483 retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
  102. 483 if retry_after
  103. # apply jitter
  104. 96 if (jitter = request.options.retry_jitter)
  105. 16 retry_after = jitter.call(retry_after)
  106. end
  107. 96 retry_start = Utils.now
  108. 96 log { "retrying after #{retry_after} secs..." }
  109. 96 selector.after(retry_after) do
  110. 96 if (response = request.response)
  111. response.finish!
  112. # request has terminated abruptly meanwhile
  113. request.emit(:response, response)
  114. else
  115. 96 log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
  116. 96 send_request(request, selector, options)
  117. end
  118. end
  119. else
  120. 387 send_request(request, selector, options)
  121. end
  122. 426 return
  123. end
  124. 7183277 response
  125. end
  126. # returns whether +request+ can be retried.
  127. 17 def repeatable_request?(request, options)
  128. 1061 IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
  129. end
  130. # returns whether the +ex+ exception happend for a retriable request.
  131. 17 def retryable_error?(ex)
  132. 3177 RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
  133. end
  134. 17 def proxy_error?(request, response, _)
  135. 64 super && !request.retries.positive?
  136. end
  137. #
  138. # Attempt to set the request to perform a partial range request.
  139. # This happens if the peer server accepts byte-range requests, and
  140. # the last response contains some body payload.
  141. #
  142. 17 def try_partial_retry(request, response)
  143. 483 response = response.response if response.is_a?(ErrorResponse)
  144. 483 return unless response
  145. 234 unless response.headers.key?("accept-ranges") &&
  146. response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
  147. 16 (original_body = response.body)
  148. 218 response.body.close
  149. 192 return
  150. end
  151. 16 request.partial_response = response
  152. 16 size = original_body.bytesize
  153. 14 request.headers["range"] = "bytes=#{size}-"
  154. end
  155. end
  156. 17 module RequestMethods
  157. # number of retries left.
  158. 17 attr_accessor :retries
  159. # a response partially received before.
  160. 17 attr_writer :partial_response
  161. # initializes the request instance, sets the number of retries for the request.
  162. 17 def initialize(*args)
  163. 799 super
  164. 799 @retries = @options.max_retries
  165. end
  166. 17 def response=(response)
  167. 1298 if @partial_response
  168. 16 if response.is_a?(Response) && response.status == 206
  169. 16 response.from_partial_response(@partial_response)
  170. else
  171. @partial_response.close
  172. end
  173. 16 @partial_response = nil
  174. end
  175. 1298 super
  176. end
  177. end
  178. 17 module ResponseMethods
  179. 17 def from_partial_response(response)
  180. 16 @status = response.status
  181. 16 @headers = response.headers
  182. 16 @body = response.body
  183. end
  184. end
  185. end
  186. 17 register_plugin :retries, Retries
  187. end
  188. end

lib/httpx/plugins/ssrf_filter.rb

100.0% lines covered

61 relevant lines. 61 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 class ServerSideRequestForgeryError < Error; end
  4. 8 module Plugins
  5. #
  6. # This plugin adds support for preventing Server-Side Request Forgery attacks.
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Server-Side-Request-Forgery-Filter
  9. #
  10. 8 module SsrfFilter
  11. 8 module IPAddrExtensions
  12. 8 refine IPAddr do
  13. 8 def prefixlen
  14. 128 mask_addr = @mask_addr
  15. 128 raise "Invalid mask" if mask_addr.zero?
  16. 387 mask_addr >>= 1 while (mask_addr & 0x1).zero?
  17. 128 length = 0
  18. 381 while mask_addr & 0x1 == 0x1
  19. 1771 length += 1
  20. 1771 mask_addr >>= 1
  21. end
  22. 128 length
  23. end
  24. end
  25. end
  26. 8 using IPAddrExtensions
  27. # https://en.wikipedia.org/wiki/Reserved_IP_addresses
  28. 2 IPV4_BLACKLIST = [
  29. 6 IPAddr.new("0.0.0.0/8"), # Current network (only valid as source address)
  30. IPAddr.new("10.0.0.0/8"), # Private network
  31. IPAddr.new("100.64.0.0/10"), # Shared Address Space
  32. IPAddr.new("127.0.0.0/8"), # Loopback
  33. IPAddr.new("169.254.0.0/16"), # Link-local
  34. IPAddr.new("172.16.0.0/12"), # Private network
  35. IPAddr.new("192.0.0.0/24"), # IETF Protocol Assignments
  36. IPAddr.new("192.0.2.0/24"), # TEST-NET-1, documentation and examples
  37. IPAddr.new("192.88.99.0/24"), # IPv6 to IPv4 relay (includes 2002::/16)
  38. IPAddr.new("192.168.0.0/16"), # Private network
  39. IPAddr.new("198.18.0.0/15"), # Network benchmark tests
  40. IPAddr.new("198.51.100.0/24"), # TEST-NET-2, documentation and examples
  41. IPAddr.new("203.0.113.0/24"), # TEST-NET-3, documentation and examples
  42. IPAddr.new("224.0.0.0/4"), # IP multicast (former Class D network)
  43. IPAddr.new("240.0.0.0/4"), # Reserved (former Class E network)
  44. IPAddr.new("255.255.255.255"), # Broadcast
  45. ].freeze
  46. 3 IPV6_BLACKLIST = ([
  47. 6 IPAddr.new("::1/128"), # Loopback
  48. IPAddr.new("64:ff9b::/96"), # IPv4/IPv6 translation (RFC 6052)
  49. IPAddr.new("100::/64"), # Discard prefix (RFC 6666)
  50. IPAddr.new("2001::/32"), # Teredo tunneling
  51. IPAddr.new("2001:10::/28"), # Deprecated (previously ORCHID)
  52. IPAddr.new("2001:20::/28"), # ORCHIDv2
  53. IPAddr.new("2001:db8::/32"), # Addresses used in documentation and example source code
  54. IPAddr.new("2002::/16"), # 6to4
  55. IPAddr.new("fc00::/7"), # Unique local address
  56. IPAddr.new("fe80::/10"), # Link-local address
  57. IPAddr.new("ff00::/8"), # Multicast
  58. ] + IPV4_BLACKLIST.flat_map do |ipaddr|
  59. 128 prefixlen = ipaddr.prefixlen
  60. 128 ipv4_compatible = ipaddr.ipv4_compat.mask(96 + prefixlen)
  61. 128 ipv4_mapped = ipaddr.ipv4_mapped.mask(80 + prefixlen)
  62. 128 [ipv4_compatible, ipv4_mapped]
  63. end).freeze
  64. 8 class << self
  65. 8 def extra_options(options)
  66. 70 options.merge(allowed_schemes: %w[https http])
  67. end
  68. 8 def unsafe_ip_address?(ipaddr)
  69. 126 range = ipaddr.to_range
  70. 126 return true if range.first != range.last
  71. 142 return IPV6_BLACKLIST.any? { |r| r.include?(ipaddr) } if ipaddr.ipv6?
  72. 1488 IPV4_BLACKLIST.any? { |r| r.include?(ipaddr) } # then it's IPv4
  73. end
  74. end
  75. # adds support for the following options:
  76. #
  77. # :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
  78. 8 module OptionsMethods
  79. 8 def option_allowed_schemes(value)
  80. 78 Array(value)
  81. end
  82. end
  83. 8 module InstanceMethods
  84. 8 def send_requests(*requests)
  85. 86 responses = requests.map do |request|
  86. 86 next if @options.allowed_schemes.include?(request.uri.scheme)
  87. 8 error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
  88. 8 error.set_backtrace(caller)
  89. 8 response = ErrorResponse.new(request, error)
  90. 8 request.emit(:response, response)
  91. 8 response
  92. end
  93. 172 allowed_requests = requests.select { |req| responses[requests.index(req)].nil? }
  94. 86 allowed_responses = super(*allowed_requests)
  95. 86 allowed_responses.each_with_index do |res, idx|
  96. 78 req = allowed_requests[idx]
  97. 68 responses[requests.index(req)] = res
  98. end
  99. 86 responses
  100. end
  101. end
  102. 8 module ConnectionMethods
  103. 8 def initialize(*)
  104. begin
  105. 78 super
  106. 8 rescue ServerSideRequestForgeryError => e
  107. # may raise when IPs are passed as options via :addresses
  108. 16 throw(:resolve_error, e)
  109. end
  110. end
  111. 8 def addresses=(addrs)
  112. 204 addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
  113. 78 addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
  114. 78 raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
  115. 16 super
  116. end
  117. end
  118. end
  119. 8 register_plugin :ssrf_filter, SsrfFilter
  120. end
  121. end

lib/httpx/plugins/stream.rb

97.78% lines covered

90 relevant lines. 88 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 14 module HTTPX
  3. 14 class StreamResponse
  4. 14 attr_reader :request
  5. 14 def initialize(request, session)
  6. 178 @request = request
  7. 178 @options = @request.options
  8. 178 @session = session
  9. 178 @response_enum = nil
  10. 178 @buffered_chunks = []
  11. end
  12. 14 def each(&block)
  13. 298 return enum_for(__method__) unless block
  14. 194 if (response_enum = @response_enum)
  15. 16 @response_enum = nil
  16. # streaming already started, let's finish it
  17. 42 while (chunk = @buffered_chunks.shift)
  18. 16 block.call(chunk)
  19. end
  20. # consume enum til the end
  21. 1 begin
  22. 53 while (chunk = response_enum.next)
  23. 28 block.call(chunk)
  24. end
  25. rescue StopIteration
  26. 16 return
  27. end
  28. end
  29. 178 @request.stream = self
  30. 16 begin
  31. 178 @on_chunk = block
  32. 178 response = @session.request(@request)
  33. 162 response.raise_for_status
  34. ensure
  35. 162 @on_chunk = nil
  36. end
  37. end
  38. 14 def each_line
  39. 108 return enum_for(__method__) unless block_given?
  40. 54 line = "".b
  41. 54 each do |chunk|
  42. 48 line << chunk
  43. 136 while (idx = line.index("\n"))
  44. 54 yield line.byteslice(0..idx - 1)
  45. 54 line = line.byteslice(idx + 1..-1)
  46. end
  47. end
  48. 22 yield line unless line.empty?
  49. end
  50. # This is a ghost method. It's to be used ONLY internally, when processing streams
  51. 14 def on_chunk(chunk)
  52. 318 raise NoMethodError unless @on_chunk
  53. 318 @on_chunk.call(chunk)
  54. end
  55. skipped # :nocov:
  56. skipped def inspect
  57. skipped "#<#{self.class}:#{object_id}>"
  58. skipped end
  59. skipped # :nocov:
  60. 14 def to_s
  61. 16 if @request.response
  62. @request.response.to_s
  63. else
  64. 16 @buffered_chunks.join
  65. end
  66. end
  67. 14 private
  68. 14 def response
  69. 356 @request.response || begin
  70. 32 response_enum = each
  71. 56 while (chunk = response_enum.next)
  72. 32 @buffered_chunks << chunk
  73. 32 break if @request.response
  74. end
  75. 32 @response_enum = response_enum
  76. 32 @request.response
  77. end
  78. end
  79. 14 def respond_to_missing?(meth, include_private)
  80. 24 if (response = @request.response)
  81. response.respond_to_missing?(meth, include_private)
  82. else
  83. 24 @options.response_class.method_defined?(meth) || (include_private && @options.response_class.private_method_defined?(meth))
  84. end || super
  85. end
  86. 14 def method_missing(meth, *args, **kwargs, &block)
  87. 178 return super unless response.respond_to?(meth)
  88. 178 response.__send__(meth, *args, **kwargs, &block)
  89. end
  90. end
  91. 14 module Plugins
  92. #
  93. # This plugin adds support for streaming a response (useful for i.e. "text/event-stream" payloads).
  94. #
  95. # https://gitlab.com/os85/httpx/wikis/Stream
  96. #
  97. 14 module Stream
  98. 14 def self.extra_options(options)
  99. 308 options.merge(timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 })
  100. end
  101. 14 module InstanceMethods
  102. 14 def request(*args, stream: false, **options)
  103. 490 return super(*args, **options) unless stream
  104. 194 requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
  105. 194 raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
  106. 178 request = requests.first
  107. 178 StreamResponse.new(request, self)
  108. end
  109. end
  110. 14 module RequestMethods
  111. 14 attr_accessor :stream
  112. end
  113. 14 module ResponseMethods
  114. 14 def stream
  115. 290 request = @request.root_request if @request.respond_to?(:root_request)
  116. 290 request ||= @request
  117. 290 request.stream
  118. end
  119. end
  120. 14 module ResponseBodyMethods
  121. 14 def initialize(*)
  122. 290 super
  123. 290 @stream = @response.stream
  124. end
  125. 14 def write(chunk)
  126. 448 return super unless @stream
  127. 364 return 0 if chunk.empty?
  128. 318 chunk = decode_chunk(chunk)
  129. 318 @stream.on_chunk(chunk.dup)
  130. 302 chunk.size
  131. end
  132. 14 private
  133. 14 def transition(*)
  134. 108 return if @stream
  135. 108 super
  136. end
  137. end
  138. end
  139. 14 register_plugin :stream, Stream
  140. end
  141. end

lib/httpx/plugins/stream_bidi.rb

98.36% lines covered

61 relevant lines. 60 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. # Extension of the Connection::HTTP2 class, which adds functionality to
  5. # deal with a request that can't be drained and must be interleaved with
  6. # the response streams.
  7. #
  8. # The streams keeps send DATA frames while there's data; when they're ain't,
  9. # the stream is kept open; it must be explicitly closed by the end user.
  10. #
  11. 8 class HTTP2Bidi < Connection::HTTP2
  12. 8 private
  13. 8 def handle_stream(stream, request)
  14. 8 request.on(:body) do
  15. 48 next unless request.headers_sent
  16. 40 handle(request, stream)
  17. end
  18. 8 super
  19. end
  20. # when there ain't more chunks, it makes the buffer as full.
  21. 8 def send_chunk(request, stream, chunk, next_chunk)
  22. 48 super
  23. 48 return if next_chunk
  24. 48 request.transition(:waiting_for_chunk)
  25. 48 throw(:buffer_full)
  26. end
  27. 8 def end_stream?(request, *)
  28. 48 request.closed?
  29. end
  30. end
  31. #
  32. # This plugin adds support for bidirectional HTTP/2 streams.
  33. #
  34. # https://gitlab.com/os85/httpx/wikis/StreamBidi
  35. #
  36. 8 module StreamBidi
  37. 8 class << self
  38. 8 def load_dependencies(klass)
  39. 8 klass.plugin(:stream)
  40. end
  41. 8 def extra_options(options)
  42. 8 options.merge(fallback_protocol: "h2")
  43. end
  44. end
  45. 8 module RequestMethods
  46. 8 attr_accessor :headers_sent
  47. 8 def initialize(*)
  48. 8 super
  49. 8 @headers_sent = false
  50. 8 @closed = false
  51. end
  52. 8 def closed?
  53. 48 @closed
  54. end
  55. 8 def can_buffer?
  56. 94 super && @state != :waiting_for_chunk
  57. end
  58. 8 def transition(nextstate)
  59. 184 headers_sent = @headers_sent
  60. 161 case nextstate
  61. when :waiting_for_chunk
  62. 48 return unless @state == :body
  63. when :body
  64. 77 case @state
  65. when :headers
  66. 8 headers_sent = true
  67. when :waiting_for_chunk
  68. # HACK: to allow super to pass through
  69. 40 @state = :headers
  70. end
  71. end
  72. 184 super.tap do
  73. # delay setting this up until after the first transition to :body
  74. 184 @headers_sent = headers_sent
  75. end
  76. end
  77. 8 def <<(chunk)
  78. 40 if @drainer
  79. 40 @body.clear if @body.respond_to?(:clear)
  80. 40 @drainer = nil
  81. end
  82. 40 @body << chunk
  83. 40 transition(:body)
  84. end
  85. 8 def close
  86. 8 @closed = true
  87. # last chunk to send which ends the stream
  88. 8 self << ""
  89. end
  90. end
  91. 8 module RequestBodyMethods
  92. 8 def initialize(*, **)
  93. 8 super
  94. 8 @headers.delete("content-length")
  95. end
  96. 8 def empty?
  97. 56 false
  98. end
  99. end
  100. 8 module ConnectionMethods
  101. 8 def parser_type(protocol)
  102. 8 return HTTP2Bidi if protocol == "h2"
  103. super
  104. end
  105. end
  106. end
  107. 8 register_plugin :stream_bidi, StreamBidi
  108. end
  109. end

lib/httpx/plugins/upgrade.rb

100.0% lines covered

34 relevant lines. 34 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin helps negotiating a new protocol from an HTTP/1.1 connection, via the
  6. # Upgrade header.
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Upgrade
  9. #
  10. 8 module Upgrade
  11. 8 class << self
  12. 8 def configure(klass)
  13. 32 klass.plugin(:"upgrade/h2")
  14. end
  15. 8 def extra_options(options)
  16. 32 options.merge(upgrade_handlers: {})
  17. end
  18. end
  19. 8 module OptionsMethods
  20. 8 def option_upgrade_handlers(value)
  21. 88 raise TypeError, ":upgrade_handlers must be a Hash" unless value.is_a?(Hash)
  22. 88 value
  23. end
  24. end
  25. 8 module InstanceMethods
  26. 8 def fetch_response(request, selector, options)
  27. 290 response = super
  28. 290 if response
  29. 94 return response unless response.is_a?(Response)
  30. 94 return response unless response.headers.key?("upgrade")
  31. 38 upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
  32. 38 return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
  33. 38 protocol_handler = options.upgrade_handlers[upgrade_protocol]
  34. 38 return response unless protocol_handler
  35. 38 log { "upgrading to #{upgrade_protocol}..." }
  36. 38 connection = find_connection(request.uri, selector, options)
  37. # do not upgrade already upgraded connections
  38. 38 return if connection.upgrade_protocol == upgrade_protocol
  39. 32 protocol_handler.call(connection, request, response)
  40. # keep in the loop if the server is switching, unless
  41. # the connection has been hijacked, in which case you want
  42. # to terminante immediately
  43. 32 return if response.status == 101 && !connection.hijacked
  44. end
  45. 212 response
  46. end
  47. end
  48. 8 module ConnectionMethods
  49. 8 attr_reader :upgrade_protocol, :hijacked
  50. 8 def hijack_io
  51. 8 @hijacked = true
  52. # connection is taken away from selector and not given back to the pool.
  53. 8 @current_session.deselect_connection(self, @current_selector, true)
  54. end
  55. end
  56. end
  57. 8 register_plugin(:upgrade, Upgrade)
  58. end
  59. end

lib/httpx/plugins/upgrade/h2.rb

91.67% lines covered

24 relevant lines. 22 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for upgrading an HTTP/1.1 connection to HTTP/2
  6. # via an Upgrade: h2 response declaration
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2
  9. #
  10. 8 module H2
  11. 8 class << self
  12. 8 def extra_options(options)
  13. 32 options.merge(upgrade_handlers: options.upgrade_handlers.merge("h2" => self))
  14. end
  15. 8 def call(connection, _request, _response)
  16. 8 connection.upgrade_to_h2
  17. end
  18. end
  19. 8 module ConnectionMethods
  20. 8 using URIExtensions
  21. 8 def upgrade_to_h2
  22. 8 prev_parser = @parser
  23. 8 if prev_parser
  24. 8 prev_parser.reset
  25. 7 @inflight -= prev_parser.requests.size
  26. end
  27. 8 @parser = Connection::HTTP2.new(@write_buffer, @options)
  28. 8 set_parser_callbacks(@parser)
  29. 8 @upgrade_protocol = "h2"
  30. # what's happening here:
  31. # a deviation from the state machine is done to perform the actions when a
  32. # connection is closed, without transitioning, so the connection is kept in the pool.
  33. # the state is reset to initial, so that the socket reconnect works out of the box,
  34. # while the parser is already here.
  35. 8 purge_after_closed
  36. 8 transition(:idle)
  37. 8 prev_parser.requests.each do |req|
  38. req.transition(:idle)
  39. send(req)
  40. end
  41. end
  42. end
  43. end
  44. 8 register_plugin(:"upgrade/h2", H2)
  45. end
  46. end

lib/httpx/plugins/webdav.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin implements convenience methods for performing WEBDAV requests.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/WebDav
  8. #
  9. 8 module WebDav
  10. 8 def self.configure(klass)
  11. 96 klass.plugin(:xml)
  12. end
  13. 8 module InstanceMethods
  14. 8 def copy(src, dest)
  15. 16 request("COPY", src, headers: { "destination" => @options.origin.merge(dest) })
  16. end
  17. 8 def move(src, dest)
  18. 16 request("MOVE", src, headers: { "destination" => @options.origin.merge(dest) })
  19. end
  20. 8 def lock(path, timeout: nil, &blk)
  21. 48 headers = {}
  22. 42 headers["timeout"] = if timeout && timeout.positive?
  23. 16 "Second-#{timeout}"
  24. else
  25. 32 "Infinite, Second-4100000000"
  26. end
  27. 48 xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" \
  28. "<D:lockinfo xmlns:D=\"DAV:\">" \
  29. "<D:lockscope><D:exclusive/></D:lockscope>" \
  30. "<D:locktype><D:write/></D:locktype>" \
  31. "<D:owner>null</D:owner>" \
  32. "</D:lockinfo>"
  33. 48 response = request("LOCK", path, headers: headers, xml: xml)
  34. 48 return response unless response.is_a?(Response)
  35. 48 return response unless blk && response.status == 200
  36. 16 lock_token = response.headers["lock-token"]
  37. 1 begin
  38. 16 blk.call(response)
  39. ensure
  40. 16 unlock(path, lock_token)
  41. end
  42. 16 response
  43. end
  44. 8 def unlock(path, lock_token)
  45. 32 request("UNLOCK", path, headers: { "lock-token" => lock_token })
  46. end
  47. 8 def mkcol(dir)
  48. 16 request("MKCOL", dir)
  49. end
  50. 8 def propfind(path, xml = nil)
  51. 64 body = case xml
  52. when :acl
  53. 16 '<?xml version="1.0" encoding="utf-8" ?><D:propfind xmlns:D="DAV:"><D:prop><D:owner/>' \
  54. "<D:supported-privilege-set/><D:current-user-privilege-set/><D:acl/></D:prop></D:propfind>"
  55. when nil
  56. 32 '<?xml version="1.0" encoding="utf-8"?><DAV:propfind xmlns:DAV="DAV:"><DAV:allprop/></DAV:propfind>'
  57. else
  58. 16 xml
  59. end
  60. 64 request("PROPFIND", path, headers: { "depth" => "1" }, xml: body)
  61. end
  62. 8 def proppatch(path, xml)
  63. 6 body = "<?xml version=\"1.0\"?>" \
  64. 12 "<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"
  65. 16 request("PROPPATCH", path, xml: body)
  66. end
  67. # %i[ orderpatch acl report search]
  68. end
  69. end
  70. 8 register_plugin(:webdav, WebDav)
  71. end
  72. end

lib/httpx/plugins/xml.rb

100.0% lines covered

34 relevant lines. 34 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin supports request XML encoding/response decoding using the nokogiri gem.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/XML
  8. #
  9. 8 module XML
  10. 8 MIME_TYPES = %r{\b(application|text)/(.+\+)?xml\b}.freeze
  11. 8 module Transcoder
  12. 8 module_function
  13. 8 class Encoder
  14. 8 def initialize(xml)
  15. 160 @raw = xml
  16. end
  17. 8 def content_type
  18. 160 charset = @raw.respond_to?(:encoding) && @raw.encoding ? @raw.encoding.to_s.downcase : "utf-8"
  19. 160 "application/xml; charset=#{charset}"
  20. end
  21. 8 def bytesize
  22. 512 @raw.to_s.bytesize
  23. end
  24. 8 def to_s
  25. 160 @raw.to_s
  26. end
  27. end
  28. 8 def encode(xml)
  29. 160 Encoder.new(xml)
  30. end
  31. 8 def decode(response)
  32. 24 content_type = response.content_type.mime_type
  33. 24 raise HTTPX::Error, "invalid form mime type (#{content_type})" unless MIME_TYPES.match?(content_type)
  34. 24 Nokogiri::XML.method(:parse)
  35. end
  36. end
  37. 8 class << self
  38. 8 def load_dependencies(*)
  39. 144 require "nokogiri"
  40. end
  41. end
  42. 8 module ResponseMethods
  43. # decodes the response payload into a Nokogiri::XML::Node object **if** the payload is valid
  44. # "application/xml" (requires the "nokogiri" gem).
  45. 8 def xml
  46. 16 decode(Transcoder)
  47. end
  48. end
  49. 8 module RequestBodyClassMethods
  50. # ..., xml: Nokogiri::XML::Node #=> xml encoder
  51. 8 def initialize_body(params)
  52. 592 if (xml = params.delete(:xml))
  53. # @type var xml: Nokogiri::XML::Node | String
  54. 140 return Transcoder.encode(xml)
  55. end
  56. 432 super
  57. end
  58. end
  59. end
  60. 8 register_plugin(:xml, XML)
  61. end
  62. end

lib/httpx/pmatch_extensions.rb

100.0% lines covered

17 relevant lines. 17 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module ResponsePatternMatchExtensions
  4. 27 def deconstruct
  5. 41 [@status, @headers, @body]
  6. end
  7. 27 def deconstruct_keys(_keys)
  8. 70 { status: @status, headers: @headers, body: @body }
  9. end
  10. end
  11. 27 module ErrorResponsePatternMatchExtensions
  12. 27 def deconstruct
  13. 11 [@error]
  14. end
  15. 27 def deconstruct_keys(_keys)
  16. 35 { error: @error }
  17. end
  18. end
  19. 27 module HeadersPatternMatchExtensions
  20. 27 def deconstruct
  21. 7 to_a
  22. end
  23. end
  24. 27 Headers.include HeadersPatternMatchExtensions
  25. 27 Response.include ResponsePatternMatchExtensions
  26. 27 ErrorResponse.include ErrorResponsePatternMatchExtensions
  27. end

lib/httpx/pool.rb

98.9% lines covered

91 relevant lines. 90 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "httpx/selector"
  3. 27 require "httpx/connection"
  4. 27 require "httpx/resolver"
  5. 27 module HTTPX
  6. 27 class Pool
  7. 27 using ArrayExtensions::FilterMap
  8. 27 using URIExtensions
  9. 27 POOL_TIMEOUT = 5
  10. # Sets up the connection pool with the given +options+, which can be the following:
  11. #
  12. # :max_connections:: the maximum number of connections held in the pool.
  13. # :max_connections_per_origin :: the maximum number of connections held in the pool pointing to a given origin.
  14. # :pool_timeout :: the number of seconds to wait for a connection to a given origin (before raising HTTPX::PoolTimeoutError)
  15. #
  16. 27 def initialize(options)
  17. 11079 @max_connections = options.fetch(:max_connections, Float::INFINITY)
  18. 11079 @max_connections_per_origin = options.fetch(:max_connections_per_origin, Float::INFINITY)
  19. 11079 @pool_timeout = options.fetch(:pool_timeout, POOL_TIMEOUT)
  20. 17177 @resolvers = Hash.new { |hs, resolver_type| hs[resolver_type] = [] }
  21. 11079 @resolver_mtx = Thread::Mutex.new
  22. 11079 @connections = []
  23. 11079 @connection_mtx = Thread::Mutex.new
  24. 11079 @connections_counter = 0
  25. 11079 @max_connections_cond = ConditionVariable.new
  26. 11079 @origin_counters = Hash.new(0)
  27. 16459 @origin_conds = Hash.new { |hs, orig| hs[orig] = ConditionVariable.new }
  28. end
  29. # connections returned by this function are not expected to return to the connection pool.
  30. 27 def pop_connection
  31. 11594 @connection_mtx.synchronize do
  32. 11594 drop_connection
  33. end
  34. end
  35. # opens a connection to the IP reachable through +uri+.
  36. # Many hostnames are reachable through the same IP, so we try to
  37. # maximize pipelining by opening as few connections as possible.
  38. #
  39. 27 def checkout_connection(uri, options)
  40. 8051 return checkout_new_connection(uri, options) if options.io
  41. 7985 @connection_mtx.synchronize do
  42. 7985 acquire_connection(uri, options) || begin
  43. 7342 if @connections_counter == @max_connections
  44. # this takes precedence over per-origin
  45. 16 @max_connections_cond.wait(@connection_mtx, @pool_timeout)
  46. 16 acquire_connection(uri, options) || begin
  47. 12 if @connections_counter == @max_connections
  48. # if no matching usable connection was found, the pool will make room and drop a closed connection. if none is found,
  49. # this means that all of them are persistent or being used, so raise a timeout error.
  50. 10 conn = @connections.find { |c| c.state == :closed }
  51. 1 raise PoolTimeoutError.new(@pool_timeout,
  52. 9 "Timed out after #{@pool_timeout} seconds while waiting for a connection") unless conn
  53. 1 drop_connection(conn)
  54. end
  55. end
  56. end
  57. 7334 if @origin_counters[uri.origin] == @max_connections_per_origin
  58. 16 @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
  59. 16 return acquire_connection(uri, options) ||
  60. raise(PoolTimeoutError.new(@pool_timeout,
  61. 1 "Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
  62. end
  63. 6507 @connections_counter += 1
  64. 7318 @origin_counters[uri.origin] += 1
  65. 7318 checkout_new_connection(uri, options)
  66. end
  67. end
  68. end
  69. 27 def checkin_connection(connection)
  70. 7932 return if connection.options.io
  71. 7866 @connection_mtx.synchronize do
  72. 7866 @connections << connection
  73. 7866 @max_connections_cond.signal
  74. 7866 @origin_conds[connection.origin.to_s].signal
  75. end
  76. end
  77. 27 def checkout_mergeable_connection(connection)
  78. 7313 return if connection.options.io
  79. 7313 @connection_mtx.synchronize do
  80. 7313 idx = @connections.find_index do |ch|
  81. 240 ch != connection && ch.mergeable?(connection)
  82. end
  83. 7313 if idx
  84. @connections_counter -= 1
  85. @connections.delete_at(idx)
  86. end
  87. end
  88. end
  89. 27 def reset_resolvers
  90. 14360 @resolver_mtx.synchronize { @resolvers.clear }
  91. end
  92. 27 def checkout_resolver(options)
  93. 7134 resolver_type = options.resolver_class
  94. 7134 resolver_type = Resolver.resolver_for(resolver_type)
  95. 7134 @resolver_mtx.synchronize do
  96. 7134 resolvers = @resolvers[resolver_type]
  97. 7134 idx = resolvers.find_index do |res|
  98. 31 res.options == options
  99. end
  100. 7134 resolvers.delete_at(idx) if idx
  101. end || checkout_new_resolver(resolver_type, options)
  102. end
  103. 27 def checkin_resolver(resolver)
  104. 358 @resolver_mtx.synchronize do
  105. 358 resolvers = @resolvers[resolver.class]
  106. 358 resolver = resolver.multi
  107. 358 resolvers << resolver unless resolvers.include?(resolver)
  108. end
  109. end
  110. skipped # :nocov:
  111. skipped def inspect
  112. skipped "#<#{self.class}:#{object_id} " \
  113. skipped "@max_connections_per_origin=#{@max_connections_per_origin} " \
  114. skipped "@pool_timeout=#{@pool_timeout} " \
  115. skipped "@connections=#{@connections.size}>"
  116. skipped end
  117. skipped # :nocov:
  118. 27 private
  119. 27 def acquire_connection(uri, options)
  120. 8017 idx = @connections.find_index do |connection|
  121. 872 connection.match?(uri, options)
  122. end
  123. 8017 return unless idx
  124. 575 @connections_counter += 1
  125. 655 @connections.delete_at(idx)
  126. end
  127. 27 def checkout_new_connection(uri, options)
  128. 7384 options.connection_class.new(uri, options)
  129. end
  130. 27 def checkout_new_resolver(resolver_type, options)
  131. 7107 if resolver_type.multi?
  132. 7082 Resolver::Multi.new(resolver_type, options)
  133. else
  134. 25 resolver_type.new(options)
  135. end
  136. end
  137. # drops and returns the +connection+ from the connection pool; if +connection+ is <tt>nil</tt> (default),
  138. # the first available connection from the pool will be dropped.
  139. 27 def drop_connection(connection = nil)
  140. 11595 if connection
  141. 1 @connections.delete(connection)
  142. else
  143. 11594 connection = @connections.shift
  144. 11594 return unless connection
  145. end
  146. 3899 @connections_counter -= 1
  147. 4415 @origin_conds.delete(connection.origin) if (@origin_counters[connection.origin.to_s] -= 1).zero?
  148. 4415 connection
  149. end
  150. end
  151. end

lib/httpx/punycode.rb

100.0% lines covered

9 relevant lines. 9 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Punycode
  4. 27 module_function
  5. begin
  6. 27 require "idnx"
  7. 26 def encode_hostname(hostname)
  8. 32 Idnx.to_punycode(hostname)
  9. end
  10. rescue LoadError
  11. 1 def encode_hostname(hostname)
  12. 1 warn "#{hostname} cannot be converted to punycode. Install the " \
  13. "\"idnx\" gem: https://github.com/HoneyryderChuck/idnx"
  14. 1 hostname
  15. end
  16. end
  17. end
  18. end

lib/httpx/request.rb

100.0% lines covered

132 relevant lines. 132 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "delegate"
  3. 27 require "forwardable"
  4. 27 module HTTPX
  5. # Defines how an HTTP request is handled internally, both in terms of making attributes accessible,
  6. # as well as maintaining the state machine which manages streaming the request onto the wire.
  7. 27 class Request
  8. 27 extend Forwardable
  9. 27 include Callbacks
  10. 27 using URIExtensions
  11. 27 ALLOWED_URI_SCHEMES = %w[https http].freeze
  12. # default value used for "user-agent" header, when not overridden.
  13. 27 USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
  14. # the upcased string HTTP verb for this request.
  15. 27 attr_reader :verb
  16. # the absolute URI object for this request.
  17. 27 attr_reader :uri
  18. # an HTTPX::Headers object containing the request HTTP headers.
  19. 27 attr_reader :headers
  20. # an HTTPX::Request::Body object containing the request body payload (or +nil+, whenn there is none).
  21. 27 attr_reader :body
  22. # a symbol describing which frame is currently being flushed.
  23. 27 attr_reader :state
  24. # an HTTPX::Options object containing request options.
  25. 27 attr_reader :options
  26. # the corresponding HTTPX::Response object, when there is one.
  27. 27 attr_reader :response
  28. # Exception raised during enumerable body writes.
  29. 27 attr_reader :drain_error
  30. # The IP address from the peer server.
  31. 27 attr_accessor :peer_address
  32. 27 attr_writer :persistent
  33. 27 attr_reader :active_timeouts
  34. # will be +true+ when request body has been completely flushed.
  35. 27 def_delegator :@body, :empty?
  36. # closes the body
  37. 27 def_delegator :@body, :close
  38. # initializes the instance with the given +verb+ (an upppercase String, ex. 'GEt'),
  39. # an absolute or relative +uri+ (either as String or URI::HTTP object), the
  40. # request +options+ (instance of HTTPX::Options) and an optional Hash of +params+.
  41. #
  42. # Besides any of the options documented in HTTPX::Options (which would override or merge with what
  43. # +options+ sets), it accepts also the following:
  44. #
  45. # :params :: hash or array of key-values which will be encoded and set in the query string of request uris.
  46. # :body :: to be encoded in the request body payload. can be a String, an IO object (i.e. a File), or an Enumerable.
  47. # :form :: hash of array of key-values which will be form-urlencoded- or multipart-encoded in requests body payload.
  48. # :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
  49. # :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
  50. #
  51. # :body, :form, :json and :xml are all mutually exclusive, i.e. only one of them gets picked up.
  52. 27 def initialize(verb, uri, options, params = EMPTY_HASH)
  53. 10216 @verb = verb.to_s.upcase
  54. 10216 @uri = Utils.to_uri(uri)
  55. 10215 @headers = options.headers.dup
  56. 10215 merge_headers(params.delete(:headers)) if params.key?(:headers)
  57. 10215 @headers["user-agent"] ||= USER_AGENT
  58. 10215 @headers["accept"] ||= "*/*"
  59. # forego compression in the Range request case
  60. 10215 if @headers.key?("range")
  61. 8 @headers.delete("accept-encoding")
  62. else
  63. 10207 @headers["accept-encoding"] ||= options.supported_compression_formats
  64. end
  65. 10215 @query_params = params.delete(:params) if params.key?(:params)
  66. 10215 @body = options.request_body_class.new(@headers, options, **params)
  67. 10207 @options = @body.options
  68. 10207 if @uri.relative? || @uri.host.nil?
  69. 600 origin = @options.origin
  70. 600 raise(Error, "invalid URI: #{@uri}") unless origin
  71. 576 base_path = @options.base_path
  72. 576 @uri = origin.merge("#{base_path}#{@uri}")
  73. end
  74. 10183 raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
  75. 10171 @state = :idle
  76. 10171 @response = nil
  77. 10171 @peer_address = nil
  78. 10171 @ping = false
  79. 10171 @persistent = @options.persistent
  80. 10171 @active_timeouts = []
  81. end
  82. # whether request has been buffered with a ping
  83. 27 def ping?
  84. 679 @ping
  85. end
  86. # marks the request as having been buffered with a ping
  87. 27 def ping!
  88. 20 @ping = true
  89. end
  90. # the read timeout defined for this request.
  91. 27 def read_timeout
  92. 19114 @options.timeout[:read_timeout]
  93. end
  94. # the write timeout defined for this request.
  95. 27 def write_timeout
  96. 19114 @options.timeout[:write_timeout]
  97. end
  98. # the request timeout defined for this request.
  99. 27 def request_timeout
  100. 18830 @options.timeout[:request_timeout]
  101. end
  102. 27 def persistent?
  103. 4884 @persistent
  104. end
  105. # if the request contains trailer headers
  106. 27 def trailers?
  107. 3118 defined?(@trailers)
  108. end
  109. # returns an instance of HTTPX::Headers containing the trailer headers
  110. 27 def trailers
  111. 88 @trailers ||= @options.headers_class.new
  112. end
  113. # returns +:r+ or +:w+, depending on whether the request is waiting for a response or flushing.
  114. 27 def interests
  115. 25542 return :r if @state == :done || @state == :expect
  116. 2930 :w
  117. end
  118. 27 def can_buffer?
  119. 26095 @state != :done
  120. end
  121. # merges +h+ into the instance of HTTPX::Headers of the request.
  122. 27 def merge_headers(h)
  123. 1091 @headers = @headers.merge(h)
  124. end
  125. # the URI scheme of the request +uri+.
  126. 27 def scheme
  127. 3675 @uri.scheme
  128. end
  129. # sets the +response+ on this request.
  130. 27 def response=(response)
  131. 9534 return unless response
  132. 9534 if response.is_a?(Response) && response.status < 200
  133. # deal with informational responses
  134. 160 if response.status == 100 && @headers.key?("expect")
  135. 136 @informational_status = response.status
  136. 136 return
  137. end
  138. # 103 Early Hints advertises resources in document to browsers.
  139. # not very relevant for an HTTP client, discard.
  140. 24 return if response.status >= 103
  141. end
  142. 9398 @response = response
  143. 9398 emit(:response_started, response)
  144. end
  145. # returnns the URI path of the request +uri+.
  146. 27 def path
  147. 8709 path = uri.path.dup
  148. 8709 path = +"" if path.nil?
  149. 8709 path << "/" if path.empty?
  150. 8709 path << "?#{query}" unless query.empty?
  151. 8709 path
  152. end
  153. # returs the URI authority of the request.
  154. #
  155. # session.build_request("GET", "https://google.com/query").authority #=> "google.com"
  156. # session.build_request("GET", "http://internal:3182/a").authority #=> "internal:3182"
  157. 27 def authority
  158. 8799 @uri.authority
  159. end
  160. # returs the URI origin of the request.
  161. #
  162. # session.build_request("GET", "https://google.com/query").authority #=> "https://google.com"
  163. # session.build_request("GET", "http://internal:3182/a").authority #=> "http://internal:3182"
  164. 27 def origin
  165. 3955 @uri.origin
  166. end
  167. # returs the URI query string of the request (when available).
  168. #
  169. # session.build_request("GET", "https://search.com").query #=> ""
  170. # session.build_request("GET", "https://search.com?q=a").query #=> "q=a"
  171. # session.build_request("GET", "https://search.com", params: { q: "a"}).query #=> "q=a"
  172. # session.build_request("GET", "https://search.com?q=a", params: { foo: "bar"}).query #=> "q=a&foo&bar"
  173. 27 def query
  174. 9672 return @query if defined?(@query)
  175. 8135 query = []
  176. 8135 if (q = @query_params) && !q.empty?
  177. 152 query << Transcoder::Form.encode(q)
  178. end
  179. 8135 query << @uri.query if @uri.query
  180. 8135 @query = query.join("&")
  181. end
  182. # consumes and returns the next available chunk of request body that can be sent
  183. 27 def drain_body
  184. 9188 return nil if @body.nil?
  185. 9188 @drainer ||= @body.each
  186. 9188 chunk = @drainer.next.dup
  187. 6067 emit(:body_chunk, chunk)
  188. 6067 chunk
  189. rescue StopIteration
  190. 3109 nil
  191. rescue StandardError => e
  192. 12 @drain_error = e
  193. 12 nil
  194. end
  195. skipped # :nocov:
  196. skipped def inspect
  197. skipped "#<#{self.class}:#{object_id} " \
  198. skipped "#{@verb} " \
  199. skipped "#{uri} " \
  200. skipped "@headers=#{@headers} " \
  201. skipped "@body=#{@body}>"
  202. skipped end
  203. skipped # :nocov:
  204. # moves on to the +nextstate+ of the request state machine (when all preconditions are met)
  205. 27 def transition(nextstate)
  206. 36926 case nextstate
  207. when :idle
  208. 738 @body.rewind
  209. 738 @ping = false
  210. 738 @response = nil
  211. 738 @drainer = nil
  212. 738 @active_timeouts.clear
  213. when :headers
  214. 11215 return unless @state == :idle
  215. when :body
  216. 11239 return unless @state == :headers ||
  217. @state == :expect
  218. 9121 if @headers.key?("expect")
  219. 516 if @informational_status && @informational_status == 100
  220. # check for 100 Continue response, and deallocate the var
  221. # if @informational_status == 100
  222. # @response = nil
  223. # end
  224. else
  225. 388 return if @state == :expect # do not re-set it
  226. 144 nextstate = :expect
  227. end
  228. end
  229. when :trailers
  230. 9065 return unless @state == :body
  231. when :done
  232. 9073 return if @state == :expect
  233. end
  234. 35726 @state = nextstate
  235. 35726 emit(@state, self)
  236. 14352 nil
  237. end
  238. # whether the request supports the 100-continue handshake and already processed the 100 response.
  239. 27 def expects?
  240. 8232 @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
  241. end
  242. 27 def set_timeout_callback(event, &callback)
  243. 96016 clb = once(event, &callback)
  244. # reset timeout callbacks when requests get rerouted to a different connection
  245. 96016 once(:idle) do
  246. 3554 callbacks(event).delete(clb)
  247. end
  248. end
  249. end
  250. end
  251. 27 require_relative "request/body"

lib/httpx/request/body.rb

100.0% lines covered

66 relevant lines. 66 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # Implementation of the HTTP Request body as a delegator which iterates (responds to +each+) payload chunks.
  4. 27 class Request::Body < SimpleDelegator
  5. 27 class << self
  6. 27 def new(_, options, body: nil, **params)
  7. 10223 if body.is_a?(self)
  8. # request derives its options from body
  9. 16 body.options = options.merge(params)
  10. 14 return body
  11. end
  12. 10207 super
  13. end
  14. end
  15. 27 attr_accessor :options
  16. # inits the instance with the request +headers+, +options+ and +params+, which contain the payload definition.
  17. # it wraps the given body with the appropriate encoder on initialization.
  18. #
  19. # ..., json: { foo: "bar" }) #=> json encoder
  20. # ..., form: { foo: "bar" }) #=> form urlencoded encoder
  21. # ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
  22. # ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
  23. # ..., form: { body: "bla") }) #=> raw data encoder
  24. 27 def initialize(h, options, **params)
  25. 10207 @headers = h
  26. 10207 @body = self.class.initialize_body(params)
  27. 10207 @options = options.merge(params)
  28. 10207 if @body
  29. 3188 if @options.compress_request_body && @headers.key?("content-encoding")
  30. 96 @headers.get("content-encoding").each do |encoding|
  31. 96 @body = self.class.initialize_deflater_body(@body, encoding)
  32. end
  33. end
  34. 3188 @headers["content-type"] ||= @body.content_type
  35. 3188 @headers["content-length"] = @body.bytesize unless unbounded_body?
  36. end
  37. 10199 super(@body)
  38. end
  39. # consumes and yields the request payload in chunks.
  40. 27 def each(&block)
  41. 6490 return enum_for(__method__) unless block
  42. 3249 return if @body.nil?
  43. 3177 body = stream(@body)
  44. 3177 if body.respond_to?(:read)
  45. 5410 while (chunk = body.read(16_384))
  46. 3363 block.call(chunk)
  47. end
  48. # TODO: use copy_stream once bug is resolved: https://bugs.ruby-lang.org/issues/21131
  49. # ::IO.copy_stream(body, ProcIO.new(block))
  50. 1799 elsif body.respond_to?(:each)
  51. 462 body.each(&block)
  52. else
  53. 1338 block[body.to_s]
  54. end
  55. end
  56. 27 def close
  57. 459 @body.close if @body.respond_to?(:close)
  58. end
  59. # if the +@body+ is rewindable, it rewinnds it.
  60. 27 def rewind
  61. 802 return if empty?
  62. 160 @body.rewind if @body.respond_to?(:rewind)
  63. end
  64. # return +true+ if the +body+ has been fully drained (or does nnot exist).
  65. 27 def empty?
  66. 19462 return true if @body.nil?
  67. 8677 return false if chunked?
  68. 8581 @body.bytesize.zero?
  69. end
  70. # returns the +@body+ payload size in bytes.
  71. 27 def bytesize
  72. 3464 return 0 if @body.nil?
  73. 128 @body.bytesize
  74. end
  75. # sets the body to yield using chunked trannsfer encoding format.
  76. 27 def stream(body)
  77. 3177 return body unless chunked?
  78. 96 Transcoder::Chunker.encode(body.enum_for(:each))
  79. end
  80. # returns whether the body yields infinitely.
  81. 27 def unbounded_body?
  82. 3712 return @unbounded_body if defined?(@unbounded_body)
  83. 3260 @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
  84. end
  85. # returns whether the chunked transfer encoding header is set.
  86. 27 def chunked?
  87. 20144 @headers["transfer-encoding"] == "chunked"
  88. end
  89. # sets the chunked transfer encoding header.
  90. 27 def chunk!
  91. 32 @headers.add("transfer-encoding", "chunked")
  92. end
  93. skipped # :nocov:
  94. skipped def inspect
  95. skipped "#<#{self.class}:#{object_id} " \
  96. skipped "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
  97. skipped end
  98. skipped # :nocov:
  99. 27 class << self
  100. 27 def initialize_body(params)
  101. 10047 if (body = params.delete(:body))
  102. # @type var body: bodyIO
  103. 1440 Transcoder::Body.encode(body)
  104. 8607 elsif (form = params.delete(:form))
  105. # @type var form: Transcoder::urlencoded_input
  106. 1505 Transcoder::Form.encode(form)
  107. 7102 elsif (json = params.delete(:json))
  108. # @type var body: _ToJson
  109. 83 Transcoder::JSON.encode(json)
  110. end
  111. end
  112. # returns the +body+ wrapped with the correct deflater accordinng to the given +encodisng+.
  113. 27 def initialize_deflater_body(body, encoding)
  114. 87 case encoding
  115. when "gzip"
  116. 48 Transcoder::GZIP.encode(body)
  117. when "deflate"
  118. 24 Transcoder::Deflate.encode(body)
  119. when "identity"
  120. 16 body
  121. else
  122. 8 body
  123. end
  124. end
  125. end
  126. end
  127. end

lib/httpx/resolver.rb

100.0% lines covered

82 relevant lines. 82 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "ipaddr"
  4. 27 module HTTPX
  5. 27 module Resolver
  6. 27 RESOLVE_TIMEOUT = [2, 3].freeze
  7. 27 require "httpx/resolver/resolver"
  8. 27 require "httpx/resolver/system"
  9. 27 require "httpx/resolver/native"
  10. 27 require "httpx/resolver/https"
  11. 27 require "httpx/resolver/multi"
  12. 27 @lookup_mutex = Thread::Mutex.new
  13. 201 @lookups = Hash.new { |h, k| h[k] = [] }
  14. 27 @identifier_mutex = Thread::Mutex.new
  15. 27 @identifier = 1
  16. 27 @system_resolver = Resolv::Hosts.new
  17. 27 module_function
  18. 27 def resolver_for(resolver_type)
  19. 6368 case resolver_type
  20. 6997 when :native then Native
  21. 33 when :system then System
  22. 74 when :https then HTTPS
  23. else
  24. 76 return resolver_type if resolver_type.is_a?(Class) && resolver_type < Resolver
  25. 8 raise Error, "unsupported resolver type (#{resolver_type})"
  26. end
  27. end
  28. 27 def nolookup_resolve(hostname)
  29. 6979 ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
  30. end
  31. 27 def ip_resolve(hostname)
  32. 6979 [IPAddr.new(hostname)]
  33. rescue ArgumentError
  34. end
  35. 27 def system_resolve(hostname)
  36. 581 ips = @system_resolver.getaddresses(hostname)
  37. 581 return if ips.empty?
  38. 771 ips.map { |ip| IPAddr.new(ip) }
  39. rescue IOError
  40. end
  41. 27 def cached_lookup(hostname)
  42. 6442 now = Utils.now
  43. 6442 lookup_synchronize do |lookups|
  44. 6442 lookup(hostname, lookups, now)
  45. end
  46. end
  47. 27 def cached_lookup_set(hostname, family, entries)
  48. 242 now = Utils.now
  49. 242 entries.each do |entry|
  50. 336 entry["TTL"] += now
  51. end
  52. 242 lookup_synchronize do |lookups|
  53. 214 case family
  54. when Socket::AF_INET6
  55. 40 lookups[hostname].concat(entries)
  56. when Socket::AF_INET
  57. 202 lookups[hostname].unshift(*entries)
  58. end
  59. 242 entries.each do |entry|
  60. 336 next unless entry["name"] != hostname
  61. 30 case family
  62. when Socket::AF_INET6
  63. 8 lookups[entry["name"]] << entry
  64. when Socket::AF_INET
  65. 24 lookups[entry["name"]].unshift(entry)
  66. end
  67. end
  68. end
  69. end
  70. # do not use directly!
  71. 27 def lookup(hostname, lookups, ttl)
  72. 6450 return unless lookups.key?(hostname)
  73. 5870 entries = lookups[hostname] = lookups[hostname].select do |address|
  74. 15376 address["TTL"] > ttl
  75. end
  76. 5870 ips = entries.flat_map do |address|
  77. 15341 if address.key?("alias")
  78. 8 lookup(address["alias"], lookups, ttl)
  79. else
  80. 15333 IPAddr.new(address["data"])
  81. end
  82. end.compact
  83. 5870 ips unless ips.empty?
  84. end
  85. 27 def generate_id
  86. 1614 id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
  87. end
  88. 27 def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
  89. 748 Resolv::DNS::Message.new(message_id).tap do |query|
  90. 807 query.rd = 1
  91. 807 query.add_question(hostname, type)
  92. 117 end.encode
  93. end
  94. 27 def decode_dns_answer(payload)
  95. 58 begin
  96. 720 message = Resolv::DNS::Message.decode(payload)
  97. rescue Resolv::DNS::DecodeError => e
  98. 6 return :decode_error, e
  99. end
  100. # no domain was found
  101. 714 return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
  102. 270 return :message_truncated if message.tc == 1
  103. 258 return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
  104. 252 addresses = []
  105. 252 message.each_answer do |question, _, value|
  106. 1034 case value
  107. when Resolv::DNS::Resource::IN::CNAME
  108. 18 addresses << {
  109. "name" => question.to_s,
  110. "TTL" => value.ttl,
  111. "alias" => value.name.to_s,
  112. }
  113. when Resolv::DNS::Resource::IN::A,
  114. Resolv::DNS::Resource::IN::AAAA
  115. 1048 addresses << {
  116. 30 "name" => question.to_s,
  117. "TTL" => value.ttl,
  118. "data" => value.address.to_s,
  119. }
  120. end
  121. end
  122. 252 [:ok, addresses]
  123. end
  124. 27 def lookup_synchronize
  125. 13368 @lookup_mutex.synchronize { yield(@lookups) }
  126. end
  127. 27 def id_synchronize(&block)
  128. 807 @identifier_mutex.synchronize(&block)
  129. end
  130. end
  131. end

lib/httpx/resolver/https.rb

86.11% lines covered

144 relevant lines. 124 lines covered and 20 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "uri"
  4. 27 require "cgi"
  5. 27 require "forwardable"
  6. 27 require "httpx/base64"
  7. 27 module HTTPX
  8. 27 class Resolver::HTTPS < Resolver::Resolver
  9. 27 extend Forwardable
  10. 27 using URIExtensions
  11. 27 module DNSExtensions
  12. 27 refine Resolv::DNS do
  13. 27 def generate_candidates(name)
  14. 42 @config.generate_candidates(name)
  15. end
  16. end
  17. end
  18. 27 using DNSExtensions
  19. 27 NAMESERVER = "https://1.1.1.1/dns-query"
  20. 2 DEFAULTS = {
  21. 25 uri: NAMESERVER,
  22. use_get: false,
  23. }.freeze
  24. 27 def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?
  25. 27 def initialize(_, options)
  26. 90 super
  27. 90 @resolver_options = DEFAULTS.merge(@options.resolver_options)
  28. 90 @queries = {}
  29. 90 @requests = {}
  30. 90 @connections = []
  31. 90 @uri = URI(@resolver_options[:uri])
  32. 90 @uri_addresses = nil
  33. 90 @resolver = Resolv::DNS.new
  34. 90 @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
  35. 90 @resolver.lazy_initialize
  36. end
  37. 27 def <<(connection)
  38. 90 return if @uri.origin == connection.peer.to_s
  39. 48 @uri_addresses ||= HTTPX::Resolver.nolookup_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
  40. 48 if @uri_addresses.empty?
  41. 6 ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
  42. 6 ex.set_backtrace(caller)
  43. 6 connection.force_reset
  44. 6 throw(:resolve_error, ex)
  45. end
  46. 42 resolve(connection)
  47. end
  48. 27 def closed?
  49. true
  50. end
  51. 27 def empty?
  52. 84 true
  53. end
  54. 27 def resolver_connection
  55. # TODO: leaks connection object into the pool
  56. 66 @resolver_connection ||= @current_session.find_connection(@uri, @current_selector,
  57. @options.merge(ssl: { alpn_protocols: %w[h2] })).tap do |conn|
  58. 42 emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
  59. end
  60. end
  61. 27 private
  62. 27 def resolve(connection = @connections.first, hostname = nil)
  63. 66 return unless connection
  64. 66 hostname ||= @queries.key(connection)
  65. 66 if hostname.nil?
  66. 42 hostname = connection.peer.host
  67. log do
  68. "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
  69. 42 end if connection.peer.non_ascii_hostname
  70. 42 hostname = @resolver.generate_candidates(hostname).each do |name|
  71. 126 @queries[name.to_s] = connection
  72. end.first.to_s
  73. else
  74. 24 @queries[hostname] = connection
  75. end
  76. 66 log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
  77. begin
  78. 66 request = build_request(hostname)
  79. 66 request.on(:response, &method(:on_response).curry(2)[request])
  80. 66 request.on(:promise, &method(:on_promise))
  81. 66 @requests[request] = hostname
  82. 66 resolver_connection.send(request)
  83. 66 @connections << connection
  84. rescue ResolveError, Resolv::DNS::EncodeError => e
  85. reset_hostname(hostname)
  86. emit_resolve_error(connection, connection.peer.host, e)
  87. end
  88. end
  89. 27 def on_response(request, response)
  90. 66 response.raise_for_status
  91. rescue StandardError => e
  92. 6 hostname = @requests.delete(request)
  93. 6 connection = reset_hostname(hostname)
  94. 6 emit_resolve_error(connection, connection.peer.host, e)
  95. else
  96. # @type var response: HTTPX::Response
  97. 60 parse(request, response)
  98. ensure
  99. 66 @requests.delete(request)
  100. end
  101. 27 def on_promise(_, stream)
  102. log(level: 2) { "#{stream.id}: refusing stream!" }
  103. stream.refuse
  104. end
  105. 27 def parse(request, response)
  106. 60 code, result = decode_response_body(response)
  107. 60 case code
  108. when :ok
  109. 18 parse_addresses(result, request)
  110. when :no_domain_found
  111. # Indicates no such domain was found.
  112. 36 host = @requests.delete(request)
  113. 36 connection = reset_hostname(host, reset_candidates: false)
  114. 36 unless @queries.value?(connection)
  115. 12 emit_resolve_error(connection)
  116. 12 return
  117. end
  118. 24 resolve
  119. when :dns_error
  120. host = @requests.delete(request)
  121. connection = reset_hostname(host)
  122. emit_resolve_error(connection)
  123. when :decode_error
  124. 6 host = @requests.delete(request)
  125. 6 connection = reset_hostname(host)
  126. 6 emit_resolve_error(connection, connection.peer.host, result)
  127. end
  128. end
  129. 27 def parse_addresses(answers, request)
  130. 18 if answers.empty?
  131. # no address found, eliminate candidates
  132. host = @requests.delete(request)
  133. connection = reset_hostname(host)
  134. emit_resolve_error(connection)
  135. return
  136. else
  137. 42 answers = answers.group_by { |answer| answer["name"] }
  138. 18 answers.each do |hostname, addresses|
  139. 24 addresses = addresses.flat_map do |address|
  140. 24 if address.key?("alias")
  141. 6 alias_address = answers[address["alias"]]
  142. 6 if alias_address.nil?
  143. reset_hostname(address["name"])
  144. if early_resolve(connection, hostname: address["alias"])
  145. @connections.delete(connection)
  146. else
  147. resolve(connection, address["alias"])
  148. return # rubocop:disable Lint/NonLocalExitFromIterator
  149. end
  150. else
  151. 6 alias_address
  152. end
  153. else
  154. 18 address
  155. end
  156. end.compact
  157. 24 next if addresses.empty?
  158. 24 hostname.delete_suffix!(".") if hostname.end_with?(".")
  159. 24 connection = reset_hostname(hostname, reset_candidates: false)
  160. 24 next unless connection # probably a retried query for which there's an answer
  161. 18 @connections.delete(connection)
  162. # eliminate other candidates
  163. 54 @queries.delete_if { |_, conn| connection == conn }
  164. 18 Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
  165. 54 catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
  166. end
  167. end
  168. 18 return if @connections.empty?
  169. resolve
  170. end
  171. 27 def build_request(hostname)
  172. 60 uri = @uri.dup
  173. 60 rklass = @options.request_class
  174. 60 payload = Resolver.encode_dns_query(hostname, type: @record_type)
  175. 60 if @resolver_options[:use_get]
  176. 6 params = URI.decode_www_form(uri.query.to_s)
  177. 6 params << ["type", FAMILY_TYPES[@record_type]]
  178. 6 params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
  179. 6 uri.query = URI.encode_www_form(params)
  180. 6 request = rklass.new("GET", uri, @options)
  181. else
  182. 54 request = rklass.new("POST", uri, @options, body: [payload])
  183. 54 request.headers["content-type"] = "application/dns-message"
  184. end
  185. 60 request.headers["accept"] = "application/dns-message"
  186. 60 request
  187. end
  188. 27 def decode_response_body(response)
  189. 54 case response.headers["content-type"]
  190. when "application/dns-udpwireformat",
  191. "application/dns-message"
  192. 54 Resolver.decode_dns_answer(response.to_s)
  193. else
  194. raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
  195. end
  196. end
  197. 27 def reset_hostname(hostname, reset_candidates: true)
  198. 72 connection = @queries.delete(hostname)
  199. 72 return connection unless connection && reset_candidates
  200. # eliminate other candidates
  201. 36 candidates = @queries.select { |_, conn| connection == conn }.keys
  202. 36 @queries.delete_if { |h, _| candidates.include?(h) }
  203. 12 connection
  204. end
  205. end
  206. end

lib/httpx/resolver/multi.rb

88.24% lines covered

51 relevant lines. 45 lines covered and 6 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 require "resolv"
  4. 27 module HTTPX
  5. 27 class Resolver::Multi
  6. 27 include Callbacks
  7. 27 using ArrayExtensions::FilterMap
  8. 27 attr_reader :resolvers, :options
  9. 27 def initialize(resolver_type, options)
  10. 7082 @current_selector = nil
  11. 7082 @current_session = nil
  12. 7082 @options = options
  13. 7082 @resolver_options = @options.resolver_options
  14. 7082 @resolvers = options.ip_families.map do |ip_family|
  15. 7082 resolver = resolver_type.new(ip_family, options)
  16. 7082 resolver.multi = self
  17. 7082 resolver
  18. end
  19. 7082 @errors = Hash.new { |hs, k| hs[k] = [] }
  20. end
  21. 27 def current_selector=(s)
  22. 7109 @current_selector = s
  23. 14218 @resolvers.each { |r| r.__send__(__method__, s) }
  24. end
  25. 27 def current_session=(s)
  26. 7109 @current_session = s
  27. 14218 @resolvers.each { |r| r.__send__(__method__, s) }
  28. end
  29. 27 def closed?
  30. @resolvers.all?(&:closed?)
  31. end
  32. 27 def empty?
  33. @resolvers.all?(&:empty?)
  34. end
  35. 27 def inflight?
  36. @resolvers.any(&:inflight?)
  37. end
  38. 27 def timeout
  39. @resolvers.filter_map(&:timeout).min
  40. end
  41. 27 def close
  42. @resolvers.each(&:close)
  43. end
  44. 27 def connections
  45. @resolvers.filter_map { |r| r.resolver_connection if r.respond_to?(:resolver_connection) }
  46. end
  47. 27 def early_resolve(connection)
  48. 7111 hostname = connection.peer.host
  49. 7111 addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
  50. 7111 return false unless addresses
  51. 6655 resolved = false
  52. 6912 addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
  53. # try to match the resolver by family. However, there are cases where that's not possible, as when
  54. # the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
  55. 13792 resolver = @resolvers.find { |r| r.family == family } || @resolvers.first
  56. 6896 next unless resolver # this should ever happen
  57. # it does not matter which resolver it is, as early-resolve code is shared.
  58. 6896 resolver.emit_addresses(connection, family, addrs, true)
  59. 6857 resolved = true
  60. end
  61. 6616 resolved
  62. end
  63. 27 def lazy_resolve(connection)
  64. 456 @resolvers.each do |resolver|
  65. 456 resolver << @current_session.try_clone_connection(connection, @current_selector, resolver.family)
  66. 444 next if resolver.empty?
  67. 360 @current_session.select_resolver(resolver, @current_selector)
  68. end
  69. end
  70. end
  71. end

lib/httpx/resolver/native.rb

88.96% lines covered

308 relevant lines. 274 lines covered and 34 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 require "resolv"
  4. 27 module HTTPX
  5. 27 class Resolver::Native < Resolver::Resolver
  6. 27 extend Forwardable
  7. 27 using URIExtensions
  8. 17 DEFAULTS = {
  9. 10 nameserver: nil,
  10. **Resolv::DNS::Config.default_config_hash,
  11. packet_size: 512,
  12. timeouts: Resolver::RESOLVE_TIMEOUT,
  13. }.freeze
  14. 27 DNS_PORT = 53
  15. 27 def_delegator :@connections, :empty?
  16. 27 attr_reader :state
  17. 27 def initialize(family, options)
  18. 6992 super
  19. 6992 @ns_index = 0
  20. 6992 @resolver_options = DEFAULTS.merge(@options.resolver_options)
  21. 6992 @socket_type = @resolver_options.fetch(:socket_type, :udp)
  22. 6992 @nameserver = if (nameserver = @resolver_options[:nameserver])
  23. 6986 nameserver = nameserver[family] if nameserver.is_a?(Hash)
  24. 6986 Array(nameserver)
  25. end
  26. 6992 @ndots = @resolver_options.fetch(:ndots, 1)
  27. 20976 @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
  28. 6992 @_timeouts = Array(@resolver_options[:timeouts])
  29. 8384 @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
  30. 6992 @connections = []
  31. 6992 @name = nil
  32. 6992 @queries = {}
  33. 6992 @read_buffer = "".b
  34. 6992 @write_buffer = Buffer.new(@resolver_options[:packet_size])
  35. 6992 @state = :idle
  36. end
  37. 27 def close
  38. 358 transition(:closed)
  39. end
  40. 27 def closed?
  41. 734 @state == :closed
  42. end
  43. 27 def to_io
  44. 1102 @io.to_io
  45. end
  46. 27 def call
  47. 907 case @state
  48. when :open
  49. 965 consume
  50. end
  51. end
  52. 27 def interests
  53. 27267 case @state
  54. when :idle
  55. 26634 transition(:open)
  56. when :closed
  57. 12 transition(:idle)
  58. 12 transition(:open)
  59. end
  60. 27332 calculate_interests
  61. end
  62. 27 def <<(connection)
  63. 366 if @nameserver.nil?
  64. 6 ex = ResolveError.new("No available nameserver")
  65. 6 ex.set_backtrace(caller)
  66. 6 connection.force_reset
  67. 6 throw(:resolve_error, ex)
  68. else
  69. 360 @connections << connection
  70. 360 resolve
  71. end
  72. end
  73. 27 def timeout
  74. 27332 return if @connections.empty?
  75. 27332 @start_timeout = Utils.now
  76. 27332 hosts = @queries.keys
  77. 27332 @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
  78. end
  79. 27 def handle_socket_timeout(interval); end
  80. 27 private
  81. 27 def calculate_interests
  82. 31491 return :w unless @write_buffer.empty?
  83. 29836 return :r unless @queries.empty?
  84. 270 nil
  85. end
  86. 27 def consume
  87. 977 loop do
  88. 1718 dread if calculate_interests == :r
  89. 1700 break unless calculate_interests == :w
  90. # do_retry
  91. 759 dwrite
  92. 741 break unless calculate_interests == :r
  93. end
  94. rescue Errno::EHOSTUNREACH => e
  95. 18 @ns_index += 1
  96. 18 nameserver = @nameserver
  97. 18 if nameserver && @ns_index < nameserver.size
  98. 12 log { "resolver #{FAMILY_TYPES[@record_type]}: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
  99. 12 transition(:idle)
  100. 12 @timeouts.clear
  101. 12 retry
  102. else
  103. 6 handle_error(e)
  104. 6 emit(:close, self)
  105. end
  106. rescue NativeResolveError => e
  107. 18 handle_error(e)
  108. 18 close_or_resolve
  109. 18 retry unless closed?
  110. end
  111. 27 def schedule_retry
  112. 741 h = @name
  113. 741 return unless h
  114. 741 connection = @queries[h]
  115. 741 timeouts = @timeouts[h]
  116. 741 timeout = timeouts.shift
  117. 741 @timer = @current_selector.after(timeout) do
  118. 75 next unless @connections.include?(connection)
  119. 75 do_retry(h, connection, timeout)
  120. end
  121. end
  122. 27 def do_retry(h, connection, interval)
  123. 75 timeouts = @timeouts[h]
  124. 75 if !timeouts.empty?
  125. 45 log { "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..." }
  126. # must downgrade to tcp AND retry on same host as last
  127. 45 downgrade_socket
  128. 45 resolve(connection, h)
  129. 30 elsif @ns_index + 1 < @nameserver.size
  130. # try on the next nameserver
  131. 6 @ns_index += 1
  132. 6 log do
  133. "resolver #{FAMILY_TYPES[@record_type]}: failed resolving #{h} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)"
  134. end
  135. 6 transition(:idle)
  136. 6 @timeouts.clear
  137. 6 resolve(connection, h)
  138. else
  139. 24 @timeouts.delete(h)
  140. 24 reset_hostname(h, reset_candidates: false)
  141. 24 unless @queries.empty?
  142. 18 resolve(connection)
  143. 18 return
  144. end
  145. 6 @connections.delete(connection)
  146. 6 host = connection.peer.host
  147. # This loop_time passed to the exception is bogus. Ideally we would pass the total
  148. # resolve timeout, including from the previous retries.
  149. 6 ex = ResolveTimeoutError.new(interval, "Timed out while resolving #{host}")
  150. 6 ex.set_backtrace(ex ? ex.backtrace : caller)
  151. 6 emit_resolve_error(connection, host, ex)
  152. 6 close_or_resolve
  153. end
  154. end
  155. 27 def dread(wsize = @resolver_options[:packet_size])
  156. 1267 loop do
  157. 1279 wsize = @large_packet.capacity if @large_packet
  158. 1279 siz = @io.read(wsize, @read_buffer)
  159. 1279 unless siz
  160. ex = EOFError.new("descriptor closed")
  161. ex.set_backtrace(caller)
  162. raise ex
  163. end
  164. 1279 return unless siz.positive?
  165. 678 if @socket_type == :tcp
  166. # packet may be incomplete, need to keep draining from the socket
  167. 30 if @large_packet
  168. # large packet buffer already exists, continue pumping
  169. 12 @large_packet << @read_buffer
  170. 12 next unless @large_packet.full?
  171. 12 parse(@large_packet.to_s)
  172. 12 @large_packet = nil
  173. # downgrade to udp again
  174. 12 downgrade_socket
  175. 12 return
  176. else
  177. 18 size = @read_buffer[0, 2].unpack1("n")
  178. 18 buffer = @read_buffer.byteslice(2..-1)
  179. 18 if size > @read_buffer.bytesize
  180. # only do buffer logic if it's worth it, and the whole packet isn't here already
  181. 12 @large_packet = Buffer.new(size)
  182. 12 @large_packet << buffer
  183. 12 next
  184. else
  185. 6 parse(buffer)
  186. end
  187. end
  188. else # udp
  189. 648 parse(@read_buffer)
  190. end
  191. 636 return if @state == :closed || !@write_buffer.empty?
  192. end
  193. end
  194. 27 def dwrite
  195. 741 loop do
  196. 1482 return if @write_buffer.empty?
  197. 741 siz = @io.write(@write_buffer)
  198. 741 unless siz
  199. ex = EOFError.new("descriptor closed")
  200. ex.set_backtrace(caller)
  201. raise ex
  202. end
  203. 741 return unless siz.positive?
  204. 741 schedule_retry if @write_buffer.empty?
  205. 741 return if @state == :closed
  206. end
  207. end
  208. 27 def parse(buffer)
  209. 666 @timer.cancel
  210. 666 code, result = Resolver.decode_dns_answer(buffer)
  211. 609 case code
  212. when :ok
  213. 234 parse_addresses(result)
  214. when :no_domain_found
  215. # Indicates no such domain was found.
  216. 408 hostname, connection = @queries.first
  217. 408 reset_hostname(hostname, reset_candidates: false)
  218. 714 other_candidate, _ = @queries.find { |_, conn| conn == connection }
  219. 408 if other_candidate
  220. 306 resolve(connection, other_candidate)
  221. else
  222. 102 @connections.delete(connection)
  223. 102 ex = NativeResolveError.new(connection, connection.peer.host, "name or service not known")
  224. 102 ex.set_backtrace(ex ? ex.backtrace : caller)
  225. 102 emit_resolve_error(connection, connection.peer.host, ex)
  226. 102 close_or_resolve
  227. end
  228. when :message_truncated
  229. # TODO: what to do if it's already tcp??
  230. 12 return if @socket_type == :tcp
  231. 12 @socket_type = :tcp
  232. 12 hostname, _ = @queries.first
  233. 12 reset_hostname(hostname)
  234. 12 transition(:closed)
  235. when :dns_error
  236. 6 hostname, connection = @queries.first
  237. 6 reset_hostname(hostname)
  238. 6 @connections.delete(connection)
  239. 6 ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
  240. 6 raise ex
  241. when :decode_error
  242. 6 hostname, connection = @queries.first
  243. 6 reset_hostname(hostname)
  244. 6 @connections.delete(connection)
  245. 6 ex = NativeResolveError.new(connection, connection.peer.host, result.message)
  246. 6 ex.set_backtrace(result.backtrace)
  247. 6 raise ex
  248. end
  249. end
  250. 27 def parse_addresses(addresses)
  251. 234 if addresses.empty?
  252. # no address found, eliminate candidates
  253. 6 hostname, connection = @queries.first
  254. 6 reset_hostname(hostname)
  255. 6 @connections.delete(connection)
  256. 6 raise NativeResolveError.new(connection, connection.peer.host)
  257. else
  258. 228 address = addresses.first
  259. 228 name = address["name"]
  260. 228 connection = @queries.delete(name)
  261. 228 unless connection
  262. orig_name = name
  263. # absolute name
  264. name_labels = Resolv::DNS::Name.create(name).to_a
  265. name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
  266. # probably a retried query for which there's an answer
  267. unless name
  268. @timeouts.delete(orig_name)
  269. return
  270. end
  271. address["name"] = name
  272. connection = @queries.delete(name)
  273. end
  274. 1276 alias_addresses, addresses = addresses.partition { |addr| addr.key?("alias") }
  275. 228 if addresses.empty? && !alias_addresses.empty? # CNAME
  276. hostname_alias = alias_addresses.first["alias"]
  277. # clean up intermediate queries
  278. @timeouts.delete(name) unless connection.peer.host == name
  279. if early_resolve(connection, hostname: hostname_alias)
  280. @connections.delete(connection)
  281. else
  282. if @socket_type == :tcp
  283. # must downgrade to udp if tcp
  284. @socket_type = @resolver_options.fetch(:socket_type, :udp)
  285. transition(:idle)
  286. transition(:open)
  287. end
  288. log { "resolver #{FAMILY_TYPES[@record_type]}: ALIAS #{hostname_alias} for #{name}" }
  289. resolve(connection, hostname_alias)
  290. return
  291. end
  292. else
  293. 228 reset_hostname(name, connection: connection)
  294. 228 @timeouts.delete(connection.peer.host)
  295. 228 @connections.delete(connection)
  296. 228 Resolver.cached_lookup_set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
  297. 1486 catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
  298. end
  299. end
  300. 228 close_or_resolve
  301. end
  302. 27 def resolve(connection = nil, hostname = nil)
  303. 749 @connections.shift until @connections.empty? || @connections.first.state != :closed
  304. 1125 connection ||= @connections.find { |c| !@queries.value?(c) }
  305. 749 raise Error, "no URI to resolve" unless connection
  306. 749 return unless @write_buffer.empty?
  307. 747 hostname ||= @queries.key(connection)
  308. 747 if hostname.nil?
  309. 372 hostname = connection.peer.host
  310. 372 if connection.peer.non_ascii_hostname
  311. log { "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" }
  312. end
  313. 372 hostname = generate_candidates(hostname).each do |name|
  314. 1368 @queries[name] = connection
  315. end.first
  316. else
  317. 347 @queries[hostname] = connection
  318. end
  319. 747 @name = hostname
  320. 747 log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
  321. 58 begin
  322. 747 @write_buffer << encode_dns_query(hostname)
  323. rescue Resolv::DNS::EncodeError => e
  324. reset_hostname(hostname, connection: connection)
  325. @connections.delete(connection)
  326. emit_resolve_error(connection, hostname, e)
  327. close_or_resolve
  328. end
  329. end
  330. 27 def encode_dns_query(hostname)
  331. 747 message_id = Resolver.generate_id
  332. 747 msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
  333. 747 msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
  334. 747 msg
  335. end
  336. 27 def generate_candidates(name)
  337. 372 return [name] if name.end_with?(".")
  338. 372 candidates = []
  339. 372 name_parts = name.scan(/[^.]+/)
  340. 372 candidates = [name] if @ndots <= name_parts.size - 1
  341. 1116 candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
  342. 372 fname = "#{name}."
  343. 372 candidates << fname unless candidates.include?(fname)
  344. 372 candidates
  345. end
  346. 27 def build_socket
  347. 388 ip, port = @nameserver[@ns_index]
  348. 388 port ||= DNS_PORT
  349. 358 case @socket_type
  350. when :udp
  351. 370 log { "resolver #{FAMILY_TYPES[@record_type]}: server: udp://#{ip}:#{port}..." }
  352. 370 UDP.new(ip, port, @options)
  353. when :tcp
  354. 18 log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." }
  355. 18 origin = URI("tcp://#{ip}:#{port}")
  356. 18 TCP.new(origin, [ip], @options)
  357. end
  358. end
  359. 27 def downgrade_socket
  360. 57 return unless @socket_type == :tcp
  361. 12 @socket_type = @resolver_options.fetch(:socket_type, :udp)
  362. 12 transition(:idle)
  363. 12 transition(:open)
  364. end
  365. 27 def transition(nextstate)
  366. 27010 case nextstate
  367. when :idle
  368. 42 if @io
  369. 36 @io.close
  370. 36 @io = nil
  371. end
  372. when :open
  373. 26658 return unless @state == :idle
  374. 26658 @io ||= build_socket
  375. 26658 @io.connect
  376. 26658 return unless @io.connected?
  377. 388 resolve if @queries.empty? && !@connections.empty?
  378. when :closed
  379. 370 return unless @state == :open
  380. 364 @io.close if @io
  381. 364 @start_timeout = nil
  382. 364 @write_buffer.clear
  383. 364 @read_buffer.clear
  384. end
  385. 794 @state = nextstate
  386. rescue Errno::ECONNREFUSED,
  387. Errno::EADDRNOTAVAIL,
  388. Errno::EHOSTUNREACH,
  389. SocketError,
  390. IOError,
  391. ConnectTimeoutError => e
  392. # these errors may happen during TCP handshake
  393. # treat them as resolve errors.
  394. handle_error(e)
  395. emit(:close, self)
  396. end
  397. 27 def handle_error(error)
  398. 24 if error.respond_to?(:connection) &&
  399. error.respond_to?(:host)
  400. 18 reset_hostname(error.host, connection: error.connection)
  401. 18 @connections.delete(error.connection)
  402. 18 emit_resolve_error(error.connection, error.host, error)
  403. else
  404. 6 @queries.each do |host, connection|
  405. 6 reset_hostname(host, connection: connection)
  406. 6 @connections.delete(connection)
  407. 6 emit_resolve_error(connection, host, error)
  408. end
  409. 12 while (connection = @connections.shift)
  410. emit_resolve_error(connection, host, error)
  411. end
  412. end
  413. end
  414. 27 def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
  415. 714 @timeouts.delete(hostname)
  416. 714 return unless connection && reset_candidates
  417. # eliminate other candidates
  418. 1080 candidates = @queries.select { |_, conn| connection == conn }.keys
  419. 1080 @queries.delete_if { |h, _| candidates.include?(h) }
  420. # reset timeouts
  421. 1056 @timeouts.delete_if { |h, _| candidates.include?(h) }
  422. end
  423. 27 def close_or_resolve
  424. # drop already closed connections
  425. 354 @connections.shift until @connections.empty? || @connections.first.state != :closed
  426. 354 if (@connections - @queries.values).empty?
  427. 352 emit(:close, self)
  428. else
  429. 2 resolve
  430. end
  431. end
  432. end
  433. end

lib/httpx/resolver/resolver.rb

83.13% lines covered

83 relevant lines. 69 lines covered and 14 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "ipaddr"
  4. 27 module HTTPX
  5. 27 class Resolver::Resolver
  6. 27 include Callbacks
  7. 27 include Loggable
  8. 27 using ArrayExtensions::Intersect
  9. 2 RECORD_TYPES = {
  10. 25 Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
  11. Socket::AF_INET => Resolv::DNS::Resource::IN::A,
  12. }.freeze
  13. 2 FAMILY_TYPES = {
  14. 25 Resolv::DNS::Resource::IN::AAAA => "AAAA",
  15. Resolv::DNS::Resource::IN::A => "A",
  16. }.freeze
  17. 27 class << self
  18. 27 def multi?
  19. 7082 true
  20. end
  21. end
  22. 27 attr_reader :family, :options
  23. 27 attr_writer :current_selector, :current_session
  24. 27 attr_accessor :multi
  25. 27 def initialize(family, options)
  26. 7107 @family = family
  27. 7107 @record_type = RECORD_TYPES[family]
  28. 7107 @options = options
  29. 7107 set_resolver_callbacks
  30. end
  31. 27 def each_connection(&block)
  32. 231 enum_for(__method__) unless block
  33. 231 return unless @connections
  34. 231 @connections.each(&block)
  35. end
  36. 27 def close; end
  37. 27 alias_method :terminate, :close
  38. 27 def closed?
  39. true
  40. end
  41. 27 def empty?
  42. true
  43. end
  44. 27 def inflight?
  45. false
  46. end
  47. 27 def emit_addresses(connection, family, addresses, early_resolve = false)
  48. 7196 addresses.map! do |address|
  49. 17446 address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
  50. end
  51. # double emission check, but allow early resolution to work
  52. 7196 return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
  53. 7196 log do
  54. 64 "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
  55. 5 "answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
  56. end
  57. 7196 if !early_resolve && # do not apply resolution delay for non-dns name resolution
  58. @current_selector && # just in case...
  59. family == Socket::AF_INET && # resolution delay only applies to IPv4
  60. !connection.io && # connection already has addresses and initiated/ended handshake
  61. connection.options.ip_families.size > 1 && # no need to delay if not supporting dual stack IP
  62. addresses.first.to_s != connection.peer.host.to_s # connection URL host is already the IP (early resolve included perhaps?)
  63. log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
  64. @current_selector.after(0.05) do
  65. # double emission check
  66. unless connection.addresses && addresses.intersect?(connection.addresses)
  67. emit_resolved_connection(connection, addresses, early_resolve)
  68. end
  69. end
  70. else
  71. 7196 emit_resolved_connection(connection, addresses, early_resolve)
  72. end
  73. end
  74. 27 private
  75. 27 def emit_resolved_connection(connection, addresses, early_resolve)
  76. begin
  77. 7196 connection.addresses = addresses
  78. 7150 return if connection.state == :closed
  79. 7150 emit(:resolve, connection)
  80. 24 rescue StandardError => e
  81. 46 if early_resolve
  82. 39 connection.force_reset
  83. 39 throw(:resolve_error, e)
  84. else
  85. 7 emit(:error, connection, e)
  86. end
  87. end
  88. end
  89. 27 def early_resolve(connection, hostname: connection.peer.host)
  90. addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
  91. return false unless addresses
  92. addresses = addresses.select { |addr| addr.family == @family }
  93. return false if addresses.empty?
  94. emit_addresses(connection, @family, addresses, true)
  95. true
  96. end
  97. 27 def emit_resolve_error(connection, hostname = connection.peer.host, ex = nil)
  98. 169 emit_connection_error(connection, resolve_error(hostname, ex))
  99. end
  100. 27 def resolve_error(hostname, ex = nil)
  101. 169 return ex if ex.is_a?(ResolveError) || ex.is_a?(ResolveTimeoutError)
  102. 42 message = ex ? ex.message : "Can't resolve #{hostname}"
  103. 42 error = ResolveError.new(message)
  104. 42 error.set_backtrace(ex ? ex.backtrace : caller)
  105. 42 error
  106. end
  107. 27 def set_resolver_callbacks
  108. 7107 on(:resolve, &method(:resolve_connection))
  109. 7107 on(:error, &method(:emit_connection_error))
  110. 7107 on(:close, &method(:close_resolver))
  111. end
  112. 27 def resolve_connection(connection)
  113. 7150 @current_session.__send__(:on_resolver_connection, connection, @current_selector)
  114. end
  115. 27 def emit_connection_error(connection, error)
  116. 163 return connection.handle_connect_error(error) if connection.connecting?
  117. connection.emit(:error, error)
  118. end
  119. 27 def close_resolver(resolver)
  120. 358 @current_session.__send__(:on_resolver_close, resolver, @current_selector)
  121. end
  122. end
  123. end

lib/httpx/resolver/system.rb

81.06% lines covered

132 relevant lines. 107 lines covered and 25 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 module HTTPX
  4. 27 class Resolver::System < Resolver::Resolver
  5. 27 using URIExtensions
  6. 27 RESOLV_ERRORS = [Resolv::ResolvError,
  7. Resolv::DNS::Requester::RequestError,
  8. Resolv::DNS::EncodeError,
  9. Resolv::DNS::DecodeError].freeze
  10. 27 DONE = 1
  11. 27 ERROR = 2
  12. 27 class << self
  13. 27 def multi?
  14. 25 false
  15. end
  16. end
  17. 27 attr_reader :state
  18. 27 def initialize(options)
  19. 25 super(nil, options)
  20. 25 @resolver_options = @options.resolver_options
  21. 25 resolv_options = @resolver_options.dup
  22. 25 timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
  23. 25 @_timeouts = Array(timeouts)
  24. 50 @timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
  25. 25 resolv_options.delete(:cache)
  26. 25 @connections = []
  27. 25 @queries = []
  28. 25 @ips = []
  29. 25 @pipe_mutex = Thread::Mutex.new
  30. 25 @state = :idle
  31. end
  32. 27 def resolvers
  33. return enum_for(__method__) unless block_given?
  34. yield self
  35. end
  36. 27 def multi
  37. self
  38. end
  39. 27 def empty?
  40. true
  41. end
  42. 27 def close
  43. transition(:closed)
  44. end
  45. 27 def closed?
  46. @state == :closed
  47. end
  48. 27 def to_io
  49. @pipe_read.to_io
  50. end
  51. 27 def call
  52. case @state
  53. when :open
  54. consume
  55. end
  56. nil
  57. end
  58. 27 def interests
  59. return if @queries.empty?
  60. :r
  61. end
  62. 27 def timeout
  63. return unless @queries.empty?
  64. _, connection = @queries.first
  65. return unless connection
  66. @timeouts[connection.peer.host].first
  67. end
  68. 27 def <<(connection)
  69. 25 @connections << connection
  70. 25 resolve
  71. end
  72. 27 def early_resolve(connection, **)
  73. 25 self << connection
  74. 12 true
  75. end
  76. 27 def handle_socket_timeout(interval)
  77. error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
  78. error.set_backtrace(caller)
  79. on_error(error)
  80. end
  81. 27 private
  82. 27 def transition(nextstate)
  83. 25 case nextstate
  84. when :idle
  85. @timeouts.clear
  86. when :open
  87. 25 return unless @state == :idle
  88. 25 @pipe_read, @pipe_write = ::IO.pipe
  89. when :closed
  90. return unless @state == :open
  91. @pipe_write.close
  92. @pipe_read.close
  93. end
  94. 25 @state = nextstate
  95. end
  96. 27 def consume
  97. 25 return if @connections.empty?
  98. 25 if @pipe_read.wait_readable
  99. 25 event = @pipe_read.getbyte
  100. 25 case event
  101. when DONE
  102. 24 *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
  103. 12 @queries.delete(pair)
  104. 12 _, connection = pair
  105. 12 @connections.delete(connection)
  106. 12 family, connection = pair
  107. 24 catch(:coalesced) { emit_addresses(connection, family, addrs) }
  108. when ERROR
  109. 26 *pair, error = @pipe_mutex.synchronize { @ips.pop }
  110. 13 @queries.delete(pair)
  111. 13 @connections.delete(connection)
  112. 13 _, connection = pair
  113. 13 emit_resolve_error(connection, connection.peer.host, error)
  114. end
  115. end
  116. 12 return emit(:close, self) if @connections.empty?
  117. resolve
  118. end
  119. 27 def resolve(connection = @connections.first)
  120. 25 raise Error, "no URI to resolve" unless connection
  121. 25 return unless @queries.empty?
  122. 25 hostname = connection.peer.host
  123. 25 scheme = connection.origin.scheme
  124. log do
  125. "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
  126. 25 end if connection.peer.non_ascii_hostname
  127. 25 transition(:open)
  128. 25 connection.options.ip_families.each do |family|
  129. 25 @queries << [family, connection]
  130. end
  131. 25 async_resolve(connection, hostname, scheme)
  132. 25 consume
  133. end
  134. 27 def async_resolve(connection, hostname, scheme)
  135. 25 families = connection.options.ip_families
  136. 25 log { "resolver: query for #{hostname}" }
  137. 25 timeouts = @timeouts[connection.peer.host]
  138. 25 resolve_timeout = timeouts.first
  139. 25 Thread.start do
  140. 25 Thread.current.report_on_exception = false
  141. begin
  142. 25 addrs = if resolve_timeout
  143. 25 Timeout.timeout(resolve_timeout) do
  144. 25 __addrinfo_resolve(hostname, scheme)
  145. end
  146. else
  147. __addrinfo_resolve(hostname, scheme)
  148. end
  149. 12 addrs = addrs.sort_by(&:afamily).group_by(&:afamily)
  150. 12 families.each do |family|
  151. 12 addresses = addrs[family]
  152. 12 next unless addresses
  153. 12 addresses.map!(&:ip_address)
  154. 12 addresses.uniq!
  155. 12 @pipe_mutex.synchronize do
  156. 12 @ips.unshift([family, connection, addresses])
  157. 12 @pipe_write.putc(DONE) unless @pipe_write.closed?
  158. end
  159. end
  160. rescue StandardError => e
  161. 13 if e.is_a?(Timeout::Error)
  162. 1 timeouts.shift
  163. 1 retry unless timeouts.empty?
  164. 1 e = ResolveTimeoutError.new(resolve_timeout, e.message)
  165. 1 e.set_backtrace(e.backtrace)
  166. end
  167. 13 @pipe_mutex.synchronize do
  168. 13 families.each do |family|
  169. 13 @ips.unshift([family, connection, e])
  170. 13 @pipe_write.putc(ERROR) unless @pipe_write.closed?
  171. end
  172. end
  173. end
  174. end
  175. end
  176. 27 def __addrinfo_resolve(host, scheme)
  177. 25 Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
  178. end
  179. 27 def emit_connection_error(_, error)
  180. 13 throw(:resolve_error, error)
  181. end
  182. 27 def close_resolver(resolver); end
  183. end
  184. end

lib/httpx/response.rb

100.0% lines covered

115 relevant lines. 115 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "objspace"
  3. 27 require "stringio"
  4. 27 require "tempfile"
  5. 27 require "fileutils"
  6. 27 require "forwardable"
  7. 27 module HTTPX
  8. # Defines a HTTP response is handled internally, with a few properties exposed as attributes.
  9. #
  10. # It delegates the following methods to the corresponding HTTPX::Request:
  11. #
  12. # * HTTPX::Request#uri
  13. # * HTTPX::Request#peer_address
  14. #
  15. # It implements (indirectly, via the +body+) the IO write protocol to internally buffer payloads.
  16. #
  17. # It implements the IO reader protocol in order for users to buffer/stream it, acts as an enumerable
  18. # (of payload chunks).
  19. #
  20. 27 class Response
  21. 27 extend Forwardable
  22. 27 include Callbacks
  23. # the HTTP response status code
  24. 27 attr_reader :status
  25. # an HTTPX::Headers object containing the response HTTP headers.
  26. 27 attr_reader :headers
  27. # a HTTPX::Response::Body object wrapping the response body. The following methods are delegated to it:
  28. #
  29. # * HTTPX::Response::Body#to_s
  30. # * HTTPX::Response::Body#to_str
  31. # * HTTPX::Response::Body#read
  32. # * HTTPX::Response::Body#copy_to
  33. # * HTTPX::Response::Body#close
  34. 27 attr_reader :body
  35. # The HTTP protocol version used to fetch the response.
  36. 27 attr_reader :version
  37. # returns the response body buffered in a string.
  38. 27 def_delegator :@body, :to_s
  39. 27 def_delegator :@body, :to_str
  40. # implements the IO reader +#read+ interface.
  41. 27 def_delegator :@body, :read
  42. # copies the response body to a different location.
  43. 27 def_delegator :@body, :copy_to
  44. # the corresponding request uri.
  45. 27 def_delegator :@request, :uri
  46. # the IP address of the peer server.
  47. 27 def_delegator :@request, :peer_address
  48. # inits the instance with the corresponding +request+ to this response, an the
  49. # response HTTP +status+, +version+ and HTTPX::Headers instance of +headers+.
  50. 27 def initialize(request, status, version, headers)
  51. 9303 @request = request
  52. 9303 @options = request.options
  53. 9303 @version = version
  54. 9303 @status = Integer(status)
  55. 9303 @headers = @options.headers_class.new(headers)
  56. 9303 @body = @options.response_body_class.new(self, @options)
  57. 9303 @finished = complete?
  58. 9303 @content_type = nil
  59. end
  60. # dupped initialization
  61. 27 def initialize_dup(orig)
  62. 64 super
  63. # if a response gets dupped, the body handle must also get dupped to prevent
  64. # two responses from using the same file handle to read.
  65. 64 @body = orig.body.dup
  66. end
  67. # closes the respective +@request+ and +@body+.
  68. 27 def close
  69. 459 @request.close
  70. 459 @body.close
  71. end
  72. # merges headers defined in +h+ into the response headers.
  73. 27 def merge_headers(h)
  74. 193 @headers = @headers.merge(h)
  75. end
  76. # writes +data+ chunk into the response body.
  77. 27 def <<(data)
  78. 12205 @body.write(data)
  79. end
  80. # returns the HTTPX::ContentType for the response, as per what's declared in the content-type header.
  81. #
  82. # response.content_type #=> #<HTTPX::ContentType:xxx @header_value="text/plain">
  83. # response.content_type.mime_type #=> "text/plain"
  84. 27 def content_type
  85. 9672 @content_type ||= ContentType.new(@headers["content-type"])
  86. end
  87. # returns whether the response has been fully fetched.
  88. 27 def finished?
  89. 14390 @finished
  90. end
  91. # marks the response as finished, freezes the headers.
  92. 27 def finish!
  93. 8088 @finished = true
  94. 8088 @headers.freeze
  95. end
  96. # returns whether the response contains body payload.
  97. 27 def bodyless?
  98. 9303 @request.verb == "HEAD" ||
  99. @status < 200 || # informational response
  100. @status == 204 ||
  101. @status == 205 ||
  102. @status == 304 || begin
  103. 8857 content_length = @headers["content-length"]
  104. 8857 return false if content_length.nil?
  105. 7516 content_length == "0"
  106. end
  107. end
  108. 27 def complete?
  109. 9303 bodyless? || (@request.verb == "CONNECT" && @status == 200)
  110. end
  111. skipped # :nocov:
  112. skipped def inspect
  113. skipped "#<#{self.class}:#{object_id} " \
  114. skipped "HTTP/#{version} " \
  115. skipped "@status=#{@status} " \
  116. skipped "@headers=#{@headers} " \
  117. skipped "@body=#{@body.bytesize}>"
  118. skipped end
  119. skipped # :nocov:
  120. # returns an instance of HTTPX::HTTPError if the response has a 4xx or 5xx
  121. # status code, or nothing.
  122. #
  123. # ok_response.error #=> nil
  124. # not_found_response.error #=> HTTPX::HTTPError instance, status 404
  125. 27 def error
  126. 570 return if @status < 400
  127. 54 HTTPError.new(self)
  128. end
  129. # it raises the exception returned by +error+, or itself otherwise.
  130. #
  131. # ok_response.raise_for_status #=> ok_response
  132. # not_found_response.raise_for_status #=> raises HTTPX::HTTPError exception
  133. 27 def raise_for_status
  134. 530 return self unless (err = error)
  135. 38 raise err
  136. end
  137. # decodes the response payload into a ruby object **if** the payload is valid json.
  138. #
  139. # response.json #≈> { "foo" => "bar" } for "{\"foo\":\"bar\"}" payload
  140. # response.json(symbolize_names: true) #≈> { foo: "bar" } for "{\"foo\":\"bar\"}" payload
  141. 27 def json(*args)
  142. 129 decode(Transcoder::JSON, *args)
  143. end
  144. # decodes the response payload into a ruby object **if** the payload is valid
  145. # "application/x-www-urlencoded" or "multipart/form-data".
  146. 27 def form
  147. 64 decode(Transcoder::Form)
  148. end
  149. 27 def xml
  150. # TODO: remove at next major version.
  151. 8 warn "DEPRECATION WARNING: calling `.#{__method__}` on plain HTTPX responses is deprecated. " \
  152. 1 "Use HTTPX.plugin(:xml) sessions and call `.#{__method__}` in its responses instead."
  153. 8 require "httpx/plugins/xml"
  154. 8 decode(Plugins::XML::Transcoder)
  155. end
  156. 27 private
  157. # decodes the response payload using the given +transcoder+, which implements the decoding logic.
  158. #
  159. # +transcoder+ must implement the internal transcoder API, i.e. respond to <tt>decode(HTTPX::Response response)</tt>,
  160. # which returns a decoder which responds to <tt>call(HTTPX::Response response, **kwargs)</tt>
  161. 27 def decode(transcoder, *args)
  162. # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
  163. 217 decoder = transcoder.decode(self)
  164. 193 raise Error, "no decoder available for \"#{transcoder}\"" unless decoder
  165. 193 @body.rewind
  166. 193 decoder.call(self, *args)
  167. end
  168. end
  169. # Helper class which decodes the HTTP "content-type" header.
  170. 27 class ContentType
  171. 27 MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
  172. 27 CHARSET_RE = /;\s*charset=([^;]+)/i.freeze
  173. 27 def initialize(header_value)
  174. 9634 @header_value = header_value
  175. end
  176. # returns the mime type declared in the header.
  177. #
  178. # ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
  179. 27 def mime_type
  180. 217 return @mime_type if defined?(@mime_type)
  181. 179 m = @header_value.to_s[MIME_TYPE_RE, 1]
  182. 179 m && @mime_type = m.strip.downcase
  183. end
  184. # returns the charset declared in the header.
  185. #
  186. # ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
  187. # ContentType.new("text/plain").charset #=> nil
  188. 27 def charset
  189. 9455 return @charset if defined?(@charset)
  190. 9455 m = @header_value.to_s[CHARSET_RE, 1]
  191. 9455 m && @charset = m.strip.delete('"')
  192. end
  193. end
  194. # Wraps an error which has happened while processing an HTTP Request. It has partial
  195. # public API parity with HTTPX::Response, so users should rely on it to infer whether
  196. # the returned response is one or the other.
  197. #
  198. # response = HTTPX.get("https://some-domain/path") #=> response is HTTPX::Response or HTTPX::ErrorResponse
  199. # response.raise_for_status #=> raises if it wraps an error
  200. 27 class ErrorResponse
  201. 27 include Loggable
  202. 27 extend Forwardable
  203. # the corresponding HTTPX::Request instance.
  204. 27 attr_reader :request
  205. # the HTTPX::Response instance, when there is one (i.e. error happens fetching the response).
  206. 27 attr_reader :response
  207. # the wrapped exception.
  208. 27 attr_reader :error
  209. # the request uri
  210. 27 def_delegator :@request, :uri
  211. # the IP address of the peer server.
  212. 27 def_delegator :@request, :peer_address
  213. 27 def initialize(request, error)
  214. 1176 @request = request
  215. 1176 @response = request.response if request.response.is_a?(Response)
  216. 1176 @error = error
  217. 1176 @options = request.options
  218. 1176 log_exception(@error)
  219. end
  220. # returns the exception full message.
  221. 27 def to_s
  222. 8 @error.full_message(highlight: false)
  223. end
  224. # closes the error resources.
  225. 27 def close
  226. 40 @response.close if @response
  227. end
  228. # always true for error responses.
  229. 27 def finished?
  230. 1058 true
  231. end
  232. 27 def finish!; end
  233. # raises the wrapped exception.
  234. 27 def raise_for_status
  235. 82 raise @error
  236. end
  237. # buffers lost chunks to error response
  238. 27 def <<(data)
  239. 8 return unless @response
  240. 8 @response << data
  241. end
  242. end
  243. end
  244. 27 require_relative "response/body"
  245. 27 require_relative "response/buffer"
  246. 27 require_relative "pmatch_extensions" if RUBY_VERSION >= "2.7.0"

lib/httpx/response/body.rb

100.0% lines covered

109 relevant lines. 109 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # Implementation of the HTTP Response body as a buffer which implements the IO writer protocol
  4. # (for buffering the response payload), the IO reader protocol (for consuming the response payload),
  5. # and can be iterated over (via #each, which yields the payload in chunks).
  6. 27 class Response::Body
  7. # the payload encoding (i.e. "utf-8", "ASCII-8BIT")
  8. 27 attr_reader :encoding
  9. # Array of encodings contained in the response "content-encoding" header.
  10. 27 attr_reader :encodings
  11. 27 attr_reader :buffer
  12. 27 protected :buffer
  13. # initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
  14. 27 def initialize(response, options)
  15. 9455 @response = response
  16. 9455 @headers = response.headers
  17. 9455 @options = options
  18. 9455 @window_size = options.window_size
  19. 9455 @encodings = []
  20. 9455 @length = 0
  21. 9455 @buffer = nil
  22. 9455 @reader = nil
  23. 9455 @state = :idle
  24. # initialize response encoding
  25. 9455 @encoding = if (enc = response.content_type.charset)
  26. 182 begin
  27. 1614 Encoding.find(enc)
  28. rescue ArgumentError
  29. 32 Encoding::BINARY
  30. end
  31. else
  32. 7841 Encoding::BINARY
  33. end
  34. 9455 initialize_inflaters
  35. end
  36. 27 def initialize_dup(other)
  37. 96 super
  38. 96 @buffer = other.instance_variable_get(:@buffer).dup
  39. end
  40. 27 def closed?
  41. 51 @state == :closed
  42. end
  43. # write the response payload +chunk+ into the buffer. Inflates the chunk when required
  44. # and supported.
  45. 27 def write(chunk)
  46. 12228 return if @state == :closed
  47. 12228 return 0 if chunk.empty?
  48. 11810 chunk = decode_chunk(chunk)
  49. 11810 size = chunk.bytesize
  50. 10478 @length += size
  51. 11810 transition(:open)
  52. 11810 @buffer.write(chunk)
  53. 11810 @response.emit(:chunk_received, chunk)
  54. 11794 size
  55. end
  56. # reads a chunk from the payload (implementation of the IO reader protocol).
  57. 27 def read(*args)
  58. 393 return unless @buffer
  59. 393 unless @reader
  60. 179 rewind
  61. 179 @reader = @buffer
  62. end
  63. 393 @reader.read(*args)
  64. end
  65. # size of the decoded response payload. May differ from "content-length" header if
  66. # response was encoded over-the-wire.
  67. 27 def bytesize
  68. 224 @length
  69. end
  70. # yields the payload in chunks.
  71. 27 def each
  72. 48 return enum_for(__method__) unless block_given?
  73. 3 begin
  74. 32 if @buffer
  75. 32 rewind
  76. 84 while (chunk = @buffer.read(@window_size))
  77. 32 yield(chunk.force_encoding(@encoding))
  78. end
  79. end
  80. ensure
  81. 32 close
  82. end
  83. end
  84. # returns the declared filename in the "contennt-disposition" header, when present.
  85. 27 def filename
  86. 48 return unless @headers.key?("content-disposition")
  87. 40 Utils.get_filename(@headers["content-disposition"])
  88. end
  89. # returns the full response payload as a string.
  90. 27 def to_s
  91. 4635 return "".b unless @buffer
  92. 4304 @buffer.to_s
  93. end
  94. 27 alias_method :to_str, :to_s
  95. # whether the payload is empty.
  96. 27 def empty?
  97. 32 @length.zero?
  98. end
  99. # copies the payload to +dest+.
  100. #
  101. # body.copy_to("path/to/file")
  102. # body.copy_to(Pathname.new("path/to/file"))
  103. # body.copy_to(File.new("path/to/file"))
  104. 27 def copy_to(dest)
  105. 48 return unless @buffer
  106. 48 rewind
  107. 48 if dest.respond_to?(:path) && @buffer.respond_to?(:path)
  108. 8 FileUtils.mv(@buffer.path, dest.path)
  109. else
  110. 40 ::IO.copy_stream(@buffer, dest)
  111. end
  112. end
  113. # closes/cleans the buffer, resets everything
  114. 27 def close
  115. 717 if @buffer
  116. 553 @buffer.close
  117. 553 @buffer = nil
  118. end
  119. 717 @length = 0
  120. 717 transition(:closed)
  121. end
  122. 27 def ==(other)
  123. 294 super || case other
  124. when Response::Body
  125. 152 @buffer == other.buffer
  126. else
  127. 86 @buffer = other
  128. end
  129. end
  130. skipped # :nocov:
  131. skipped def inspect
  132. skipped "#<#{self.class}:#{object_id} " \
  133. skipped "@state=#{@state} " \
  134. skipped "@length=#{@length}>"
  135. skipped end
  136. skipped # :nocov:
  137. # rewinds the response payload buffer.
  138. 27 def rewind
  139. 884 return unless @buffer
  140. # in case there's some reading going on
  141. 884 @reader = nil
  142. 884 @buffer.rewind
  143. end
  144. 27 private
  145. # prepares inflaters for the advertised encodings in "content-encoding" header.
  146. 27 def initialize_inflaters
  147. 9455 @inflaters = nil
  148. 9455 return unless @headers.key?("content-encoding")
  149. 179 return unless @options.decompress_response_body
  150. 163 @inflaters = @headers.get("content-encoding").filter_map do |encoding|
  151. 163 next if encoding == "identity"
  152. 163 inflater = self.class.initialize_inflater_by_encoding(encoding, @response)
  153. # do not uncompress if there is no decoder available. In fact, we can't reliably
  154. # continue decompressing beyond that, so ignore.
  155. 163 break unless inflater
  156. 163 @encodings << encoding
  157. 163 inflater
  158. end
  159. end
  160. # passes the +chunk+ through all inflaters to decode it.
  161. 27 def decode_chunk(chunk)
  162. 46 @inflaters.reverse_each do |inflater|
  163. 437 chunk = inflater.call(chunk)
  164. 12073 end if @inflaters
  165. 12074 chunk
  166. end
  167. # tries transitioning the body STM to the +nextstate+.
  168. 27 def transition(nextstate)
  169. 11117 case nextstate
  170. when :open
  171. 11810 return unless @state == :idle
  172. 7492 @buffer = Response::Buffer.new(
  173. threshold_size: @options.body_threshold_size,
  174. bytesize: @length,
  175. encoding: @encoding
  176. )
  177. when :closed
  178. 717 return if @state == :closed
  179. end
  180. 8209 @state = nextstate
  181. end
  182. 27 class << self
  183. 27 def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
  184. 147 case encoding
  185. when "gzip"
  186. 147 Transcoder::GZIP.decode(response, **kwargs)
  187. when "deflate"
  188. 16 Transcoder::Deflate.decode(response, **kwargs)
  189. end
  190. end
  191. end
  192. end
  193. end

lib/httpx/response/buffer.rb

92.19% lines covered

64 relevant lines. 59 lines covered and 5 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "delegate"
  3. 27 require "stringio"
  4. 27 require "tempfile"
  5. 27 module HTTPX
  6. # wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
  7. 27 class Response::Buffer < SimpleDelegator
  8. 27 attr_reader :buffer
  9. 27 protected :buffer
  10. # initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
  11. # the initial +bytesize+, and the +encoding+.
  12. 27 def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
  13. 7688 @threshold_size = threshold_size
  14. 7688 @bytesize = bytesize
  15. 7688 @encoding = encoding
  16. 7688 @buffer = StringIO.new("".b)
  17. 7688 super(@buffer)
  18. end
  19. 27 def initialize_dup(other)
  20. 96 super
  21. # create new descriptor in READ-ONLY mode
  22. 12 @buffer =
  23. 83 case other.buffer
  24. when StringIO
  25. 96 StringIO.new(other.buffer.string, mode: File::RDONLY)
  26. else
  27. other.buffer.class.new(other.buffer.path, encoding: Encoding::BINARY, mode: File::RDONLY)
  28. end
  29. end
  30. # size in bytes of the buffered content.
  31. 27 def size
  32. 324 @bytesize
  33. end
  34. # writes the +chunk+ into the buffer.
  35. 27 def write(chunk)
  36. 10817 @bytesize += chunk.bytesize
  37. 12190 try_upgrade_buffer
  38. 12190 @buffer.write(chunk)
  39. end
  40. # returns the buffered content as a string.
  41. 27 def to_s
  42. 3880 case @buffer
  43. when StringIO
  44. 494 begin
  45. 4320 @buffer.string.force_encoding(@encoding)
  46. rescue ArgumentError
  47. @buffer.string
  48. end
  49. when Tempfile
  50. 72 rewind
  51. 72 content = @buffer.read
  52. 8 begin
  53. 72 content.force_encoding(@encoding)
  54. rescue ArgumentError # ex: unknown encoding name - utf
  55. content
  56. end
  57. end
  58. end
  59. # closes the buffer.
  60. 27 def close
  61. 665 @buffer.close
  62. 665 @buffer.unlink if @buffer.respond_to?(:unlink)
  63. end
  64. 27 def ==(other)
  65. 152 super || begin
  66. 152 return false unless other.is_a?(Response::Buffer)
  67. 152 if @buffer.nil?
  68. other.buffer.nil?
  69. 151 elsif @buffer.respond_to?(:read) &&
  70. other.respond_to?(:read)
  71. 152 buffer_pos = @buffer.pos
  72. 152 other_pos = other.buffer.pos
  73. 152 @buffer.rewind
  74. 152 other.buffer.rewind
  75. 18 begin
  76. 152 FileUtils.compare_stream(@buffer, other.buffer)
  77. ensure
  78. 152 @buffer.pos = buffer_pos
  79. 152 other.buffer.pos = other_pos
  80. end
  81. else
  82. to_s == other.to_s
  83. end
  84. end
  85. end
  86. 27 private
  87. # initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
  88. # has been reached.
  89. 27 def try_upgrade_buffer
  90. 12190 return unless @bytesize > @threshold_size
  91. 452 return if @buffer.is_a?(Tempfile)
  92. 163 aux = @buffer
  93. 163 @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  94. 163 if aux
  95. 163 aux.rewind
  96. 163 ::IO.copy_stream(aux, @buffer)
  97. 163 aux.close
  98. end
  99. 163 __setobj__(@buffer)
  100. end
  101. end
  102. end

lib/httpx/selector.rb

93.27% lines covered

104 relevant lines. 97 lines covered and 7 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "io/wait"
  3. 27 module HTTPX
  4. 27 class Selector
  5. 27 extend Forwardable
  6. 27 READABLE = %i[rw r].freeze
  7. 27 WRITABLE = %i[rw w].freeze
  8. 27 private_constant :READABLE
  9. 27 private_constant :WRITABLE
  10. 27 def_delegator :@timers, :after
  11. 27 def_delegator :@selectables, :empty?
  12. 27 def initialize
  13. 7665 @timers = Timers.new
  14. 7665 @selectables = []
  15. 7665 @is_timer_interval = false
  16. end
  17. 27 def each(&blk)
  18. @selectables.each(&blk)
  19. end
  20. 27 def next_tick
  21. 11522485 catch(:jump_tick) do
  22. 11522485 timeout = next_timeout
  23. 11522485 if timeout && timeout.negative?
  24. @timers.fire
  25. throw(:jump_tick)
  26. end
  27. 308734 begin
  28. 11522485 select(timeout, &:call)
  29. 11522342 @timers.fire
  30. rescue TimeoutError => e
  31. @timers.fire(e)
  32. end
  33. end
  34. rescue StandardError => e
  35. 23 each_connection do |c|
  36. c.emit(:error, e)
  37. end
  38. rescue Exception # rubocop:disable Lint/RescueException
  39. 104 each_connection do |conn|
  40. 96 conn.force_reset
  41. 96 conn.disconnect
  42. end
  43. 104 raise
  44. end
  45. 27 def terminate
  46. # array may change during iteration
  47. 7180 selectables = @selectables.reject(&:inflight?)
  48. 7180 selectables.each(&:terminate)
  49. 7493 until selectables.empty?
  50. 2870 next_tick
  51. 2549 selectables &= @selectables
  52. end
  53. end
  54. 27 def find_resolver(options)
  55. 7136 res = @selectables.find do |c|
  56. 53 c.is_a?(Resolver::Resolver) && options == c.options
  57. end
  58. 7136 res.multi if res
  59. end
  60. 27 def each_connection(&block)
  61. 33737 return enum_for(__method__) unless block
  62. 17174 @selectables.each do |c|
  63. 2075 case c
  64. when Resolver::Resolver
  65. 231 c.each_connection(&block)
  66. when Connection
  67. 1987 yield c
  68. end
  69. end
  70. end
  71. 27 def find_connection(request_uri, options)
  72. 9223 each_connection.find do |connection|
  73. 1226 connection.match?(request_uri, options)
  74. end
  75. end
  76. 27 def find_mergeable_connection(connection)
  77. 7340 each_connection.find do |ch|
  78. 320 ch != connection && ch.mergeable?(connection)
  79. end
  80. end
  81. # deregisters +io+ from selectables.
  82. 27 def deregister(io)
  83. 8343 @selectables.delete(io)
  84. end
  85. # register +io+.
  86. 27 def register(io)
  87. 8793 return if @selectables.include?(io)
  88. 8372 @selectables << io
  89. end
  90. 27 private
  91. 27 def select(interval, &block)
  92. # do not cause an infinite loop here.
  93. #
  94. # this may happen if timeout calculation actually triggered an error which causes
  95. # the connections to be reaped (such as the total timeout error) before #select
  96. # gets called.
  97. 11522485 return if interval.nil? && @selectables.empty?
  98. 11519639 return select_one(interval, &block) if @selectables.size == 1
  99. 387 select_many(interval, &block)
  100. end
  101. 27 def select_many(interval, &block)
  102. 387 r, w = nil
  103. # first, we group IOs based on interest type. On call to #interests however,
  104. # things might already happen, and new IOs might be registered, so we might
  105. # have to start all over again. We do this until we group all selectables
  106. 387 @selectables.delete_if do |io|
  107. 387 interests = io.interests
  108. 387 (r ||= []) << io if READABLE.include?(interests)
  109. 387 (w ||= []) << io if WRITABLE.include?(interests)
  110. 387 io.state == :closed
  111. end
  112. # TODO: what to do if there are no selectables?
  113. 387 readers, writers = IO.select(r, w, nil, interval)
  114. 387 if readers.nil? && writers.nil? && interval
  115. 203 [*r, *w].each { |io| io.handle_socket_timeout(interval) }
  116. 178 return
  117. end
  118. 184 if writers
  119. 3 readers.each do |io|
  120. 108 yield io
  121. # so that we don't yield 2 times
  122. 108 writers.delete(io)
  123. 183 end if readers
  124. 184 writers.each(&block)
  125. else
  126. readers.each(&block) if readers
  127. end
  128. end
  129. 27 def select_one(interval)
  130. 11519252 io = @selectables.first
  131. 11519252 return unless io
  132. 11519252 interests = io.interests
  133. 11519251 result = case interests
  134. 13774 when :r then io.to_io.wait_readable(interval)
  135. 9819 when :w then io.to_io.wait_writable(interval)
  136. when :rw then io.to_io.wait(interval, :read_write)
  137. 11495658 when nil then return
  138. end
  139. 23593 unless result || interval.nil?
  140. 540 io.handle_socket_timeout(interval) unless @is_timer_interval
  141. 480 return
  142. end
  143. # raise TimeoutError.new(interval, "timed out while waiting on select")
  144. 23053 yield io
  145. # rescue IOError, SystemCallError
  146. # @selectables.reject!(&:closed?)
  147. # raise unless @selectables.empty?
  148. end
  149. 27 def next_timeout
  150. 11522485 @is_timer_interval = false
  151. 11522485 timer_interval = @timers.wait_interval
  152. 11522485 connection_interval = @selectables.filter_map(&:timeout).min
  153. 11522485 return connection_interval unless timer_interval
  154. 11479857 if connection_interval.nil? || timer_interval <= connection_interval
  155. 11479817 @is_timer_interval = true
  156. 10777893 return timer_interval
  157. end
  158. 40 connection_interval
  159. end
  160. end
  161. end

lib/httpx/session.rb

95.93% lines covered

270 relevant lines. 259 lines covered and 11 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # Class implementing the APIs being used publicly.
  4. #
  5. # HTTPX.get(..) #=> delegating to an internal HTTPX::Session object.
  6. # HTTPX.plugin(..).get(..) #=> creating an intermediate HTTPX::Session with plugin, then sending the GET request
  7. 27 class Session
  8. 27 include Loggable
  9. 27 include Chainable
  10. # initializes the session with a set of +options+, which will be shared by all
  11. # requests sent from it.
  12. #
  13. # When pass a block, it'll yield itself to it, then closes after the block is evaluated.
  14. 27 def initialize(options = EMPTY_HASH, &blk)
  15. 11047 @options = self.class.default_options.merge(options)
  16. 11047 @persistent = @options.persistent
  17. 11047 @pool = @options.pool_class.new(@options.pool_options)
  18. 11047 @wrapped = false
  19. 11047 @closing = false
  20. 11047 INSTANCES[self] = self if @persistent && @options.close_on_fork && INSTANCES
  21. 11047 wrap(&blk) if blk
  22. end
  23. # Yields itself the block, then closes it after the block is evaluated.
  24. #
  25. # session.wrap do |http|
  26. # http.get("https://wikipedia.com")
  27. # end # wikipedia connection closes here
  28. 27 def wrap
  29. 583 prev_wrapped = @wrapped
  30. 583 @wrapped = true
  31. 583 was_initialized = false
  32. 583 current_selector = get_current_selector do
  33. 583 selector = Selector.new
  34. 583 set_current_selector(selector)
  35. 583 was_initialized = true
  36. 583 selector
  37. end
  38. 58 begin
  39. 583 yield self
  40. ensure
  41. 583 unless prev_wrapped
  42. 583 if @persistent
  43. 1 deactivate(current_selector)
  44. else
  45. 582 close(current_selector)
  46. end
  47. end
  48. 583 @wrapped = prev_wrapped
  49. 583 set_current_selector(nil) if was_initialized
  50. end
  51. end
  52. # closes all the active connections from the session.
  53. #
  54. # when called directly without specifying +selector+, all available connections
  55. # will be picked up from the connection pool and closed. Connections in use
  56. # by other sessions, or same session in a different thread, will not be reaped.
  57. 27 def close(selector = Selector.new)
  58. # throw resolvers away from the pool
  59. 7180 @pool.reset_resolvers
  60. # preparing to throw away connections
  61. 16634 while (connection = @pool.pop_connection)
  62. 4414 next if connection.state == :closed
  63. 211 select_connection(connection, selector)
  64. end
  65. 809 begin
  66. 7180 @closing = true
  67. 7180 selector.terminate
  68. ensure
  69. 7180 @closing = false
  70. end
  71. end
  72. # performs one, or multple requests; it accepts:
  73. #
  74. # 1. one or multiple HTTPX::Request objects;
  75. # 2. an HTTP verb, then a sequence of URIs or URI/options tuples;
  76. # 3. one or multiple HTTP verb / uri / (optional) options tuples;
  77. #
  78. # when present, the set of +options+ kwargs is applied to all of the
  79. # sent requests.
  80. #
  81. # respectively returns a single HTTPX::Response response, or all of them in an Array, in the same order.
  82. #
  83. # resp1 = session.request(req1)
  84. # resp1, resp2 = session.request(req1, req2)
  85. # resp1 = session.request("GET", "https://server.org/a")
  86. # resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"])
  87. # resp1, resp2 = session.request(["GET", "https://server.org/a"], ["GET", "https://server.org/b"])
  88. # resp1 = session.request("POST", "https://server.org/a", form: { "foo" => "bar" })
  89. # resp1, resp2 = session.request(["POST", "https://server.org/a", form: { "foo" => "bar" }], ["GET", "https://server.org/b"])
  90. # resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"], headers: { "x-api-token" => "TOKEN" })
  91. #
  92. 27 def request(*args, **params)
  93. 7464 raise ArgumentError, "must perform at least one request" if args.empty?
  94. 7464 requests = args.first.is_a?(Request) ? args : build_requests(*args, params)
  95. 7427 responses = send_requests(*requests)
  96. 7293 return responses.first if responses.size == 1
  97. 199 responses
  98. end
  99. # returns a HTTP::Request instance built from the HTTP +verb+, the request +uri+, and
  100. # the optional set of request-specific +options+. This request **must** be sent through
  101. # the same session it was built from.
  102. #
  103. # req = session.build_request("GET", "https://server.com")
  104. # resp = session.request(req)
  105. 27 def build_request(verb, uri, params = EMPTY_HASH, options = @options)
  106. 9146 rklass = options.request_class
  107. 9146 request = rklass.new(verb, uri, options, params)
  108. 9109 request.persistent = @persistent
  109. 9109 set_request_callbacks(request)
  110. 9109 request
  111. end
  112. 27 def select_connection(connection, selector)
  113. 8793 pin_connection(connection, selector)
  114. 8793 selector.register(connection)
  115. end
  116. 27 def pin_connection(connection, selector)
  117. 8812 connection.current_session = self
  118. 8812 connection.current_selector = selector
  119. end
  120. 27 alias_method :select_resolver, :select_connection
  121. 27 def deselect_connection(connection, selector, cloned = false)
  122. 7985 selector.deregister(connection)
  123. # when connections coalesce
  124. 7985 return if connection.state == :idle
  125. 7948 return if cloned
  126. 7940 return if @closing && connection.state == :closed
  127. 7932 @pool.checkin_connection(connection)
  128. end
  129. 27 def deselect_resolver(resolver, selector)
  130. 358 selector.deregister(resolver)
  131. 358 return if @closing && resolver.closed?
  132. 358 @pool.checkin_resolver(resolver)
  133. end
  134. 27 def try_clone_connection(connection, selector, family)
  135. 456 connection.family ||= family
  136. 456 return connection if connection.family == family
  137. new_connection = connection.class.new(connection.origin, connection.options)
  138. new_connection.family = family
  139. connection.sibling = new_connection
  140. do_init_connection(new_connection, selector)
  141. new_connection
  142. end
  143. # returns the HTTPX::Connection through which the +request+ should be sent through.
  144. 27 def find_connection(request_uri, selector, options)
  145. 9223 if (connection = selector.find_connection(request_uri, options))
  146. 1105 return connection
  147. end
  148. 8051 connection = @pool.checkout_connection(request_uri, options)
  149. 7124 case connection.state
  150. when :idle
  151. 7302 do_init_connection(connection, selector)
  152. when :open
  153. 66 if options.io
  154. 66 select_connection(connection, selector)
  155. else
  156. pin_connection(connection, selector)
  157. end
  158. when :closing, :closed
  159. 632 connection.idling
  160. 632 select_connection(connection, selector)
  161. else
  162. 19 pin_connection(connection, selector)
  163. end
  164. 7955 connection
  165. end
  166. 27 private
  167. 27 def deactivate(selector)
  168. 484 selector.each_connection do |connection|
  169. 366 connection.deactivate
  170. 366 deselect_connection(connection, selector) if connection.state == :inactive
  171. end
  172. end
  173. # callback executed when an HTTP/2 promise frame has been received.
  174. 27 def on_promise(_, stream)
  175. 8 log(level: 2) { "#{stream.id}: refusing stream!" }
  176. 8 stream.refuse
  177. end
  178. # returns the corresponding HTTP::Response to the given +request+ if it has been received.
  179. 27 def fetch_response(request, _selector, _options)
  180. 11527835 response = request.response
  181. 11527835 response if response && response.finished?
  182. end
  183. # sends the +request+ to the corresponding HTTPX::Connection
  184. 27 def send_request(request, selector, options = request.options)
  185. 1911 error = begin
  186. 9143 catch(:resolve_error) do
  187. 9143 connection = find_connection(request.uri, selector, options)
  188. 9023 connection.send(request)
  189. end
  190. rescue StandardError => e
  191. 40 e
  192. end
  193. 9137 return unless error && error.is_a?(Exception)
  194. 120 raise error unless error.is_a?(Error)
  195. 120 response = ErrorResponse.new(request, error)
  196. 120 request.response = response
  197. 120 request.emit(:response, response)
  198. end
  199. # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
  200. 27 def build_requests(*args, params)
  201. 6870 requests = if args.size == 1
  202. 78 reqs = args.first
  203. 78 reqs.map do |verb, uri, ps = EMPTY_HASH|
  204. 156 request_params = params
  205. 156 request_params = request_params.merge(ps) unless ps.empty?
  206. 156 build_request(verb, uri, request_params)
  207. end
  208. else
  209. 6792 verb, uris = args
  210. 6792 if uris.respond_to?(:each)
  211. 6552 uris.enum_for(:each).map do |uri, ps = EMPTY_HASH|
  212. 7310 request_params = params
  213. 7310 request_params = request_params.merge(ps) unless ps.empty?
  214. 7310 build_request(verb, uri, request_params)
  215. end
  216. else
  217. 240 [build_request(verb, uris, params)]
  218. end
  219. end
  220. 6833 raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
  221. 6833 requests
  222. end
  223. 27 def set_request_callbacks(request)
  224. 8982 request.on(:promise, &method(:on_promise))
  225. end
  226. 27 def do_init_connection(connection, selector)
  227. 7302 resolve_connection(connection, selector) unless connection.family
  228. end
  229. # sends an array of HTTPX::Request +requests+, returns the respective array of HTTPX::Response objects.
  230. 27 def send_requests(*requests)
  231. 14327 selector = get_current_selector { Selector.new }
  232. 830 begin
  233. 7525 _send_requests(requests, selector)
  234. 7519 receive_requests(requests, selector)
  235. ensure
  236. 7509 unless @wrapped
  237. 6801 if @persistent
  238. 483 deactivate(selector)
  239. else
  240. 6318 close(selector)
  241. end
  242. end
  243. end
  244. end
  245. # sends an array of HTTPX::Request objects
  246. 27 def _send_requests(requests, selector)
  247. 7525 requests.each do |request|
  248. 8346 send_request(request, selector)
  249. end
  250. end
  251. # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
  252. 27 def receive_requests(requests, selector)
  253. # @type var responses: Array[response]
  254. 7519 responses = []
  255. # guarantee ordered responses
  256. 7519 loop do
  257. 8348 request = requests.first
  258. 8348 return responses unless request
  259. 12230097 catch(:coalesced) { selector.next_tick } until (response = fetch_response(request, selector, request.options))
  260. 8220 request.emit(:complete, response)
  261. 8220 responses << response
  262. 8220 requests.shift
  263. 8220 break if requests.empty?
  264. 829 next unless selector.empty?
  265. # in some cases, the pool of connections might have been drained because there was some
  266. # handshake error, and the error responses have already been emitted, but there was no
  267. # opportunity to traverse the requests, hence we're returning only a fraction of the errors
  268. # we were supposed to. This effectively fetches the existing responses and return them.
  269. while (request = requests.shift)
  270. response = fetch_response(request, selector, request.options)
  271. request.emit(:complete, response) if response
  272. responses << response
  273. end
  274. break
  275. end
  276. 7391 responses
  277. end
  278. 27 def resolve_connection(connection, selector)
  279. 7326 if connection.addresses || connection.open?
  280. #
  281. # there are two cases in which we want to activate initialization of
  282. # connection immediately:
  283. #
  284. # 1. when the connection already has addresses, i.e. it doesn't need to
  285. # resolve a name (not the same as name being an IP, yet)
  286. # 2. when the connection is initialized with an external already open IO.
  287. #
  288. 190 on_resolver_connection(connection, selector)
  289. 188 return
  290. end
  291. 7136 resolver = find_resolver_for(connection, selector)
  292. 7136 resolver.early_resolve(connection) || resolver.lazy_resolve(connection)
  293. end
  294. 27 def on_resolver_connection(connection, selector)
  295. 7340 from_pool = false
  296. 7340 found_connection = selector.find_mergeable_connection(connection) || begin
  297. 7313 from_pool = true
  298. 7313 @pool.checkout_mergeable_connection(connection)
  299. end
  300. 7340 return select_connection(connection, selector) unless found_connection
  301. 27 if found_connection.open?
  302. 26 coalesce_connections(found_connection, connection, selector, from_pool)
  303. else
  304. 1 found_connection.once(:open) do
  305. 1 next unless found_connection.current_session == self
  306. 1 coalesce_connections(found_connection, connection, selector, from_pool)
  307. end
  308. end
  309. end
  310. 27 def on_resolver_close(resolver, selector)
  311. 358 return if resolver.closed?
  312. 358 deselect_resolver(resolver, selector)
  313. 358 resolver.close unless resolver.closed?
  314. end
  315. 27 def find_resolver_for(connection, selector)
  316. 7136 resolver = selector.find_resolver(connection.options)
  317. 7136 unless resolver
  318. 7134 resolver = @pool.checkout_resolver(connection.options)
  319. 7134 resolver.current_session = self
  320. 7134 resolver.current_selector = selector
  321. end
  322. 7136 resolver
  323. end
  324. # coalesces +conn2+ into +conn1+. if +conn1+ was loaded from the connection pool
  325. # (it is known via +from_pool+), then it adds its to the +selector+.
  326. 27 def coalesce_connections(conn1, conn2, selector, from_pool)
  327. 27 unless conn1.coalescable?(conn2)
  328. 14 select_connection(conn2, selector)
  329. 14 @pool.checkin_connection(conn1) if from_pool
  330. 14 return false
  331. end
  332. 13 conn2.coalesced_connection = conn1
  333. 13 select_connection(conn1, selector) if from_pool
  334. 13 deselect_connection(conn2, selector)
  335. 13 true
  336. end
  337. 27 def get_current_selector
  338. 8108 selector_store[self] || (yield if block_given?)
  339. end
  340. 27 def set_current_selector(selector)
  341. 1633 if selector
  342. 949 selector_store[self] = selector
  343. else
  344. 583 selector_store.delete(self)
  345. end
  346. end
  347. 27 def selector_store
  348. 9741 th_current = Thread.current
  349. 9741 th_current.thread_variable_get(:httpx_persistent_selector_store) || begin
  350. 163 {}.compare_by_identity.tap do |store|
  351. 163 th_current.thread_variable_set(:httpx_persistent_selector_store, store)
  352. end
  353. end
  354. end
  355. 27 @default_options = Options.new
  356. 27 @default_options.freeze
  357. 27 @plugins = []
  358. 27 class << self
  359. 27 attr_reader :default_options
  360. 27 def inherited(klass)
  361. 5543 super
  362. 5543 klass.instance_variable_set(:@default_options, @default_options)
  363. 5543 klass.instance_variable_set(:@plugins, @plugins.dup)
  364. 5543 klass.instance_variable_set(:@callbacks, @callbacks.dup)
  365. end
  366. # returns a new HTTPX::Session instance, with the plugin pointed by +pl+ loaded.
  367. #
  368. # session_with_retries = session.plugin(:retries)
  369. # session_with_custom = session.plugin(CustomPlugin)
  370. #
  371. 27 def plugin(pl, options = nil, &block)
  372. 7553 label = pl
  373. # raise Error, "Cannot add a plugin to a frozen config" if frozen?
  374. 7553 pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
  375. 7553 if !@plugins.include?(pl)
  376. 7323 @plugins << pl
  377. 7323 pl.load_dependencies(self, &block) if pl.respond_to?(:load_dependencies)
  378. 7323 @default_options = @default_options.dup
  379. 7323 include(pl::InstanceMethods) if defined?(pl::InstanceMethods)
  380. 7323 extend(pl::ClassMethods) if defined?(pl::ClassMethods)
  381. 7323 opts = @default_options
  382. 7323 opts.extend_with_plugin_classes(pl)
  383. 7323 if defined?(pl::OptionsMethods)
  384. 2981 (pl::OptionsMethods.instance_methods - Object.instance_methods).each do |meth|
  385. 8855 opts.options_class.method_added(meth)
  386. end
  387. 2981 @default_options = opts.options_class.new(opts)
  388. end
  389. 7323 @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
  390. 7323 @default_options = @default_options.merge(options) if options
  391. 7323 if pl.respond_to?(:subplugins)
  392. 32 pl.subplugins.transform_keys(&Plugins.method(:load_plugin)).each do |main_pl, sub_pl|
  393. # in case the main plugin has already been loaded, then apply subplugin functionality
  394. # immediately
  395. 32 next unless @plugins.include?(main_pl)
  396. 8 plugin(sub_pl, options, &block)
  397. end
  398. end
  399. 7323 pl.configure(self, &block) if pl.respond_to?(:configure)
  400. 7323 if label.is_a?(Symbol)
  401. # in case an already-loaded plugin complements functionality of
  402. # the plugin currently being loaded, loaded it now
  403. 5584 @plugins.each do |registered_pl|
  404. 13539 next if registered_pl == pl
  405. 7955 next unless registered_pl.respond_to?(:subplugins)
  406. 16 sub_pl = registered_pl.subplugins[label]
  407. 16 next unless sub_pl
  408. 16 plugin(sub_pl, options, &block)
  409. end
  410. end
  411. 7323 @default_options.freeze
  412. 7323 set_temporary_name("#{superclass}/#{pl}") if respond_to?(:set_temporary_name) # ruby 3.4 only
  413. 229 elsif options
  414. # this can happen when two plugins are loaded, an one of them calls the other under the hood,
  415. # albeit changing some default.
  416. 16 @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
  417. 16 @default_options = @default_options.merge(options) if options
  418. 16 @default_options.freeze
  419. end
  420. 7553 self
  421. end
  422. end
  423. # setup of the support for close_on_fork sessions.
  424. # adapted from https://github.com/mperham/connection_pool/blob/main/lib/connection_pool.rb#L48
  425. 27 if Process.respond_to?(:fork)
  426. 25 INSTANCES = ObjectSpace::WeakMap.new
  427. 25 private_constant :INSTANCES
  428. 25 def self.after_fork
  429. 1 INSTANCES.each_value(&:close)
  430. 1 nil
  431. end
  432. 25 if ::Process.respond_to?(:_fork)
  433. 21 module ForkTracker
  434. 21 def _fork
  435. 1 pid = super
  436. 1 Session.after_fork if pid.zero?
  437. 1 pid
  438. end
  439. end
  440. 21 Process.singleton_class.prepend(ForkTracker)
  441. end
  442. else
  443. 2 INSTANCES = nil
  444. 2 private_constant :INSTANCES
  445. 2 def self.after_fork
  446. # noop
  447. end
  448. end
  449. end
  450. # session may be overridden by certain adapters.
  451. 27 S = Session
  452. end

lib/httpx/session_extensions.rb

100.0% lines covered

14 relevant lines. 14 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 unless ENV.keys.grep(/\Ahttps?_proxy\z/i).empty?
  4. 1 proxy_session = plugin(:proxy)
  5. 1 remove_const(:Session)
  6. 1 const_set(:Session, proxy_session.class)
  7. # redefine the default options static var, which needs to
  8. # refresh options_class
  9. 1 options = proxy_session.class.default_options.to_hash
  10. 1 original_verbosity = $VERBOSE
  11. 1 $VERBOSE = nil
  12. 1 const_set(:Options, proxy_session.class.default_options.options_class)
  13. 1 options[:options_class] = Class.new(options[:options_class])
  14. 1 options.freeze
  15. 1 Options.send(:const_set, :DEFAULT_OPTIONS, options)
  16. 1 Session.instance_variable_set(:@default_options, Options.new(options))
  17. 1 $VERBOSE = original_verbosity
  18. end
  19. skipped # :nocov:
  20. skipped if Session.default_options.debug_level > 2
  21. skipped proxy_session = plugin(:internal_telemetry)
  22. skipped remove_const(:Session)
  23. skipped const_set(:Session, proxy_session.class)
  24. skipped end
  25. skipped # :nocov:
  26. end

lib/httpx/timers.rb

93.94% lines covered

66 relevant lines. 62 lines covered and 4 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 class Timers
  4. 27 def initialize
  5. 7665 @intervals = []
  6. end
  7. 27 def after(interval_in_secs, cb = nil, &blk)
  8. 39223 callback = cb || blk
  9. 39223 raise Error, "timer must have a callback" unless callback
  10. # I'm assuming here that most requests will have the same
  11. # request timeout, as in most cases they share common set of
  12. # options. A user setting different request timeouts for 100s of
  13. # requests will already have a hard time dealing with that.
  14. 70556 unless (interval = @intervals.bsearch { |t| t.interval == interval_in_secs })
  15. 9430 interval = Interval.new(interval_in_secs)
  16. 9430 @intervals << interval
  17. 9430 @intervals.sort!
  18. end
  19. 39223 interval << callback
  20. 39223 @next_interval_at = nil
  21. 39223 Timer.new(interval, callback)
  22. end
  23. 27 def wait_interval
  24. 11522485 drop_elapsed!
  25. 11522485 return if @intervals.empty?
  26. 11479857 @next_interval_at = Utils.now
  27. 11479857 @intervals.first.interval
  28. end
  29. 27 def fire(error = nil)
  30. 11522342 raise error if error && error.timeout != @intervals.first
  31. 11522342 return if @intervals.empty? || !@next_interval_at
  32. 11479221 elapsed_time = Utils.elapsed_time(@next_interval_at)
  33. 11479221 drop_elapsed!(elapsed_time)
  34. 22950939 @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
  35. 11479221 @next_interval_at = nil if @intervals.empty?
  36. end
  37. 27 private
  38. 27 def drop_elapsed!(elapsed_time = 0)
  39. # check first, if not elapsed, then return
  40. 23001706 first_interval = @intervals.first
  41. 23001706 return unless first_interval && first_interval.elapsed?(elapsed_time)
  42. # TODO: would be nice to have a drop_while!
  43. 17148 @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
  44. end
  45. 27 class Timer
  46. 27 def initialize(interval, callback)
  47. 39223 @interval = interval
  48. 39223 @callback = callback
  49. end
  50. 27 def cancel
  51. 57675 @interval.delete(@callback)
  52. end
  53. end
  54. 27 class Interval
  55. 27 include Comparable
  56. 27 attr_reader :interval
  57. 27 def initialize(interval)
  58. 9430 @interval = interval
  59. 9430 @callbacks = []
  60. end
  61. 27 def <=>(other)
  62. 826 @interval <=> other.interval
  63. end
  64. 27 def ==(other)
  65. return @interval == other if other.is_a?(Numeric)
  66. @interval == other.to_f # rubocop:disable Lint/FloatComparison
  67. end
  68. 27 def to_f
  69. Float(@interval)
  70. end
  71. 27 def <<(callback)
  72. 39223 @callbacks << callback
  73. end
  74. 27 def delete(callback)
  75. 57675 @callbacks.delete(callback)
  76. end
  77. 27 def no_callbacks?
  78. @callbacks.empty?
  79. end
  80. 27 def elapsed?(elapsed = 0)
  81. 22959690 (@interval - elapsed) <= 0 || @callbacks.empty?
  82. end
  83. 27 def elapse(elapsed)
  84. # same as elapsing
  85. 11480603 return 0 if @callbacks.empty?
  86. 10771190 @interval -= elapsed
  87. 11472506 if @interval <= 0
  88. 655 cb = @callbacks.dup
  89. 655 cb.each(&:call)
  90. end
  91. 11472506 @interval
  92. end
  93. end
  94. 27 private_constant :Interval
  95. end
  96. end

lib/httpx/transcoder.rb

100.0% lines covered

52 relevant lines. 52 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Transcoder
  4. 27 module_function
  5. 27 def normalize_keys(key, value, cond = nil, &block)
  6. 3415 if cond && cond.call(value)
  7. 1065 block.call(key.to_s, value)
  8. 2349 elsif value.respond_to?(:to_ary)
  9. 454 if value.empty?
  10. 128 block.call("#{key}[]")
  11. else
  12. 326 value.to_ary.each do |element|
  13. 524 normalize_keys("#{key}[]", element, cond, &block)
  14. end
  15. end
  16. 1895 elsif value.respond_to?(:to_hash)
  17. 512 value.to_hash.each do |child_key, child_value|
  18. 512 normalize_keys("#{key}[#{child_key}]", child_value, cond, &block)
  19. end
  20. else
  21. 1384 block.call(key.to_s, value)
  22. end
  23. end
  24. # based on https://github.com/rack/rack/blob/d15dd728440710cfc35ed155d66a98dc2c07ae42/lib/rack/query_parser.rb#L82
  25. 27 def normalize_query(params, name, v, depth)
  26. 184 raise Error, "params depth surpasses what's supported" if depth <= 0
  27. 184 name =~ /\A[\[\]]*([^\[\]]+)\]*/
  28. 184 k = Regexp.last_match(1) || ""
  29. 184 after = Regexp.last_match ? Regexp.last_match.post_match : ""
  30. 184 if k.empty?
  31. 16 return Array(v) if !v.empty? && name == "[]"
  32. 7 return
  33. end
  34. 147 case after
  35. when ""
  36. 49 params[k] = v
  37. when "["
  38. 7 params[name] = v
  39. when "[]"
  40. 16 params[k] ||= []
  41. 16 raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
  42. 16 params[k] << v
  43. when /^\[\]\[([^\[\]]+)\]$/, /^\[\](.+)$/
  44. 32 child_key = Regexp.last_match(1)
  45. 32 params[k] ||= []
  46. 32 raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
  47. 32 if params[k].last.is_a?(Hash) && !params_hash_has_key?(params[k].last, child_key)
  48. 8 normalize_query(params[k].last, child_key, v, depth - 1)
  49. else
  50. 24 params[k] << normalize_query({}, child_key, v, depth - 1)
  51. end
  52. else
  53. 56 params[k] ||= {}
  54. 56 raise Error, "expected Hash (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Hash)
  55. 49 params[k] = normalize_query(params[k], after, v, depth - 1)
  56. end
  57. 168 params
  58. end
  59. 27 def params_hash_has_key?(hash, key)
  60. 16 return false if key.include?("[]")
  61. 16 key.split(/[\[\]]+/).inject(hash) do |h, part|
  62. 16 next h if part == ""
  63. 16 return false unless h.is_a?(Hash) && h.key?(part)
  64. 8 h[part]
  65. end
  66. 8 true
  67. end
  68. end
  69. end
  70. 27 require "httpx/transcoder/body"
  71. 27 require "httpx/transcoder/form"
  72. 27 require "httpx/transcoder/json"
  73. 27 require "httpx/transcoder/chunker"
  74. 27 require "httpx/transcoder/deflate"
  75. 27 require "httpx/transcoder/gzip"

lib/httpx/transcoder/body.rb

100.0% lines covered

26 relevant lines. 26 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "delegate"
  3. 27 module HTTPX::Transcoder
  4. 27 module Body
  5. 27 class Error < HTTPX::Error; end
  6. 27 module_function
  7. 27 class Encoder < SimpleDelegator
  8. 27 def initialize(body)
  9. 1440 body = body.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && body.is_a?(Pathname)
  10. 1440 @body = body
  11. 1440 super(body)
  12. end
  13. 27 def bytesize
  14. 5688 if @body.respond_to?(:bytesize)
  15. 2550 @body.bytesize
  16. 3137 elsif @body.respond_to?(:to_ary)
  17. 982 @body.sum(&:bytesize)
  18. 2155 elsif @body.respond_to?(:size)
  19. 1500 @body.size || Float::INFINITY
  20. 655 elsif @body.respond_to?(:length)
  21. 360 @body.length || Float::INFINITY
  22. 295 elsif @body.respond_to?(:each)
  23. 288 Float::INFINITY
  24. else
  25. 8 raise Error, "cannot determine size of body: #{@body.inspect}"
  26. end
  27. end
  28. 27 def content_type
  29. 1370 "application/octet-stream"
  30. end
  31. end
  32. 27 def encode(body)
  33. 1440 Encoder.new(body)
  34. end
  35. end
  36. end

lib/httpx/transcoder/chunker.rb

100.0% lines covered

66 relevant lines. 66 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 module HTTPX::Transcoder
  4. 27 module Chunker
  5. 27 class Error < HTTPX::Error; end
  6. 27 CRLF = "\r\n".b
  7. 27 class Encoder
  8. 27 extend Forwardable
  9. 27 def initialize(body)
  10. 96 @raw = body
  11. end
  12. 27 def each
  13. 96 return enum_for(__method__) unless block_given?
  14. 96 @raw.each do |chunk|
  15. 448 yield "#{chunk.bytesize.to_s(16)}#{CRLF}#{chunk}#{CRLF}"
  16. end
  17. 96 yield "0#{CRLF}"
  18. end
  19. 27 def respond_to_missing?(meth, *args)
  20. 108 @raw.respond_to?(meth, *args) || super
  21. end
  22. end
  23. 27 class Decoder
  24. 27 extend Forwardable
  25. 27 def_delegator :@buffer, :empty?
  26. 27 def_delegator :@buffer, :<<
  27. 27 def_delegator :@buffer, :clear
  28. 27 def initialize(buffer, trailers = false)
  29. 114 @buffer = buffer
  30. 114 @chunk_buffer = "".b
  31. 114 @finished = false
  32. 114 @state = :length
  33. 114 @trailers = trailers
  34. end
  35. 27 def to_s
  36. 106 @buffer
  37. end
  38. 27 def each
  39. 201 loop do
  40. 1041 case @state
  41. when :length
  42. 340 index = @buffer.index(CRLF)
  43. 340 return unless index && index.positive?
  44. # Read hex-length
  45. 340 hexlen = @buffer.byteslice(0, index)
  46. 340 @buffer = @buffer.byteslice(index..-1) || "".b
  47. 340 hexlen[/\h/] || raise(Error, "wrong chunk size line: #{hexlen}")
  48. 340 @chunk_length = hexlen.hex
  49. # check if is last chunk
  50. 340 @finished = @chunk_length.zero?
  51. 340 nextstate(:crlf)
  52. when :crlf
  53. 566 crlf_size = @finished && !@trailers ? 4 : 2
  54. # consume CRLF
  55. 566 return if @buffer.bytesize < crlf_size
  56. 566 raise Error, "wrong chunked encoding format" unless @buffer.start_with?(CRLF * (crlf_size / 2))
  57. 566 @buffer = @buffer.byteslice(crlf_size..-1)
  58. 566 if @chunk_length.nil?
  59. 226 nextstate(:length)
  60. else
  61. 340 return if @finished
  62. 234 nextstate(:data)
  63. end
  64. when :data
  65. 282 chunk = @buffer.byteslice(0, @chunk_length)
  66. 282 @buffer = @buffer.byteslice(@chunk_length..-1) || "".b
  67. 282 @chunk_buffer << chunk
  68. 247 @chunk_length -= chunk.bytesize
  69. 282 if @chunk_length.zero?
  70. 234 yield @chunk_buffer unless @chunk_buffer.empty?
  71. 226 @chunk_buffer.clear
  72. 226 @chunk_length = nil
  73. 226 nextstate(:crlf)
  74. end
  75. end
  76. 1074 break if @buffer.empty?
  77. end
  78. end
  79. 27 def finished?
  80. 193 @finished
  81. end
  82. 27 private
  83. 27 def nextstate(state)
  84. 1026 @state = state
  85. end
  86. end
  87. 27 module_function
  88. 27 def encode(chunks)
  89. 96 Encoder.new(chunks)
  90. end
  91. end
  92. end

lib/httpx/transcoder/deflate.rb

100.0% lines covered

20 relevant lines. 20 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "zlib"
  3. 27 require_relative "utils/deflater"
  4. 27 module HTTPX
  5. 27 module Transcoder
  6. 27 module Deflate
  7. 27 class Deflater < Transcoder::Deflater
  8. 27 def deflate(chunk)
  9. 72 @deflater ||= Zlib::Deflate.new
  10. 72 if chunk.nil?
  11. 48 unless @deflater.closed?
  12. 24 last = @deflater.finish
  13. 24 @deflater.close
  14. 24 last.empty? ? nil : last
  15. end
  16. else
  17. 24 @deflater.deflate(chunk)
  18. end
  19. end
  20. end
  21. 27 module_function
  22. 27 def encode(body)
  23. 24 Deflater.new(body)
  24. end
  25. 27 def decode(response, bytesize: nil)
  26. 16 bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
  27. 16 GZIP::Inflater.new(bytesize)
  28. end
  29. end
  30. end
  31. end

lib/httpx/transcoder/form.rb

100.0% lines covered

42 relevant lines. 42 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 require "uri"
  4. 27 require_relative "multipart"
  5. 27 module HTTPX
  6. 27 module Transcoder
  7. 27 module Form
  8. 27 module_function
  9. 27 PARAM_DEPTH_LIMIT = 32
  10. 27 class Encoder
  11. 27 extend Forwardable
  12. 27 def_delegator :@raw, :to_s
  13. 27 def_delegator :@raw, :to_str
  14. 27 def_delegator :@raw, :bytesize
  15. 27 def_delegator :@raw, :==
  16. 27 def initialize(form)
  17. 706 @raw = form.each_with_object("".b) do |(key, val), buf|
  18. 1186 HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
  19. 1384 buf << "&" unless buf.empty?
  20. 1384 buf << URI.encode_www_form_component(k)
  21. 1384 buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
  22. end
  23. end
  24. end
  25. 27 def content_type
  26. 554 "application/x-www-form-urlencoded"
  27. end
  28. end
  29. 27 module Decoder
  30. 27 module_function
  31. 27 def call(response, *)
  32. 40 URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
  33. 96 HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
  34. end
  35. end
  36. end
  37. 27 def encode(form)
  38. 1657 if multipart?(form)
  39. 951 Multipart::Encoder.new(form)
  40. else
  41. 706 Encoder.new(form)
  42. end
  43. end
  44. 27 def decode(response)
  45. 64 content_type = response.content_type.mime_type
  46. 56 case content_type
  47. when "application/x-www-form-urlencoded"
  48. 40 Decoder
  49. when "multipart/form-data"
  50. 16 Multipart::Decoder.new(response)
  51. else
  52. 8 raise Error, "invalid form mime type (#{content_type})"
  53. end
  54. end
  55. 27 def multipart?(data)
  56. 1657 data.any? do |_, v|
  57. 2201 Multipart::MULTIPART_VALUE_COND.call(v) ||
  58. 1698 (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
  59. 2082 (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
  60. end
  61. end
  62. end
  63. end
  64. end

lib/httpx/transcoder/gzip.rb

100.0% lines covered

40 relevant lines. 40 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "zlib"
  3. 27 module HTTPX
  4. 27 module Transcoder
  5. 27 module GZIP
  6. 27 class Deflater < Transcoder::Deflater
  7. 27 def initialize(body)
  8. 48 @compressed_chunk = "".b
  9. 48 super
  10. end
  11. 27 def deflate(chunk)
  12. 96 @deflater ||= Zlib::GzipWriter.new(self)
  13. 96 if chunk.nil?
  14. 48 unless @deflater.closed?
  15. 48 @deflater.flush
  16. 48 @deflater.close
  17. 48 compressed_chunk
  18. end
  19. else
  20. 48 @deflater.write(chunk)
  21. 48 compressed_chunk
  22. end
  23. end
  24. 27 private
  25. 27 def write(chunk)
  26. 144 @compressed_chunk << chunk
  27. end
  28. 27 def compressed_chunk
  29. 96 @compressed_chunk.dup
  30. ensure
  31. 96 @compressed_chunk.clear
  32. end
  33. end
  34. 27 class Inflater
  35. 27 def initialize(bytesize)
  36. 163 @inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
  37. 163 @bytesize = bytesize
  38. end
  39. 27 def call(chunk)
  40. 437 buffer = @inflater.inflate(chunk)
  41. 391 @bytesize -= chunk.bytesize
  42. 437 if @bytesize <= 0
  43. 108 buffer << @inflater.finish
  44. 108 @inflater.close
  45. end
  46. 437 buffer
  47. end
  48. end
  49. 27 module_function
  50. 27 def encode(body)
  51. 48 Deflater.new(body)
  52. end
  53. 27 def decode(response, bytesize: nil)
  54. 147 bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
  55. 147 Inflater.new(bytesize)
  56. end
  57. end
  58. end
  59. end

lib/httpx/transcoder/json.rb

100.0% lines covered

33 relevant lines. 33 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 module HTTPX::Transcoder
  4. 27 module JSON
  5. 27 module_function
  6. 27 JSON_REGEX = %r{
  7. \b
  8. application/
  9. # optional vendor specific type
  10. (?:
  11. # token as per https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6
  12. [!#$%&'*+\-.^_`|~0-9a-z]+
  13. # literal plus sign
  14. \+
  15. )?
  16. json
  17. \b
  18. }ix.freeze
  19. 27 class Encoder
  20. 27 extend Forwardable
  21. 27 def_delegator :@raw, :to_s
  22. 27 def_delegator :@raw, :bytesize
  23. 27 def_delegator :@raw, :==
  24. 27 def initialize(json)
  25. 83 @raw = JSON.json_dump(json)
  26. 83 @charset = @raw.encoding.name.downcase
  27. end
  28. 27 def content_type
  29. 83 "application/json; charset=#{@charset}"
  30. end
  31. end
  32. 27 def encode(json)
  33. 83 Encoder.new(json)
  34. end
  35. 27 def decode(response)
  36. 129 content_type = response.content_type.mime_type
  37. 129 raise HTTPX::Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type)
  38. 113 method(:json_load)
  39. end
  40. # rubocop:disable Style/SingleLineMethods
  41. 27 if defined?(MultiJson)
  42. 4 def json_load(*args); MultiJson.load(*args); end
  43. 2 def json_dump(*args); MultiJson.dump(*args); end
  44. 25 elsif defined?(Oj)
  45. 5 def json_load(response, *args); Oj.load(response.to_s, *args); end
  46. 3 def json_dump(obj, options = {}); Oj.dump(obj, { mode: :compat }.merge(options)); end
  47. 23 elsif defined?(Yajl)
  48. 4 def json_load(response, *args); Yajl::Parser.new(*args).parse(response.to_s); end
  49. 2 def json_dump(*args); Yajl::Encoder.encode(*args); end
  50. else
  51. 23 require "json"
  52. 114 def json_load(*args); ::JSON.parse(*args); end
  53. 93 def json_dump(*args); ::JSON.dump(*args); end
  54. end
  55. # rubocop:enable Style/SingleLineMethods
  56. end
  57. end

lib/httpx/transcoder/multipart.rb

100.0% lines covered

10 relevant lines. 10 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require_relative "multipart/encoder"
  3. 27 require_relative "multipart/decoder"
  4. 27 require_relative "multipart/part"
  5. 27 require_relative "multipart/mime_type_detector"
  6. 27 module HTTPX::Transcoder
  7. 27 module Multipart
  8. 27 MULTIPART_VALUE_COND = lambda do |value|
  9. 4878 value.respond_to?(:read) ||
  10. 3506 (value.respond_to?(:to_hash) &&
  11. value.key?(:body) &&
  12. 644 (value.key?(:filename) || value.key?(:content_type)))
  13. end
  14. end
  15. end

lib/httpx/transcoder/multipart/decoder.rb

94.05% lines covered

84 relevant lines. 79 lines covered and 5 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "tempfile"
  3. 27 require "delegate"
  4. 27 module HTTPX
  5. 27 module Transcoder
  6. 27 module Multipart
  7. 27 class FilePart < SimpleDelegator
  8. 27 attr_reader :original_filename, :content_type
  9. 27 def initialize(filename, content_type)
  10. 32 @original_filename = filename
  11. 32 @content_type = content_type
  12. 32 @current = nil
  13. 32 @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  14. 32 super(@file)
  15. end
  16. end
  17. 27 class Decoder
  18. 27 include HTTPX::Utils
  19. 27 CRLF = "\r\n"
  20. 27 BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
  21. 27 MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
  22. 27 MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
  23. 27 MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
  24. 27 WINDOW_SIZE = 2 << 14
  25. 27 def initialize(response)
  26. 2 @boundary = begin
  27. 16 m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
  28. 16 raise Error, "no boundary declared in content-type header" unless m
  29. 16 m.strip
  30. end
  31. 16 @buffer = "".b
  32. 16 @parts = {}
  33. 16 @intermediate_boundary = "--#{@boundary}"
  34. 16 @state = :idle
  35. end
  36. 27 def call(response, *)
  37. 16 response.body.each do |chunk|
  38. 16 @buffer << chunk
  39. 16 parse
  40. end
  41. 16 raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
  42. 16 @parts
  43. end
  44. 27 private
  45. 27 def parse
  46. 14 case @state
  47. when :idle
  48. 16 raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
  49. 16 @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
  50. 16 @state = :part_header
  51. when :part_header
  52. 48 idx = @buffer.index("#{CRLF}#{CRLF}")
  53. # raise Error, "couldn't parse part headers" unless idx
  54. 48 return unless idx
  55. # @type var head: String
  56. 48 head = @buffer.byteslice(0..idx + 4 - 1)
  57. 48 @buffer = @buffer.byteslice(head.bytesize..-1)
  58. 48 content_type = head[MULTIPART_CONTENT_TYPE, 1] || "text/plain"
  59. 84 if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
  60. 48 name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
  61. 48 name.gsub!(/\\(.)/, "\\1")
  62. 12 name
  63. else
  64. name = head[MULTIPART_CONTENT_ID, 1]
  65. end
  66. 48 filename = HTTPX::Utils.get_filename(head)
  67. 48 name = filename || +"#{content_type}[]" if name.nil? || name.empty?
  68. 48 @current = name
  69. 42 @parts[name] = if filename
  70. 32 FilePart.new(filename, content_type)
  71. else
  72. 16 "".b
  73. end
  74. 48 @state = :part_body
  75. when :part_body
  76. 48 part = @parts[@current]
  77. 48 body_separator = if part.is_a?(FilePart)
  78. 28 "#{CRLF}#{CRLF}"
  79. else
  80. 16 CRLF
  81. end
  82. 48 idx = @buffer.index(body_separator)
  83. 48 if idx
  84. 48 payload = @buffer.byteslice(0..idx - 1)
  85. 48 @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
  86. 48 part << payload
  87. 48 part.rewind if part.respond_to?(:rewind)
  88. 48 @state = :parse_boundary
  89. else
  90. part << @buffer
  91. @buffer.clear
  92. end
  93. when :parse_boundary
  94. 48 raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
  95. 48 @buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
  96. 48 if @buffer == "--"
  97. 16 @buffer.clear
  98. 16 @state = :done
  99. 16 return
  100. 31 elsif @buffer.start_with?(CRLF)
  101. 32 @buffer = @buffer.byteslice(2..-1)
  102. 32 @state = :part_header
  103. else
  104. return
  105. end
  106. when :done
  107. raise Error, "parsing should have been over by now"
  108. 20 end until @buffer.empty?
  109. end
  110. end
  111. end
  112. end
  113. end

lib/httpx/transcoder/multipart/encoder.rb

100.0% lines covered

70 relevant lines. 70 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Transcoder::Multipart
  4. 27 class Encoder
  5. 27 attr_reader :bytesize
  6. 27 def initialize(form)
  7. 951 @boundary = ("-" * 21) << SecureRandom.hex(21)
  8. 951 @part_index = 0
  9. 951 @buffer = "".b
  10. 951 @form = form
  11. 951 @bytesize = 0
  12. 951 @parts = to_parts(form)
  13. end
  14. 27 def content_type
  15. 951 "multipart/form-data; boundary=#{@boundary}"
  16. end
  17. 27 def to_s
  18. 18 read || ""
  19. ensure
  20. 18 rewind
  21. end
  22. 27 def read(length = nil, outbuf = nil)
  23. 3518 data = String(outbuf).clear.force_encoding(Encoding::BINARY) if outbuf
  24. 3518 data ||= "".b
  25. 3518 read_chunks(data, length)
  26. 3518 data unless length && data.empty?
  27. end
  28. 27 def rewind
  29. 50 form = @form.each_with_object([]) do |(key, val), aux|
  30. 50 if val.respond_to?(:path) && val.respond_to?(:reopen) && val.respond_to?(:closed?) && val.closed?
  31. # @type var val: File
  32. 50 val = val.reopen(val.path, File::RDONLY)
  33. end
  34. 50 val.rewind if val.respond_to?(:rewind)
  35. 50 aux << [key, val]
  36. end
  37. 50 @form = form
  38. 50 @bytesize = 0
  39. 50 @parts = to_parts(form)
  40. 50 @part_index = 0
  41. end
  42. 27 private
  43. 27 def to_parts(form)
  44. 1001 params = form.each_with_object([]) do |(key, val), aux|
  45. 1193 Transcoder.normalize_keys(key, val, MULTIPART_VALUE_COND) do |k, v|
  46. 1193 next if v.nil?
  47. 1193 value, content_type, filename = Part.call(v)
  48. 1193 header = header_part(k, content_type, filename)
  49. 1049 @bytesize += header.size
  50. 1193 aux << header
  51. 1049 @bytesize += value.size
  52. 1193 aux << value
  53. 1193 delimiter = StringIO.new("\r\n")
  54. 1049 @bytesize += delimiter.size
  55. 1193 aux << delimiter
  56. end
  57. end
  58. 1001 final_delimiter = StringIO.new("--#{@boundary}--\r\n")
  59. 881 @bytesize += final_delimiter.size
  60. 1001 params << final_delimiter
  61. 1001 params
  62. end
  63. 27 def header_part(key, content_type, filename)
  64. 1193 header = "--#{@boundary}\r\n".b
  65. 1193 header << "Content-Disposition: form-data; name=#{key.inspect}".b
  66. 1193 header << "; filename=#{filename.inspect}" if filename
  67. 1193 header << "\r\nContent-Type: #{content_type}\r\n\r\n"
  68. 1193 StringIO.new(header)
  69. end
  70. 27 def read_chunks(buffer, length = nil)
  71. 4618 while @part_index < @parts.size
  72. 10584 chunk = read_from_part(length)
  73. 10584 next unless chunk
  74. 6052 buffer << chunk.force_encoding(Encoding::BINARY)
  75. 6052 next unless length
  76. 5246 length -= chunk.bytesize
  77. 5986 break if length.zero?
  78. end
  79. end
  80. # if there's a current part to read from, tries to read a chunk.
  81. 27 def read_from_part(max_length = nil)
  82. 10584 part = @parts[@part_index]
  83. 10584 chunk = part.read(max_length, @buffer)
  84. 10584 return chunk if chunk && !chunk.empty?
  85. 4532 part.close if part.respond_to?(:close)
  86. 3980 @part_index += 1
  87. 1692 nil
  88. end
  89. end
  90. end
  91. end

lib/httpx/transcoder/multipart/mime_type_detector.rb

92.11% lines covered

38 relevant lines. 35 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Transcoder::Multipart
  4. 27 module MimeTypeDetector
  5. 27 module_function
  6. 27 DEFAULT_MIMETYPE = "application/octet-stream"
  7. # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
  8. 27 if defined?(FileMagic)
  9. 1 MAGIC_NUMBER = 256 * 1024
  10. 1 def call(file, _)
  11. 1 return nil if file.eof? # FileMagic returns "application/x-empty" for empty files
  12. 1 mime = FileMagic.open(FileMagic::MAGIC_MIME_TYPE) do |filemagic|
  13. 1 filemagic.buffer(file.read(MAGIC_NUMBER))
  14. end
  15. 1 file.rewind
  16. 1 mime
  17. end
  18. 25 elsif defined?(Marcel)
  19. 1 def call(file, filename)
  20. 1 return nil if file.eof? # marcel returns "application/octet-stream" for empty files
  21. 1 Marcel::MimeType.for(file, name: filename)
  22. end
  23. 24 elsif defined?(MimeMagic)
  24. 1 def call(file, _)
  25. 1 mime = MimeMagic.by_magic(file)
  26. 1 mime.type if mime
  27. end
  28. 23 elsif system("which file", out: File::NULL)
  29. 24 require "open3"
  30. 24 def call(file, _)
  31. 677 return if file.eof? # file command returns "application/x-empty" for empty files
  32. 633 Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
  33. 75 begin
  34. 633 ::IO.copy_stream(file, stdin.binmode)
  35. rescue Errno::EPIPE
  36. end
  37. 633 file.rewind
  38. 633 stdin.close
  39. 633 status = thread.value
  40. # call to file command failed
  41. 633 if status.nil? || !status.success?
  42. $stderr.print(stderr.read)
  43. else
  44. 633 output = stdout.read.strip
  45. 633 if output.include?("cannot open")
  46. $stderr.print(output)
  47. else
  48. 633 output
  49. end
  50. end
  51. end
  52. end
  53. else
  54. def call(_, _); end
  55. end
  56. end
  57. end
  58. end

lib/httpx/transcoder/multipart/part.rb

100.0% lines covered

18 relevant lines. 18 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Transcoder::Multipart
  4. 27 module Part
  5. 27 module_function
  6. 27 def call(value)
  7. # take out specialized objects of the way
  8. 1193 if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
  9. 112 return value, value.content_type, value.filename
  10. end
  11. 1065 content_type = filename = nil
  12. 1065 if value.is_a?(Hash)
  13. 322 content_type = value[:content_type]
  14. 322 filename = value[:filename]
  15. 322 value = value[:body]
  16. end
  17. 1065 value = value.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
  18. 1065 if value.respond_to?(:path) && value.respond_to?(:read)
  19. # either a File, a Tempfile, or something else which has to quack like a file
  20. 681 filename ||= File.basename(value.path)
  21. 681 content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
  22. 681 [value, content_type, filename]
  23. else
  24. 384 [StringIO.new(value.to_s), content_type || "text/plain", filename]
  25. end
  26. end
  27. end
  28. end
  29. end

lib/httpx/transcoder/utils/body_reader.rb

92.0% lines covered

25 relevant lines. 23 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "stringio"
  3. 27 module HTTPX
  4. 27 module Transcoder
  5. 27 class BodyReader
  6. 27 def initialize(body)
  7. 210 @body = if body.respond_to?(:read)
  8. 20 body.rewind if body.respond_to?(:rewind)
  9. 20 body
  10. 189 elsif body.respond_to?(:each)
  11. 36 body.enum_for(:each)
  12. else
  13. 154 StringIO.new(body.to_s)
  14. end
  15. end
  16. 27 def bytesize
  17. 402 return @body.bytesize if @body.respond_to?(:bytesize)
  18. 366 Float::INFINITY
  19. end
  20. 27 def read(length = nil, outbuf = nil)
  21. 456 return @body.read(length, outbuf) if @body.respond_to?(:read)
  22. begin
  23. 84 chunk = @body.next
  24. 48 if outbuf
  25. outbuf.clear.force_encoding(Encoding::BINARY)
  26. outbuf << chunk
  27. else
  28. 48 outbuf = chunk
  29. end
  30. 48 outbuf unless length && outbuf.empty?
  31. 24 rescue StopIteration
  32. end
  33. end
  34. 27 def close
  35. 48 @body.close if @body.respond_to?(:close)
  36. end
  37. end
  38. end
  39. end

lib/httpx/transcoder/utils/deflater.rb

100.0% lines covered

37 relevant lines. 37 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require_relative "body_reader"
  3. 27 module HTTPX
  4. 27 module Transcoder
  5. 27 class Deflater
  6. 27 attr_reader :content_type
  7. 27 def initialize(body)
  8. 84 @content_type = body.content_type
  9. 84 @body = BodyReader.new(body)
  10. 84 @closed = false
  11. end
  12. 27 def bytesize
  13. 324 buffer_deflate!
  14. 324 @buffer.size
  15. end
  16. 27 def read(length = nil, outbuf = nil)
  17. 416 return @buffer.read(length, outbuf) if @buffer
  18. 240 return if @closed
  19. 192 chunk = @body.read(length)
  20. 192 compressed_chunk = deflate(chunk)
  21. 192 return unless compressed_chunk
  22. 156 if outbuf
  23. 144 outbuf.clear.force_encoding(Encoding::BINARY)
  24. 144 outbuf << compressed_chunk
  25. else
  26. 12 compressed_chunk
  27. end
  28. end
  29. 27 def close
  30. 48 return if @closed
  31. 48 @buffer.close if @buffer
  32. 48 @body.close
  33. 48 @closed = true
  34. end
  35. 27 def rewind
  36. 28 return unless @buffer
  37. 16 @buffer.rewind
  38. end
  39. 27 private
  40. # rubocop:disable Naming/MemoizedInstanceVariableName
  41. 27 def buffer_deflate!
  42. 324 return @buffer if defined?(@buffer)
  43. 84 buffer = Response::Buffer.new(
  44. threshold_size: Options::MAX_BODY_THRESHOLD_SIZE
  45. )
  46. 84 ::IO.copy_stream(self, buffer)
  47. 84 buffer.rewind if buffer.respond_to?(:rewind)
  48. 84 @buffer = buffer
  49. end
  50. # rubocop:enable Naming/MemoizedInstanceVariableName
  51. end
  52. end
  53. end

lib/httpx/utils.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Utils
  4. 27 using URIExtensions
  5. 27 TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
  6. 27 VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
  7. 27 FILENAME_REGEX = /\s*filename=(#{VALUE})/.freeze
  8. 27 FILENAME_EXTENSION_REGEX = /\s*filename\*=(#{VALUE})/.freeze
  9. 27 module_function
  10. 27 def now
  11. 11529699 Process.clock_gettime(Process::CLOCK_MONOTONIC)
  12. end
  13. 27 def elapsed_time(monotonic_timestamp)
  14. 11479686 Process.clock_gettime(Process::CLOCK_MONOTONIC) - monotonic_timestamp
  15. end
  16. # The value of this field can be either an HTTP-date or a number of
  17. # seconds to delay after the response is received.
  18. 27 def parse_retry_after(retry_after)
  19. # first: bet on it being an integer
  20. 63 Integer(retry_after)
  21. rescue ArgumentError
  22. # Then it's a datetime
  23. 16 time = Time.httpdate(retry_after)
  24. 16 time - Time.now
  25. end
  26. 27 def get_filename(header, _prefix_regex = nil)
  27. 88 filename = nil
  28. 77 case header
  29. when FILENAME_REGEX
  30. 56 filename = Regexp.last_match(1)
  31. 56 filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
  32. when FILENAME_EXTENSION_REGEX
  33. 16 filename = Regexp.last_match(1)
  34. 16 encoding, _, filename = filename.split("'", 3)
  35. end
  36. 88 return unless filename
  37. 136 filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
  38. 72 filename.scrub!
  39. 72 filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
  40. 72 filename.force_encoding ::Encoding.find(encoding) if encoding
  41. 72 filename
  42. end
  43. 27 URIParser = URI::RFC2396_Parser.new
  44. 27 def to_uri(uri)
  45. 17913 return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
  46. 33 uri = URI(URIParser.escape(uri))
  47. 33 non_ascii_hostname = URIParser.unescape(uri.host)
  48. 33 non_ascii_hostname.force_encoding(Encoding::UTF_8)
  49. 33 idna_hostname = Punycode.encode_hostname(non_ascii_hostname)
  50. 33 uri.host = idna_hostname
  51. 32 uri.non_ascii_hostname = non_ascii_hostname
  52. 32 uri
  53. end
  54. end
  55. end