#$@%#$%!

Ngiler SH3LL 360
: / opt / alt / ruby30 / share / ruby /

Information Server
MySQL : OFF Perl : OFF CURL : ON WGET : OFF PKEXEC : OFF

Directive	Local Value
IP Address	162.0.209.101
System	Linux premium169.web-hosting.com 4.18.0-553.50.1.lve.el8.x86_64 #1 SMP Thu Apr 17 19:10:24 UTC 2025 x86_64
User	luxeelil
PHP Version	8.0.30
Software	LiteSpeed
Doc root	Writable

Edit File : open-uri.rb \| Size : 24.74 KB
# frozen_string_literal: true require 'uri' require 'stringio' require 'time' module URI # Allows the opening of various resources including URIs. # # If the first argument responds to the 'open' method, 'open' is called on # it with the rest of the arguments. # # If the first argument is a string that begins with <code>(protocol)://<code>, it is parsed by # URI.parse. If the parsed object responds to the 'open' method, # 'open' is called on it with the rest of the arguments. # # Otherwise, Kernel#open is called. # # OpenURI::OpenRead#open provides URI::HTTP#open, URI::HTTPS#open and # URI::FTP#open, Kernel#open. # # We can accept URIs and strings that begin with http://, https:// and # ftp://. In these cases, the opened file object is extended by OpenURI::Meta. def self.open(name, rest, &block) if name.respond_to?(:open) name.open(rest, &block) elsif name.respond_to?(:to_str) && %r{\A[A-Za-z][A-Za-z0-9+\-\.]://} =~ name && (uri = URI.parse(name)).respond_to?(:open) uri.open(rest, &block) else super end end end # OpenURI is an easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP. # # == Example # # It is possible to open an http, https or ftp URL as though it were a file: # # URI.open("http://www.ruby-lang.org/") {\|f\| # f.each_line {\|line\| p line} # } # # The opened file has several getter methods for its meta-information, as # follows, since it is extended by OpenURI::Meta. # # URI.open("http://www.ruby-lang.org/en") {\|f\| # f.each_line {\|line\| p line} # p f.base_uri # <URI::HTTP:0x40e6ef2 URL:http://www.ruby-lang.org/en/> # p f.content_type # "text/html" # p f.charset # "iso-8859-1" # p f.content_encoding # [] # p f.last_modified # Thu Dec 05 02:45:02 UTC 2002 # } # # Additional header fields can be specified by an optional hash argument. # # URI.open("http://www.ruby-lang.org/en/", # "User-Agent" => "Ruby/#{RUBY_VERSION}", # "From" => "foo@bar.invalid", # "Referer" => "http://www.ruby-lang.org/") {\|f\| # # ... # } # # The environment variables such as http_proxy, https_proxy and ftp_proxy # are in effect by default. Here we disable proxy: # # URI.open("http://www.ruby-lang.org/en/", :proxy => nil) {\|f\| # # ... # } # # See OpenURI::OpenRead.open and URI.open for more on available options. # # URI objects can be opened in a similar way. # # uri = URI.parse("http://www.ruby-lang.org/en/") # uri.open {\|f\| # # ... # } # # URI objects can be read directly. The returned string is also extended by # OpenURI::Meta. # # str = uri.read # p str.base_uri # # Author:: Tanaka Akira <akr@m17n.org> module OpenURI Options = { :proxy => true, :proxy_http_basic_authentication => true, :progress_proc => true, :content_length_proc => true, :http_basic_authentication => true, :read_timeout => true, :open_timeout => true, :ssl_ca_cert => nil, :ssl_verify_mode => nil, :ftp_active_mode => false, :redirect => true, :encoding => nil, } def OpenURI.check_options(options) # :nodoc: options.each {\|k, v\| next unless Symbol === k unless Options.include? k raise ArgumentError, "unrecognized option: #{k}" end } end def OpenURI.scan_open_optional_arguments(rest) # :nodoc: if !rest.empty? && (String === rest.first \|\| Integer === rest.first) mode = rest.shift if !rest.empty? && Integer === rest.first perm = rest.shift end end return mode, perm, rest end def OpenURI.open_uri(name, rest) # :nodoc: uri = URI::Generic === name ? name : URI.parse(name) mode, _, rest = OpenURI.scan_open_optional_arguments(rest) options = rest.shift if !rest.empty? && Hash === rest.first raise ArgumentError.new("extra arguments") if !rest.empty? options \|\|= {} OpenURI.check_options(options) if /\Arb?(?:\Z\|:([^:]+))/ =~ mode encoding, = $1,Encoding.find($1) if $1 mode = nil end if options.has_key? :encoding if !encoding.nil? raise ArgumentError, "encoding specified twice" end encoding = Encoding.find(options[:encoding]) end unless mode == nil \|\| mode == 'r' \|\| mode == 'rb' \|\| mode == File::RDONLY raise ArgumentError.new("invalid access mode #{mode} (#{uri.class} resource is read only.)") end io = open_loop(uri, options) io.set_encoding(encoding) if encoding if block_given? begin yield io ensure if io.respond_to? :close! io.close! # Tempfile else io.close if !io.closed? end end else io end end def OpenURI.open_loop(uri, options) # :nodoc: proxy_opts = [] proxy_opts << :proxy_http_basic_authentication if options.include? :proxy_http_basic_authentication proxy_opts << :proxy if options.include? :proxy proxy_opts.compact! if 1 < proxy_opts.length raise ArgumentError, "multiple proxy options specified" end case proxy_opts.first when :proxy_http_basic_authentication opt_proxy, proxy_user, proxy_pass = options.fetch(:proxy_http_basic_authentication) proxy_user = proxy_user.to_str proxy_pass = proxy_pass.to_str if opt_proxy == true raise ArgumentError.new("Invalid authenticated proxy option: #{options[:proxy_http_basic_authentication].inspect}") end when :proxy opt_proxy = options.fetch(:proxy) proxy_user = nil proxy_pass = nil when nil opt_proxy = true proxy_user = nil proxy_pass = nil end case opt_proxy when true find_proxy = lambda {\|u\| pxy = u.find_proxy; pxy ? [pxy, nil, nil] : nil} when nil, false find_proxy = lambda {\|u\| nil} when String opt_proxy = URI.parse(opt_proxy) find_proxy = lambda {\|u\| [opt_proxy, proxy_user, proxy_pass]} when URI::Generic find_proxy = lambda {\|u\| [opt_proxy, proxy_user, proxy_pass]} else raise ArgumentError.new("Invalid proxy option: #{opt_proxy}") end uri_set = {} buf = nil while true redirect = catch(:open_uri_redirect) { buf = Buffer.new uri.buffer_open(buf, find_proxy.call(uri), options) nil } if redirect if redirect.relative? # Although it violates RFC2616, Location: field may have relative # URI. It is converted to absolute URI using uri as a base URI. redirect = uri + redirect end if !options.fetch(:redirect, true) raise HTTPRedirect.new(buf.io.status.join(' '), buf.io, redirect) end unless OpenURI.redirectable?(uri, redirect) raise "redirection forbidden: #{uri} -> #{redirect}" end if options.include? :http_basic_authentication # send authentication only for the URI directly specified. options = options.dup options.delete :http_basic_authentication end uri = redirect raise "HTTP redirection loop: #{uri}" if uri_set.include? uri.to_s uri_set[uri.to_s] = true else break end end io = buf.io io.base_uri = uri io end def OpenURI.redirectable?(uri1, uri2) # :nodoc: # This test is intended to forbid a redirection from http://... to # file:///etc/passwd, file:///dev/zero, etc. CVE-2011-1521 # https to http redirect is also forbidden intentionally. # It avoids sending secure cookie or referer by non-secure HTTP protocol. # (RFC 2109 4.3.1, RFC 2965 3.3, RFC 2616 15.1.3) # However this is ad hoc. It should be extensible/configurable. uri1.scheme.downcase == uri2.scheme.downcase \|\| (/\A(?:http\|ftp)\z/i =~ uri1.scheme && /\A(?:https?\|ftp)\z/i =~ uri2.scheme) end def OpenURI.open_http(buf, target, proxy, options) # :nodoc: if proxy proxy_uri, proxy_user, proxy_pass = proxy raise "Non-HTTP proxy URI: #{proxy_uri}" if proxy_uri.class != URI::HTTP end if target.userinfo raise ArgumentError, "userinfo not supported. [RFC3986]" end header = {} options.each {\|k, v\| header[k] = v if String === k } require 'net/http' klass = Net::HTTP if URI::HTTP === target # HTTP or HTTPS if proxy unless proxy_user && proxy_pass proxy_user, proxy_pass = proxy_uri.userinfo.split(':') if proxy_uri.userinfo end if proxy_user && proxy_pass klass = Net::HTTP::Proxy(proxy_uri.hostname, proxy_uri.port, proxy_user, proxy_pass) else klass = Net::HTTP::Proxy(proxy_uri.hostname, proxy_uri.port) end end target_host = target.hostname target_port = target.port request_uri = target.request_uri else # FTP over HTTP proxy target_host = proxy_uri.hostname target_port = proxy_uri.port request_uri = target.to_s if proxy_user && proxy_pass header["Proxy-Authorization"] = 'Basic ' + ["#{proxy_user}:#{proxy_pass}"].pack('m0') end end http = proxy ? klass.new(target_host, target_port) : klass.new(target_host, target_port, nil) if target.class == URI::HTTPS require 'net/https' http.use_ssl = true http.verify_mode = options[:ssl_verify_mode] \|\| OpenSSL::SSL::VERIFY_PEER store = OpenSSL::X509::Store.new if options[:ssl_ca_cert] Array(options[:ssl_ca_cert]).each do \|cert\| if File.directory? cert store.add_path cert else store.add_file cert end end else store.set_default_paths end http.cert_store = store end if options.include? :read_timeout http.read_timeout = options[:read_timeout] end if options.include? :open_timeout http.open_timeout = options[:open_timeout] end resp = nil http.start { req = Net::HTTP::Get.new(request_uri, header) if options.include? :http_basic_authentication user, pass = options[:http_basic_authentication] req.basic_auth user, pass end http.request(req) {\|response\| resp = response if options[:content_length_proc] && Net::HTTPSuccess === resp if resp.key?('Content-Length') options[:content_length_proc].call(resp['Content-Length'].to_i) else options[:content_length_proc].call(nil) end end resp.read_body {\|str\| buf << str if options[:progress_proc] && Net::HTTPSuccess === resp options[:progress_proc].call(buf.size) end str.clear } } } io = buf.io io.rewind io.status = [resp.code, resp.message] resp.each_name {\|name\| buf.io.meta_add_field2 name, resp.get_fields(name) } case resp when Net::HTTPSuccess when Net::HTTPMovedPermanently, # 301 Net::HTTPFound, # 302 Net::HTTPSeeOther, # 303 Net::HTTPTemporaryRedirect # 307 begin loc_uri = URI.parse(resp['location']) rescue URI::InvalidURIError raise OpenURI::HTTPError.new(io.status.join(' ') + ' (Invalid Location URI)', io) end throw :open_uri_redirect, loc_uri else raise OpenURI::HTTPError.new(io.status.join(' '), io) end end class HTTPError < StandardError def initialize(message, io) super(message) @io = io end attr_reader :io end # Raised on redirection, # only occurs when +redirect+ option for HTTP is +false+. class HTTPRedirect < HTTPError def initialize(message, io, uri) super(message, io) @uri = uri end attr_reader :uri end class Buffer # :nodoc: all def initialize @io = StringIO.new @size = 0 end attr_reader :size StringMax = 10240 def <<(str) @io << str @size += str.length if StringIO === @io && StringMax < @size require 'tempfile' io = Tempfile.new('open-uri') io.binmode Meta.init io, @io if Meta === @io io << @io.string @io = io end end def io Meta.init @io unless Meta === @io @io end end # Mixin for holding meta-information. module Meta def Meta.init(obj, src=nil) # :nodoc: obj.extend Meta obj.instance_eval { @base_uri = nil @meta = {} # name to string. legacy. @metas = {} # name to array of strings. } if src obj.status = src.status obj.base_uri = src.base_uri src.metas.each {\|name, values\| obj.meta_add_field2(name, values) } end end # returns an Array that consists of status code and message. attr_accessor :status # returns a URI that is the base of relative URIs in the data. # It may differ from the URI supplied by a user due to redirection. attr_accessor :base_uri # returns a Hash that represents header fields. # The Hash keys are downcased for canonicalization. # The Hash values are a field body. # If there are multiple field with same field name, # the field values are concatenated with a comma. attr_reader :meta # returns a Hash that represents header fields. # The Hash keys are downcased for canonicalization. # The Hash value are an array of field values. attr_reader :metas def meta_setup_encoding # :nodoc: charset = self.charset enc = nil if charset begin enc = Encoding.find(charset) rescue ArgumentError end end enc = Encoding::ASCII_8BIT unless enc if self.respond_to? :force_encoding self.force_encoding(enc) elsif self.respond_to? :string self.string.force_encoding(enc) else # Tempfile self.set_encoding enc end end def meta_add_field2(name, values) # :nodoc: name = name.downcase @metas[name] = values @meta[name] = values.join(', ') meta_setup_encoding if name == 'content-type' end def meta_add_field(name, value) # :nodoc: meta_add_field2(name, [value]) end # returns a Time that represents the Last-Modified field. def last_modified if vs = @metas['last-modified'] v = vs.join(', ') Time.httpdate(v) else nil end end # :stopdoc: RE_LWS = /[\r\n\t ]+/n RE_TOKEN = %r{[^\x00- ()<>@,;:\\"/\[\]?={}\x7f]+}n RE_QUOTED_STRING = %r{"(?:[\r\n\t !#-\[\]-~\x80-\xff]\|\\[\x00-\x7f])"}n RE_PARAMETERS = %r{(?:;#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?=#{RE_LWS}?(?:#{RE_TOKEN}\|#{RE_QUOTED_STRING})#{RE_LWS}?)}n # :startdoc: def content_type_parse # :nodoc: vs = @metas['content-type'] # The last (?:;#{RE_LWS}?)? matches extra ";" which violates RFC2045. if vs && %r{\A#{RE_LWS}?(#{RE_TOKEN})#{RE_LWS}?/(#{RE_TOKEN})#{RE_LWS}?(#{RE_PARAMETERS})(?:;#{RE_LWS}?)?\z}no =~ vs.join(', ') type = $1.downcase subtype = $2.downcase parameters = [] $3.scan(/;#{RE_LWS}?(#{RE_TOKEN})#{RE_LWS}?=#{RE_LWS}?(?:(#{RE_TOKEN})\|(#{RE_QUOTED_STRING}))/no) {\|att, val, qval\| if qval val = qval[1...-1].gsub(/[\r\n\t !#-\[\]-~\x80-\xff]+\|(\\[\x00-\x7f])/n) { $1 ? $1[1,1] : $& } end parameters << [att.downcase, val] } ["#{type}/#{subtype}", parameters] else nil end end # returns "type/subtype" which is MIME Content-Type. # It is downcased for canonicalization. # Content-Type parameters are stripped. def content_type type, _ = content_type_parse type \|\| 'application/octet-stream' end # returns a charset parameter in Content-Type field. # It is downcased for canonicalization. # # If charset parameter is not given but a block is given, # the block is called and its result is returned. # It can be used to guess charset. # # If charset parameter and block is not given, # nil is returned except text type. # In that case, "utf-8" is returned as defined by RFC6838 4.2.1 def charset type, parameters = content_type_parse if pair = parameters.assoc('charset') pair.last.downcase elsif block_given? yield elsif type && %r{\Atext/} =~ type "utf-8" # RFC6838 4.2.1 else nil end end # Returns a list of encodings in Content-Encoding field as an array of # strings. # # The encodings are downcased for canonicalization. def content_encoding vs = @metas['content-encoding'] if vs && %r{\A#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?(?:,#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?)}o =~ (v = vs.join(', ')) v.scan(RE_TOKEN).map {\|content_coding\| content_coding.downcase} else [] end end end # Mixin for HTTP and FTP URIs. module OpenRead # OpenURI::OpenRead#open provides `open' for URI::HTTP and URI::FTP. # # OpenURI::OpenRead#open takes optional 3 arguments as: # # OpenURI::OpenRead#open([mode [, perm]] [, options]) [{\|io\| ... }] # # OpenURI::OpenRead#open returns an IO-like object if block is not given. # Otherwise it yields the IO object and return the value of the block. # The IO object is extended with OpenURI::Meta. # # +mode+ and +perm+ are the same as Kernel#open. # # However, +mode+ must be read mode because OpenURI::OpenRead#open doesn't # support write mode (yet). # Also +perm+ is ignored because it is meaningful only for file creation. # # +options+ must be a hash. # # Each option with a string key specifies an extra header field for HTTP. # I.e., it is ignored for FTP without HTTP proxy. # # The hash may include other options, where keys are symbols: # # [:proxy] # Synopsis: # :proxy => "http://proxy.foo.com:8000/" # :proxy => URI.parse("http://proxy.foo.com:8000/") # :proxy => true # :proxy => false # :proxy => nil # # If :proxy option is specified, the value should be String, URI, # boolean or nil. # # When String or URI is given, it is treated as proxy URI. # # When true is given or the option itself is not specified, # environment variable `scheme_proxy' is examined. # `scheme' is replaced by `http', `https' or `ftp'. # # When false or nil is given, the environment variables are ignored and # connection will be made to a server directly. # # [:proxy_http_basic_authentication] # Synopsis: # :proxy_http_basic_authentication => # ["http://proxy.foo.com:8000/", "proxy-user", "proxy-password"] # :proxy_http_basic_authentication => # [URI.parse("http://proxy.foo.com:8000/"), # "proxy-user", "proxy-password"] # # If :proxy option is specified, the value should be an Array with 3 # elements. It should contain a proxy URI, a proxy user name and a proxy # password. The proxy URI should be a String, an URI or nil. The proxy # user name and password should be a String. # # If nil is given for the proxy URI, this option is just ignored. # # If :proxy and :proxy_http_basic_authentication is specified, # ArgumentError is raised. # # [:http_basic_authentication] # Synopsis: # :http_basic_authentication=>[user, password] # # If :http_basic_authentication is specified, # the value should be an array which contains 2 strings: # username and password. # It is used for HTTP Basic authentication defined by RFC 2617. # # [:content_length_proc] # Synopsis: # :content_length_proc => lambda {\|content_length\| ... } # # If :content_length_proc option is specified, the option value procedure # is called before actual transfer is started. # It takes one argument, which is expected content length in bytes. # # If two or more transfers are performed by HTTP redirection, the # procedure is called only once for the last transfer. # # When expected content length is unknown, the procedure is called with # nil. This happens when the HTTP response has no Content-Length header. # # [:progress_proc] # Synopsis: # :progress_proc => lambda {\|size\| ...} # # If :progress_proc option is specified, the proc is called with one # argument each time when `open' gets content fragment from network. # The argument +size+ is the accumulated transferred size in bytes. # # If two or more transfer is done by HTTP redirection, the procedure # is called only one for a last transfer. # # :progress_proc and :content_length_proc are intended to be used for # progress bar. # For example, it can be implemented as follows using Ruby/ProgressBar. # # pbar = nil # open("http://...", # :content_length_proc => lambda {\|t\| # if t && 0 < t # pbar = ProgressBar.new("...", t) # pbar.file_transfer_mode # end # }, # :progress_proc => lambda {\|s\| # pbar.set s if pbar # }) {\|f\| ... } # # [:read_timeout] # Synopsis: # :read_timeout=>nil (no timeout) # :read_timeout=>10 (10 second) # # :read_timeout option specifies a timeout of read for http connections. # # [:open_timeout] # Synopsis: # :open_timeout=>nil (no timeout) # :open_timeout=>10 (10 second) # # :open_timeout option specifies a timeout of open for http connections. # # [:ssl_ca_cert] # Synopsis: # :ssl_ca_cert=>filename or an Array of filenames # # :ssl_ca_cert is used to specify CA certificate for SSL. # If it is given, default certificates are not used. # # [:ssl_verify_mode] # Synopsis: # :ssl_verify_mode=>mode # # :ssl_verify_mode is used to specify openssl verify mode. # # [:ftp_active_mode] # Synopsis: # :ftp_active_mode=>bool # # <tt>:ftp_active_mode => true</tt> is used to make ftp active mode. # Ruby 1.9 uses passive mode by default. # Note that the active mode is default in Ruby 1.8 or prior. # # [:redirect] # Synopsis: # :redirect=>bool # # +:redirect+ is true by default. <tt>:redirect => false</tt> is used to # disable all HTTP redirects. # # OpenURI::HTTPRedirect exception raised on redirection. # Using +true+ also means that redirections between http and ftp are # permitted. # def open(rest, &block) OpenURI.open_uri(self, *rest, &block) end # OpenURI::OpenRead#read([ options ]) reads a content referenced by self and # returns the content as string. # The string is extended with OpenURI::Meta. # The argument +options+ is same as OpenURI::OpenRead#open. def read(options={}) self.open(options) {\|f\| str = f.read Meta.init str, f str } end end end module URI class HTTP def buffer_open(buf, proxy, options) # :nodoc: OpenURI.open_http(buf, self, proxy, options) end include OpenURI::OpenRead end class FTP def buffer_open(buf, proxy, options) # :nodoc: if proxy OpenURI.open_http(buf, self, proxy, options) return end require 'net/ftp' path = self.path path = path.sub(%r{\A/}, '%2F') # re-encode the beginning slash because uri library decodes it. directories = path.split(%r{/}, -1) directories.each {\|d\| d.gsub!(/%([0-9A-Fa-f][0-9A-Fa-f])/) { [$1].pack("H2") } } unless filename = directories.pop raise ArgumentError, "no filename: #{self.inspect}" end directories.each {\|d\| if /[\r\n]/ =~ d raise ArgumentError, "invalid directory: #{d.inspect}" end } if /[\r\n]/ =~ filename raise ArgumentError, "invalid filename: #{filename.inspect}" end typecode = self.typecode if typecode && /\A[aid]\z/ !~ typecode raise ArgumentError, "invalid typecode: #{typecode.inspect}" end # The access sequence is defined by RFC 1738 ftp = Net::FTP.new ftp.connect(self.hostname, self.port) ftp.passive = !options[:ftp_active_mode] # todo: extract user/passwd from .netrc. user = 'anonymous' passwd = nil user, passwd = self.userinfo.split(/:/) if self.userinfo ftp.login(user, passwd) directories.each {\|cwd\| ftp.voidcmd("CWD #{cwd}") } if typecode # xxx: typecode D is not handled. ftp.voidcmd("TYPE #{typecode.upcase}") end if options[:content_length_proc] options[:content_length_proc].call(ftp.size(filename)) end ftp.retrbinary("RETR #{filename}", 4096) { \|str\| buf << str options[:progress_proc].call(buf.size) if options[:progress_proc] } ftp.close buf.io.rewind end include OpenURI::OpenRead end end Back

Edit File : open-uri.rb | Size : 24.74 KB

# frozen_string_literal: true
require 'uri'
require 'stringio'
require 'time'

module URI
  # Allows the opening of various resources including URIs.
  #
  # If the first argument responds to the 'open' method, 'open' is called on
  # it with the rest of the arguments.
  #
  # If the first argument is a string that begins with <code>(protocol)://<code>, it is parsed by
  # URI.parse.  If the parsed object responds to the 'open' method,
  # 'open' is called on it with the rest of the arguments.
  #
  # Otherwise, Kernel#open is called.
  #
  # OpenURI::OpenRead#open provides URI::HTTP#open, URI::HTTPS#open and
  # URI::FTP#open, Kernel#open.
  #
  # We can accept URIs and strings that begin with http://, https:// and
  # ftp://. In these cases, the opened file object is extended by OpenURI::Meta.
  def self.open(name, *rest, &block)
    if name.respond_to?(:open)
      name.open(*rest, &block)
    elsif name.respond_to?(:to_str) &&
          %r{\A[A-Za-z][A-Za-z0-9+\-\.]*://} =~ name &&
          (uri = URI.parse(name)).respond_to?(:open)
      uri.open(*rest, &block)
    else
      super
    end
  end
end

# OpenURI is an easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP.
#
# == Example
#
# It is possible to open an http, https or ftp URL as though it were a file:
#
#   URI.open("http://www.ruby-lang.org/") {|f|
#     f.each_line {|line| p line}
#   }
#
# The opened file has several getter methods for its meta-information, as
# follows, since it is extended by OpenURI::Meta.
#
#   URI.open("http://www.ruby-lang.org/en") {|f|
#     f.each_line {|line| p line}
#     p f.base_uri         # <URI::HTTP:0x40e6ef2 URL:http://www.ruby-lang.org/en/>
#     p f.content_type     # "text/html"
#     p f.charset          # "iso-8859-1"
#     p f.content_encoding # []
#     p f.last_modified    # Thu Dec 05 02:45:02 UTC 2002
#   }
#
# Additional header fields can be specified by an optional hash argument.
#
#   URI.open("http://www.ruby-lang.org/en/",
#     "User-Agent" => "Ruby/#{RUBY_VERSION}",
#     "From" => "foo@bar.invalid",
#     "Referer" => "http://www.ruby-lang.org/") {|f|
#     # ...
#   }
#
# The environment variables such as http_proxy, https_proxy and ftp_proxy
# are in effect by default. Here we disable proxy:
#
#   URI.open("http://www.ruby-lang.org/en/", :proxy => nil) {|f|
#     # ...
#   }
#
# See OpenURI::OpenRead.open and URI.open for more on available options.
#
# URI objects can be opened in a similar way.
#
#   uri = URI.parse("http://www.ruby-lang.org/en/")
#   uri.open {|f|
#     # ...
#   }
#
# URI objects can be read directly. The returned string is also extended by
# OpenURI::Meta.
#
#   str = uri.read
#   p str.base_uri
#
# Author:: Tanaka Akira <akr@m17n.org>

module OpenURI
  Options = {
    :proxy => true,
    :proxy_http_basic_authentication => true,
    :progress_proc => true,
    :content_length_proc => true,
    :http_basic_authentication => true,
    :read_timeout => true,
    :open_timeout => true,
    :ssl_ca_cert => nil,
    :ssl_verify_mode => nil,
    :ftp_active_mode => false,
    :redirect => true,
    :encoding => nil,
  }

def OpenURI.check_options(options) # :nodoc:
    options.each {|k, v|
      next unless Symbol === k
      unless Options.include? k
        raise ArgumentError, "unrecognized option: #{k}"
      end
    }
  end

def OpenURI.scan_open_optional_arguments(*rest) # :nodoc:
    if !rest.empty? && (String === rest.first || Integer === rest.first)
      mode = rest.shift
      if !rest.empty? && Integer === rest.first
        perm = rest.shift
      end
    end
    return mode, perm, rest
  end

def OpenURI.open_uri(name, *rest) # :nodoc:
    uri = URI::Generic === name ? name : URI.parse(name)
    mode, _, rest = OpenURI.scan_open_optional_arguments(*rest)
    options = rest.shift if !rest.empty? && Hash === rest.first
    raise ArgumentError.new("extra arguments") if !rest.empty?
    options ||= {}
    OpenURI.check_options(options)

if /\Arb?(?:\Z|:([^:]+))/ =~ mode
      encoding, = $1,Encoding.find($1) if $1
      mode = nil
    end
    if options.has_key? :encoding
      if !encoding.nil?
        raise ArgumentError, "encoding specified twice"
      end
      encoding = Encoding.find(options[:encoding])
    end

unless mode == nil ||
           mode == 'r' || mode == 'rb' ||
           mode == File::RDONLY
      raise ArgumentError.new("invalid access mode #{mode} (#{uri.class} resource is read only.)")
    end

io = open_loop(uri, options)
    io.set_encoding(encoding) if encoding
    if block_given?
      begin
        yield io
      ensure
        if io.respond_to? :close!
          io.close! # Tempfile
        else
          io.close if !io.closed?
        end
      end
    else
      io
    end
  end

def OpenURI.open_loop(uri, options) # :nodoc:
    proxy_opts = []
    proxy_opts << :proxy_http_basic_authentication if options.include? :proxy_http_basic_authentication
    proxy_opts << :proxy if options.include? :proxy
    proxy_opts.compact!
    if 1 < proxy_opts.length
      raise ArgumentError, "multiple proxy options specified"
    end
    case proxy_opts.first
    when :proxy_http_basic_authentication
      opt_proxy, proxy_user, proxy_pass = options.fetch(:proxy_http_basic_authentication)
      proxy_user = proxy_user.to_str
      proxy_pass = proxy_pass.to_str
      if opt_proxy == true
        raise ArgumentError.new("Invalid authenticated proxy option: #{options[:proxy_http_basic_authentication].inspect}")
      end
    when :proxy
      opt_proxy = options.fetch(:proxy)
      proxy_user = nil
      proxy_pass = nil
    when nil
      opt_proxy = true
      proxy_user = nil
      proxy_pass = nil
    end
    case opt_proxy
    when true
      find_proxy = lambda {|u| pxy = u.find_proxy; pxy ? [pxy, nil, nil] : nil}
    when nil, false
      find_proxy = lambda {|u| nil}
    when String
      opt_proxy = URI.parse(opt_proxy)
      find_proxy = lambda {|u| [opt_proxy, proxy_user, proxy_pass]}
    when URI::Generic
      find_proxy = lambda {|u| [opt_proxy, proxy_user, proxy_pass]}
    else
      raise ArgumentError.new("Invalid proxy option: #{opt_proxy}")
    end

uri_set = {}
    buf = nil
    while true
      redirect = catch(:open_uri_redirect) {
        buf = Buffer.new
        uri.buffer_open(buf, find_proxy.call(uri), options)
        nil
      }
      if redirect
        if redirect.relative?
          # Although it violates RFC2616, Location: field may have relative
          # URI.  It is converted to absolute URI using uri as a base URI.
          redirect = uri + redirect
        end
        if !options.fetch(:redirect, true)
          raise HTTPRedirect.new(buf.io.status.join(' '), buf.io, redirect)
        end
        unless OpenURI.redirectable?(uri, redirect)
          raise "redirection forbidden: #{uri} -> #{redirect}"
        end
        if options.include? :http_basic_authentication
          # send authentication only for the URI directly specified.
          options = options.dup
          options.delete :http_basic_authentication
        end
        uri = redirect
        raise "HTTP redirection loop: #{uri}" if uri_set.include? uri.to_s
        uri_set[uri.to_s] = true
      else
        break
      end
    end
    io = buf.io
    io.base_uri = uri
    io
  end

def OpenURI.redirectable?(uri1, uri2) # :nodoc:
    # This test is intended to forbid a redirection from http://... to
    # file:///etc/passwd, file:///dev/zero, etc.  CVE-2011-1521
    # https to http redirect is also forbidden intentionally.
    # It avoids sending secure cookie or referer by non-secure HTTP protocol.
    # (RFC 2109 4.3.1, RFC 2965 3.3, RFC 2616 15.1.3)
    # However this is ad hoc.  It should be extensible/configurable.
    uri1.scheme.downcase == uri2.scheme.downcase ||
    (/\A(?:http|ftp)\z/i =~ uri1.scheme && /\A(?:https?|ftp)\z/i =~ uri2.scheme)
  end

def OpenURI.open_http(buf, target, proxy, options) # :nodoc:
    if proxy
      proxy_uri, proxy_user, proxy_pass = proxy
      raise "Non-HTTP proxy URI: #{proxy_uri}" if proxy_uri.class != URI::HTTP
    end

if target.userinfo
      raise ArgumentError, "userinfo not supported.  [RFC3986]"
    end

header = {}
    options.each {|k, v| header[k] = v if String === k }

require 'net/http'
    klass = Net::HTTP
    if URI::HTTP === target
      # HTTP or HTTPS
      if proxy
        unless proxy_user && proxy_pass
          proxy_user, proxy_pass = proxy_uri.userinfo.split(':') if proxy_uri.userinfo
        end
        if proxy_user && proxy_pass
          klass = Net::HTTP::Proxy(proxy_uri.hostname, proxy_uri.port, proxy_user, proxy_pass)
        else
          klass = Net::HTTP::Proxy(proxy_uri.hostname, proxy_uri.port)
        end
      end
      target_host = target.hostname
      target_port = target.port
      request_uri = target.request_uri
    else
      # FTP over HTTP proxy
      target_host = proxy_uri.hostname
      target_port = proxy_uri.port
      request_uri = target.to_s
      if proxy_user && proxy_pass
        header["Proxy-Authorization"] =
                        'Basic ' + ["#{proxy_user}:#{proxy_pass}"].pack('m0')
      end
    end

http = proxy ? klass.new(target_host, target_port) : klass.new(target_host, target_port, nil)
    if target.class == URI::HTTPS
      require 'net/https'
      http.use_ssl = true
      http.verify_mode = options[:ssl_verify_mode] || OpenSSL::SSL::VERIFY_PEER
      store = OpenSSL::X509::Store.new
      if options[:ssl_ca_cert]
        Array(options[:ssl_ca_cert]).each do |cert|
          if File.directory? cert
            store.add_path cert
          else
            store.add_file cert
          end
        end
      else
        store.set_default_paths
      end
      http.cert_store = store
    end
    if options.include? :read_timeout
      http.read_timeout = options[:read_timeout]
    end
    if options.include? :open_timeout
      http.open_timeout = options[:open_timeout]
    end

resp = nil
    http.start {
      req = Net::HTTP::Get.new(request_uri, header)
      if options.include? :http_basic_authentication
        user, pass = options[:http_basic_authentication]
        req.basic_auth user, pass
      end
      http.request(req) {|response|
        resp = response
        if options[:content_length_proc] && Net::HTTPSuccess === resp
          if resp.key?('Content-Length')
            options[:content_length_proc].call(resp['Content-Length'].to_i)
          else
            options[:content_length_proc].call(nil)
          end
        end
        resp.read_body {|str|
          buf << str
          if options[:progress_proc] && Net::HTTPSuccess === resp
            options[:progress_proc].call(buf.size)
          end
          str.clear
        }
      }
    }
    io = buf.io
    io.rewind
    io.status = [resp.code, resp.message]
    resp.each_name {|name| buf.io.meta_add_field2 name, resp.get_fields(name) }
    case resp
    when Net::HTTPSuccess
    when Net::HTTPMovedPermanently, # 301
         Net::HTTPFound, # 302
         Net::HTTPSeeOther, # 303
         Net::HTTPTemporaryRedirect # 307
      begin
        loc_uri = URI.parse(resp['location'])
      rescue URI::InvalidURIError
        raise OpenURI::HTTPError.new(io.status.join(' ') + ' (Invalid Location URI)', io)
      end
      throw :open_uri_redirect, loc_uri
    else
      raise OpenURI::HTTPError.new(io.status.join(' '), io)
    end
  end

class HTTPError < StandardError
    def initialize(message, io)
      super(message)
      @io = io
    end
    attr_reader :io
  end

# Raised on redirection,
  # only occurs when +redirect+ option for HTTP is +false+.
  class HTTPRedirect < HTTPError
    def initialize(message, io, uri)
      super(message, io)
      @uri = uri
    end
    attr_reader :uri
  end

class Buffer # :nodoc: all
    def initialize
      @io = StringIO.new
      @size = 0
    end
    attr_reader :size

StringMax = 10240
    def <<(str)
      @io << str
      @size += str.length
      if StringIO === @io && StringMax < @size
        require 'tempfile'
        io = Tempfile.new('open-uri')
        io.binmode
        Meta.init io, @io if Meta === @io
        io << @io.string
        @io = io
      end
    end

def io
      Meta.init @io unless Meta === @io
      @io
    end
  end

# Mixin for holding meta-information.
  module Meta
    def Meta.init(obj, src=nil) # :nodoc:
      obj.extend Meta
      obj.instance_eval {
        @base_uri = nil
        @meta = {} # name to string.  legacy.
        @metas = {} # name to array of strings.
      }
      if src
        obj.status = src.status
        obj.base_uri = src.base_uri
        src.metas.each {|name, values|
          obj.meta_add_field2(name, values)
        }
      end
    end

# returns an Array that consists of status code and message.
    attr_accessor :status

# returns a URI that is the base of relative URIs in the data.
    # It may differ from the URI supplied by a user due to redirection.
    attr_accessor :base_uri

# returns a Hash that represents header fields.
    # The Hash keys are downcased for canonicalization.
    # The Hash values are a field body.
    # If there are multiple field with same field name,
    # the field values are concatenated with a comma.
    attr_reader :meta

# returns a Hash that represents header fields.
    # The Hash keys are downcased for canonicalization.
    # The Hash value are an array of field values.
    attr_reader :metas

def meta_setup_encoding # :nodoc:
      charset = self.charset
      enc = nil
      if charset
        begin
          enc = Encoding.find(charset)
        rescue ArgumentError
        end
      end
      enc = Encoding::ASCII_8BIT unless enc
      if self.respond_to? :force_encoding
        self.force_encoding(enc)
      elsif self.respond_to? :string
        self.string.force_encoding(enc)
      else # Tempfile
        self.set_encoding enc
      end
    end

def meta_add_field2(name, values) # :nodoc:
      name = name.downcase
      @metas[name] = values
      @meta[name] = values.join(', ')
      meta_setup_encoding if name == 'content-type'
    end

def meta_add_field(name, value) # :nodoc:
      meta_add_field2(name, [value])
    end

# returns a Time that represents the Last-Modified field.
    def last_modified
      if vs = @metas['last-modified']
        v = vs.join(', ')
        Time.httpdate(v)
      else
        nil
      end
    end

# :stopdoc:
    RE_LWS = /[\r\n\t ]+/n
    RE_TOKEN = %r{[^\x00- ()<>@,;:\\"/\[\]?={}\x7f]+}n
    RE_QUOTED_STRING = %r{"(?:[\r\n\t !#-\[\]-~\x80-\xff]|\\[\x00-\x7f])*"}n
    RE_PARAMETERS = %r{(?:;#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?=#{RE_LWS}?(?:#{RE_TOKEN}|#{RE_QUOTED_STRING})#{RE_LWS}?)*}n
    # :startdoc:

def content_type_parse # :nodoc:
      vs = @metas['content-type']
      # The last (?:;#{RE_LWS}?)? matches extra ";" which violates RFC2045.
      if vs && %r{\A#{RE_LWS}?(#{RE_TOKEN})#{RE_LWS}?/(#{RE_TOKEN})#{RE_LWS}?(#{RE_PARAMETERS})(?:;#{RE_LWS}?)?\z}no =~ vs.join(', ')
        type = $1.downcase
        subtype = $2.downcase
        parameters = []
        $3.scan(/;#{RE_LWS}?(#{RE_TOKEN})#{RE_LWS}?=#{RE_LWS}?(?:(#{RE_TOKEN})|(#{RE_QUOTED_STRING}))/no) {|att, val, qval|
          if qval
            val = qval[1...-1].gsub(/[\r\n\t !#-\[\]-~\x80-\xff]+|(\\[\x00-\x7f])/n) { $1 ? $1[1,1] : $& }
          end
          parameters << [att.downcase, val]
        }
        ["#{type}/#{subtype}", *parameters]
      else
        nil
      end
    end

# returns "type/subtype" which is MIME Content-Type.
    # It is downcased for canonicalization.
    # Content-Type parameters are stripped.
    def content_type
      type, *_ = content_type_parse
      type || 'application/octet-stream'
    end

# returns a charset parameter in Content-Type field.
    # It is downcased for canonicalization.
    #
    # If charset parameter is not given but a block is given,
    # the block is called and its result is returned.
    # It can be used to guess charset.
    #
    # If charset parameter and block is not given,
    # nil is returned except text type.
    # In that case, "utf-8" is returned as defined by RFC6838 4.2.1
    def charset
      type, *parameters = content_type_parse
      if pair = parameters.assoc('charset')
        pair.last.downcase
      elsif block_given?
        yield
      elsif type && %r{\Atext/} =~ type
        "utf-8" # RFC6838 4.2.1
      else
        nil
      end
    end

# Returns a list of encodings in Content-Encoding field as an array of
    # strings.
    #
    # The encodings are downcased for canonicalization.
    def content_encoding
      vs = @metas['content-encoding']
      if vs && %r{\A#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?(?:,#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?)*}o =~ (v = vs.join(', '))
        v.scan(RE_TOKEN).map {|content_coding| content_coding.downcase}
      else
        []
      end
    end
  end

# Mixin for HTTP and FTP URIs.
  module OpenRead
    # OpenURI::OpenRead#open provides `open' for URI::HTTP and URI::FTP.
    #
    # OpenURI::OpenRead#open takes optional 3 arguments as:
    #
    #   OpenURI::OpenRead#open([mode [, perm]] [, options]) [{|io| ... }]
    #
    # OpenURI::OpenRead#open returns an IO-like object if block is not given.
    # Otherwise it yields the IO object and return the value of the block.
    # The IO object is extended with OpenURI::Meta.
    #
    # +mode+ and +perm+ are the same as Kernel#open.
    #
    # However, +mode+ must be read mode because OpenURI::OpenRead#open doesn't
    # support write mode (yet).
    # Also +perm+ is ignored because it is meaningful only for file creation.
    #
    # +options+ must be a hash.
    #
    # Each option with a string key specifies an extra header field for HTTP.
    # I.e., it is ignored for FTP without HTTP proxy.
    #
    # The hash may include other options, where keys are symbols:
    #
    # [:proxy]
    #  Synopsis:
    #    :proxy => "http://proxy.foo.com:8000/"
    #    :proxy => URI.parse("http://proxy.foo.com:8000/")
    #    :proxy => true
    #    :proxy => false
    #    :proxy => nil
    #
    #  If :proxy option is specified, the value should be String, URI,
    #  boolean or nil.
    #
    #  When String or URI is given, it is treated as proxy URI.
    #
    #  When true is given or the option itself is not specified,
    #  environment variable `scheme_proxy' is examined.
    #  `scheme' is replaced by `http', `https' or `ftp'.
    #
    #  When false or nil is given, the environment variables are ignored and
    #  connection will be made to a server directly.
    #
    # [:proxy_http_basic_authentication]
    #  Synopsis:
    #    :proxy_http_basic_authentication =>
    #      ["http://proxy.foo.com:8000/", "proxy-user", "proxy-password"]
    #    :proxy_http_basic_authentication =>
    #      [URI.parse("http://proxy.foo.com:8000/"),
    #       "proxy-user", "proxy-password"]
    #
    #  If :proxy option is specified, the value should be an Array with 3
    #  elements.  It should contain a proxy URI, a proxy user name and a proxy
    #  password.  The proxy URI should be a String, an URI or nil.  The proxy
    #  user name and password should be a String.
    #
    #  If nil is given for the proxy URI, this option is just ignored.
    #
    #  If :proxy and :proxy_http_basic_authentication is specified,
    #  ArgumentError is raised.
    #
    # [:http_basic_authentication]
    #  Synopsis:
    #    :http_basic_authentication=>[user, password]
    #
    #  If :http_basic_authentication is specified,
    #  the value should be an array which contains 2 strings:
    #  username and password.
    #  It is used for HTTP Basic authentication defined by RFC 2617.
    #
    # [:content_length_proc]
    #  Synopsis:
    #    :content_length_proc => lambda {|content_length| ... }
    #
    #  If :content_length_proc option is specified, the option value procedure
    #  is called before actual transfer is started.
    #  It takes one argument, which is expected content length in bytes.
    #
    #  If two or more transfers are performed by HTTP redirection, the
    #  procedure is called only once for the last transfer.
    #
    #  When expected content length is unknown, the procedure is called with
    #  nil.  This happens when the HTTP response has no Content-Length header.
    #
    # [:progress_proc]
    #  Synopsis:
    #    :progress_proc => lambda {|size| ...}
    #
    #  If :progress_proc option is specified, the proc is called with one
    #  argument each time when `open' gets content fragment from network.
    #  The argument +size+ is the accumulated transferred size in bytes.
    #
    #  If two or more transfer is done by HTTP redirection, the procedure
    #  is called only one for a last transfer.
    #
    #  :progress_proc and :content_length_proc are intended to be used for
    #  progress bar.
    #  For example, it can be implemented as follows using Ruby/ProgressBar.
    #
    #    pbar = nil
    #    open("http://...",
    #      :content_length_proc => lambda {|t|
    #        if t && 0 < t
    #          pbar = ProgressBar.new("...", t)
    #          pbar.file_transfer_mode
    #        end
    #      },
    #      :progress_proc => lambda {|s|
    #        pbar.set s if pbar
    #      }) {|f| ... }
    #
    # [:read_timeout]
    #  Synopsis:
    #    :read_timeout=>nil     (no timeout)
    #    :read_timeout=>10      (10 second)
    #
    #  :read_timeout option specifies a timeout of read for http connections.
    #
    # [:open_timeout]
    #  Synopsis:
    #    :open_timeout=>nil     (no timeout)
    #    :open_timeout=>10      (10 second)
    #
    #  :open_timeout option specifies a timeout of open for http connections.
    #
    # [:ssl_ca_cert]
    #  Synopsis:
    #    :ssl_ca_cert=>filename or an Array of filenames
    #
    #  :ssl_ca_cert is used to specify CA certificate for SSL.
    #  If it is given, default certificates are not used.
    #
    # [:ssl_verify_mode]
    #  Synopsis:
    #    :ssl_verify_mode=>mode
    #
    #  :ssl_verify_mode is used to specify openssl verify mode.
    #
    # [:ftp_active_mode]
    #  Synopsis:
    #    :ftp_active_mode=>bool
    #
    #  <tt>:ftp_active_mode => true</tt> is used to make ftp active mode.
    #  Ruby 1.9 uses passive mode by default.
    #  Note that the active mode is default in Ruby 1.8 or prior.
    #
    # [:redirect]
    #  Synopsis:
    #    :redirect=>bool
    #
    #  +:redirect+ is true by default.  <tt>:redirect => false</tt> is used to
    #  disable all HTTP redirects.
    #
    #  OpenURI::HTTPRedirect exception raised on redirection.
    #  Using +true+ also means that redirections between http and ftp are
    #  permitted.
    #
    def open(*rest, &block)
      OpenURI.open_uri(self, *rest, &block)
    end

# OpenURI::OpenRead#read([ options ]) reads a content referenced by self and
    # returns the content as string.
    # The string is extended with OpenURI::Meta.
    # The argument +options+ is same as OpenURI::OpenRead#open.
    def read(options={})
      self.open(options) {|f|
        str = f.read
        Meta.init str, f
        str
      }
    end
  end
end

module URI
  class HTTP
    def buffer_open(buf, proxy, options) # :nodoc:
      OpenURI.open_http(buf, self, proxy, options)
    end

include OpenURI::OpenRead
  end

class FTP
    def buffer_open(buf, proxy, options) # :nodoc:
      if proxy
        OpenURI.open_http(buf, self, proxy, options)
        return
      end
      require 'net/ftp'

path = self.path
      path = path.sub(%r{\A/}, '%2F') # re-encode the beginning slash because uri library decodes it.
      directories = path.split(%r{/}, -1)
      directories.each {|d|
        d.gsub!(/%([0-9A-Fa-f][0-9A-Fa-f])/) { [$1].pack("H2") }
      }
      unless filename = directories.pop
        raise ArgumentError, "no filename: #{self.inspect}"
      end
      directories.each {|d|
        if /[\r\n]/ =~ d
          raise ArgumentError, "invalid directory: #{d.inspect}"
        end
      }
      if /[\r\n]/ =~ filename
        raise ArgumentError, "invalid filename: #{filename.inspect}"
      end
      typecode = self.typecode
      if typecode && /\A[aid]\z/ !~ typecode
        raise ArgumentError, "invalid typecode: #{typecode.inspect}"
      end

# The access sequence is defined by RFC 1738
      ftp = Net::FTP.new
      ftp.connect(self.hostname, self.port)
      ftp.passive = !options[:ftp_active_mode]
      # todo: extract user/passwd from .netrc.
      user = 'anonymous'
      passwd = nil
      user, passwd = self.userinfo.split(/:/) if self.userinfo
      ftp.login(user, passwd)
      directories.each {|cwd|
        ftp.voidcmd("CWD #{cwd}")
      }
      if typecode
        # xxx: typecode D is not handled.
        ftp.voidcmd("TYPE #{typecode.upcase}")
      end
      if options[:content_length_proc]
        options[:content_length_proc].call(ftp.size(filename))
      end
      ftp.retrbinary("RETR #{filename}", 4096) { |str|
        buf << str
        options[:progress_proc].call(buf.size) if options[:progress_proc]
      }
      ftp.close
      buf.io.rewind
    end

include OpenURI::OpenRead
  end
end

Back