require 'open-uri'
require 'iconv'

def get_web_page_as_utf8(url)
  open(url) do |io|
    source = io.read
    type, *parameters = io.content_type_parse
    # Don't transcode anything that isn't (X)HTML
    unless type =~ %r!^(?:text/html|application/xhtml+xml)$!
    return source
  end
  # Check server headers first:
  if pair = parameters.assoc('charset')
    encoding = pair.last
  # Next, look in the HTML:
  elsif source =~ /\]*?charset=([^\s'"]+)/i
    encoding = $1
  # Finally, use the HTTP default
  else
    encoding = 'ISO-8859-1'
  end
  converter = Iconv.new('UTF-8//IGNORE', encoding)
  return converter.iconv(source)
end
  end

