Web Site Crawler

##
# This module requires Metasploit: https://metasploit.com/download
# Current source: https://github.com/rapid7/metasploit-framework
##



class MetasploitModule < Msf::Auxiliary
  include Msf::Auxiliary::HttpCrawler

  def initialize
    super(
      'Name'        => 'Web Site Crawler',
      'Description' => 'Crawl a web site and store information about what was found',
      'Author'      => %w(hdm tasos),
      'License'     => MSF_LICENSE
    )

    register_advanced_options([
      OptString.new('ExcludePathPatterns', [false, 'Newline-separated list of path patterns to ignore (\'*\' is a wildcard)']),
    ])
    @for_each_page_blocks = []
  end

=begin
  # Prefer dynamic content over non-dynamic
  def focus_crawl(page)
    page.links
  end
=end

  # Overrides Msf::Auxiliary::HttpCrawler#get_link_filter to add
  # datastore['ExcludePathPatterns']
  def get_link_filter
    return super if datastore['ExcludePathPatterns'].to_s.empty?

    patterns = opt_patterns_to_regexps( datastore['ExcludePathPatterns'].to_s )
    patterns = patterns.map { |r| "(#{r.source})" }

    Regexp.new( [["(#{super.source})"] | patterns].join( '|' ) )
  end

  def run
    super

    if form = form_from_url( @current_site, datastore['URI'] )
      print_status((" " * 24) + "FORM: #{form[:method]} #{form[:path]}")
      report_web_form( form )
      self.form_count += 1
    end
  end

  def for_each_page( &block )
    @for_each_page_blocks << block if block_given?
  end

  #
  # The main callback from the crawler, redefines crawler_process_page() as
  # defined by Msf::Auxiliary::HttpCrawler
  #
  # Data we will report:
  # - The path of any URL found by the crawler (web.uri, :path => page.path)
  # - The occurrence of any form (web.form :path, :type (get|post|path_info), :params)
  #
  def crawler_process_page(t, page, cnt)
    return if page.nil? # Skip over pages that don't contain any info aka page is nil. We can't process these types of pages since there is no data to process.
    msg = "[#{"%.5d" % cnt}/#{"%.5d" % max_page_count}]    #{page ? page.code || "ERR" : "ERR"} - #{t[:vhost]} - #{page.url}"
    if page.error
      print_error("Error accessing page #{page.error.to_s}")
      elog(page.error)
    end
    case page.code
      when 301,302
        if page.headers and page.headers["location"]
          print_status(msg + " -> " + page.headers["location"].to_s)
        else
          print_status(msg)
        end
      when 500...599
        # XXX: Log the fact that we hit an error page
        print_good(msg)
      when 401,403
        print_good(msg)
      when 200
        print_status(msg)
      when 404
        print_error(msg)
      else
        print_error(msg)
    end

    #
    # Process the web page
    #

    info = {
      :web_site => t[:site],
      :path     => page.url.path,
      :query    => page.url.query,
      :code     => page.code,
      :body     => page.body,
      :headers  => page.headers
    }

    if page.headers['content-type']
      info[:ctype] = page.headers['content-type']
    end

    if !page.cookies.empty?
      info[:cookie] = page.cookies
    end

    if page.headers['authorization']
      info[:auth] = page.headers['authorization']
    end

    if page.headers['location']
      info[:location] = page.headers['location']
    end

    if page.headers['last-modified']
      info[:mtime] = page.headers['last-modified']
    end

    # Report the web page to the database
    report_web_page(info)

    # Only process interesting response codes
    return if not [302, 301, 200, 500, 401, 403, 404].include?(page.code)

    #
    # Skip certain types of forms right off the bat
    #

    # Apache multiview directories
    return if page.url.query =~ /^C=[A-Z];O=/ # Apache

    forms = []
    form_template = { :web_site => t[:site] }

    if form = form_from_url( t[:site], page.url )
      forms << form
    end

    if page.doc
      page.doc.css("form").each do |f|

        target = page.url

        if f['action'] and not f['action'].strip.empty?
          action = f['action']

          # Prepend relative URLs with the current directory
          if action[0,1] != "/" and action !~ /\:\/\//
            # Extract the base href first
            base = target.path.gsub(/(.*\/)[^\/]+$/, "\\1")
            page.doc.css("base").each do |bref|
              if bref['href']
                base = bref['href']
              end
            end
            action = (base + "/").sub(/\/\/$/, '/') + action
          end

          target = page.to_absolute(URI( action )) rescue next

          if not page.in_domain?(target)
            # Replace 127.0.0.1 and non-qualified hostnames with our page.host
            # ex: http://localhost/url OR http://www01/url
            target_uri = URI(target.to_s)
            if (target_uri.host.index(".").nil? or target_uri.host == "127.0.0.1")
              target_uri.host = page.url.host
              target = target_uri
            else
              next
            end
          end
        end

        # skip this form if it matches exclusion criteria
        if !(target.to_s =~ get_link_filter)
          form = {}.merge!(form_template)
          form[:method] = (f['method'] || 'GET').upcase
          form[:query]  = target.query.to_s if form[:method] != "GET"
          form[:path]   = target.path
          form[:params] = []
          f.css('input', 'textarea').each do |inp|
            form[:params] << [inp['name'].to_s, inp['value'] || inp.content || '', { :type => inp['type'].to_s }]
          end

          f.css( 'select' ).each do |s|
            value = nil

            # iterate over each option to find the default value (if there is a selected one)
            s.children.each do |opt|
              ov = opt['value'] || opt.content
              value = ov if opt['selected']
            end

            # set the first one as the default value if we don't already have one
            value ||= s.children.first['value'] || s.children.first.content rescue ''

            form[:params] << [ s['name'].to_s, value.to_s, [ :type => 'select'] ]
          end

          forms << form
        end
      end
    end

    # Report each of the discovered forms
    forms.each do |form|
      next if not form[:method]
      print_status((" " * 24) + "FORM: #{form[:method]} #{form[:path]}")
      report_web_form(form)
      self.form_count += 1
    end

    @for_each_page_blocks.each { |p| p.call( page ) }
  end

  def form_from_url( website, url )
    url = URI( url.to_s ) if !url.is_a?( URI )

    begin
      # Scrub out the jsessionid appends
      url.path = url.path.sub(/;jsessionid=[a-zA-Z0-9]+/, '')
    rescue URI::Error
    end

    #
    # Continue processing forms
    #
    forms = []
    form_template = { :web_site => website }
    form  = {}.merge(form_template)

    # This page has a query parameter we can test with GET parameters
    # ex: /test.php?a=b&c=d
    if url.query and not url.query.empty?
      form[:method] = 'GET'
      form[:path]   = url.path
      vars = url.query.split('&').map{|x| x.split("=", 2) }
      form[:params] = vars
    end

    # This is a REST-ish application with numeric parameters
    # ex: /customers/343
    if not form[:path] and url.path.to_s =~ /(.*)\/(\d+)$/
      path_base = $1
      path_info = $2
      form[:method] = 'PATH'
      form[:path]   = path_base
      form[:params] = [['PATH', path_info]]
      form[:query]  = url.query.to_s
    end

    # This is an application that uses PATH_INFO for parameters:
    # ex:  /index.php/Main_Page/Article01
    if not form[:path] and url.path.to_s =~ /(.*\/[a-z0-9A-Z]{3,256}\.[a-z0-9A-Z]{2,8})(\/.*)/
      path_base = $1
      path_info = $2
      form[:method] = 'PATH'
      form[:path]   = path_base
      form[:params] = [['PATH', path_info]]
      form[:query]  = url.query.to_s
    end

    form[:method] ? form : nil
  end

  private
  def opt_patterns_to_regexps( patterns )
    magic_wildcard_replacement = Rex::Text.rand_text_alphanumeric( 10 )
    patterns.to_s.split( /[\r\n]+/).map do |p|
      Regexp.new '^' + Regexp.escape( p.gsub( '*', magic_wildcard_replacement ) ).
        gsub( magic_wildcard_replacement, '.*' ) + '$'
    end
  end


end