Web Server robots.txt Information Disclosure

#
# (C) Tenable Network Security, Inc.
#

include("compat.inc");

if (description) {
  script_id(10302);
  script_version("1.42");
  script_set_attribute(attribute:"plugin_modification_date", value:"2026/03/10");

  script_name(english:"Web Server robots.txt Information Disclosure");
 
 script_set_attribute(attribute:"synopsis", value:
"The remote web server contains a 'robots.txt' file." );
 script_set_attribute(attribute:"description", value:
"The remote host contains a file named 'robots.txt' that is intended to
prevent web 'robots' from visiting certain directories in a website for
maintenance or indexing purposes.  A malicious user may also be able to
use the contents of this file to learn of sensitive documents or
directories on the affected site and either retrieve them directly or
target them for other attacks." );
 script_set_attribute(attribute:"see_also", value:"http://www.robotstxt.org/orig.html" );
 script_set_attribute(attribute:"solution", value:
"Review the contents of the site's robots.txt file, use Robots META tags
instead of entries in the robots.txt file, and/or adjust the web
server's access controls to limit access to sensitive material." );
 script_set_attribute(attribute:"risk_factor", value:"None" );
 script_set_attribute(attribute:"plugin_publication_date", value: "1999/10/12");
script_set_attribute(attribute:"plugin_type", value:"remote");
script_end_attributes();

 
  script_summary(english:"Checks for a web server's robots.txt");
 
  script_category(ACT_GATHER_INFO);
  script_family(english:"Web Servers");

  script_copyright(english:"This script is Copyright (C) 2005-2026 Tenable Network Security, Inc.");

  script_dependencies("http_version.nasl");
  script_require_ports("Services/www", 80);

  exit(0);
}

include("http.inc");

var embedded = get_kb_item("Settings/HTTP/test_embedded");
var port = get_http_port(default: 80, embedded: embedded);

# Try to retrieve the file.
var res = http::get_response(port:port, method:'GET', item:'/robots.txt');
if (isnull(res)) 
{
  audit(AUDIT_NOT_LISTEN, "robots.txt", port);
} 

# nb: <http://www.robotstxt.org/wc/norobots-rfc.html> describes
#     how the file should look.
var match = pregmatch(string:res[2], pattern:"[ \s]*(A|Disa)llow:", icase:TRUE);
if (!isnull(match)) 
{
  var report = 'Contents of robots.txt :\n\n' + res[2];
  security_report_v4(port:port, severity:SECURITY_NOTE, extra:report);
} 
else
{
  audit(AUDIT_LISTEN_NOT_VULN, "robots.txt", port);
}