Add option to dynamically serve a /robots.txt

Author: andrefs

README.md

@@ -62,6 +62,8 @@ This will install `http-server` globally so that it may be run from the command
 
 `-o` Open browser window after staring the server
 
+`-c` Set cache time (in seconds) for cache-control max-age header, e.g. -c10 for 10 seconds (defaults to '3600'). To disable caching, use -c-1.
+
 `-P` or `--proxy` Proxies all requests which can't be resolved locally to the given url. e.g.: -P http://someurl.com
 
 `-S` or `--ssl` Enable https.
@@ -70,7 +72,6 @@ This will install `http-server` globally so that it may be run from the command
 
 `-K` or `--key` Path to ssl key file (default: key.pem).
 
-`-h` or `--help` Print this list and exit.
+`-r` or `--robots` Provide a /robots.txt (whose content defaults to 'User-agent: *\nDisallow: /')
 
-`-c` Set cache time (in seconds) for cache-control max-age header, e.g. -c10 for 10 seconds
-(defaults to '3600'). To disable caching, use -c-1.
+`-h` or `--help` Print this list and exit.

bin/http-server

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-var colors = require('colors'),
+var colors     = require('colors'),
     httpServer = require('../lib/http-server'),
     portfinder = require('portfinder'),
     opener = require('opener'),
@@ -30,6 +30,7 @@ if (argv.h || argv.help) {
     "  -C --cert          Path to ssl cert file (default: cert.pem).",
     "  -K --key           Path to ssl key file (default: key.pem).",
     "",
+    "  -r --robots        Respond to /robots.txt [User-agent: *\\nDisallow: /]",
     "  -h --help          Print this list and exit."
   ].join('\n'));
   process.exit();
@@ -69,6 +70,7 @@ function listen(port) {
     cache: argv.c,
     showDir: argv.d,
     autoIndex: argv.i,
+    robots: argv.r || argv.robots,
     ext: argv.e || argv.ext,
     logFn: requestLogger,
     proxy: proxy

lib/http-server.js

@@ -50,6 +50,21 @@ var HTTPServer = exports.HTTPServer = function (options) {
     before.push(corser.create());
   }
 
+  if (options.robots) {
+    before.push(function (req, res) {
+      if (req.url === '/robots.txt') {
+        res.setHeader('Content-Type', 'text/plain');
+        var robots = options.robots === true
+          ? 'User-agent: *\nDisallow: /'
+          : options.robots.replace(/\\n/, '\n');
+
+        return res.end(robots);
+      }
+
+      res.emit('next');
+    });
+  }
+
   before.push(ecstatic({
     root: this.root,
     cache: this.cache,

test/http-server-test.js

@@ -12,11 +12,13 @@ vows.describe('http-server').addBatch({
     topic: function () {
       var server = httpServer.createServer({
         root: root,
+        robots: true,
         headers: {
           'Access-Control-Allow-Origin': '*',
           'Access-Control-Allow-Credentials': 'true'
         }
       });
+
       server.listen(8080);
       this.callback(null, server);
     },
@@ -57,6 +59,14 @@ vows.describe('http-server').addBatch({
         assert.include(body, '/canYouSeeMe');
       }
     },
+    'when robots options is activated': {
+      topic: function () {
+        request('http://127.0.0.1:8080/', this.callback);
+      },
+      'should respond with status code 200 to /robots.txt': function (res) {
+        assert.equal(res.statusCode, 200);
+      }
+    },
     'and options include custom set http-headers': {
       topic: function () {
         request('http://127.0.0.1:8080/', this.callback);