From 5b07d9fcef61215fd49a12d0d9517db9179c43c6 Mon Sep 17 00:00:00 2001
From: Mike Bland <michael.bland@gsa.gov>
Date: Sun, 10 May 2015 15:15:52 -0400
Subject: [PATCH] Provide a robots.txt that denies all crawlers

---
 README.md          |  1 +
 oauthproxy.go      | 11 +++++++++++
 oauthproxy_test.go | 16 ++++++++++++++++
 3 files changed, 28 insertions(+)

diff --git a/README.md b/README.md
index 8420d7c..a1a5ce4 100644
--- a/README.md
+++ b/README.md
@@ -149,6 +149,7 @@ The command line to run `google_auth_proxy` would look like this:
 
 Google Auth Proxy responds directly to the following endpoints. All other endpoints will be proxied upstream when authenticated.
 
+* /robots.txt - returns a 200 OK response that disallows all User-agents from all paths; see [robotstxt.org](http://www.robotstxt.org/) for more info
 * /ping - returns an 200 OK response
 * /oauth2/sign_in - the login page, which also doubles as a sign out page (it clears cookies)
 * /oauth2/start - a URL that will redirect to start the OAuth cycle
diff --git a/oauthproxy.go b/oauthproxy.go
index 33f4698..563633e 100644
--- a/oauthproxy.go
+++ b/oauthproxy.go
@@ -21,6 +21,7 @@ import (
 	"github.com/bitly/google_auth_proxy/providers"
 )
 
+const robotsPath = "/robots.txt"
 const pingPath = "/ping"
 const signInPath = "/oauth2/sign_in"
 const oauthStartPath = "/oauth2/start"
@@ -270,6 +271,11 @@ func (p *OauthProxy) SetCookie(rw http.ResponseWriter, req *http.Request, val st
 	http.SetCookie(rw, cookie)
 }
 
+func (p *OauthProxy) RobotsTxt(rw http.ResponseWriter) {
+	rw.WriteHeader(http.StatusOK)
+	fmt.Fprintf(rw, "User-agent: *\nDisallow: /")
+}
+
 func (p *OauthProxy) PingPage(rw http.ResponseWriter) {
 	rw.WriteHeader(http.StatusOK)
 	fmt.Fprintf(rw, "OK")
@@ -358,6 +364,11 @@ func (p *OauthProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	var email string
 	var access_token string
 
+	if req.URL.Path == robotsPath {
+		p.RobotsTxt(rw)
+		return
+	}
+
 	if req.URL.Path == pingPath {
 		p.PingPage(rw)
 		return
diff --git a/oauthproxy_test.go b/oauthproxy_test.go
index d3fe400..995c38e 100644
--- a/oauthproxy_test.go
+++ b/oauthproxy_test.go
@@ -67,6 +67,22 @@ func TestEncodedSlashes(t *testing.T) {
 	}
 }
 
+func TestRobotsTxt(t *testing.T) {
+	opts := NewOptions()
+	opts.Upstreams = append(opts.Upstreams, "unused")
+	opts.ClientID = "bazquux"
+	opts.ClientSecret = "foobar"
+	opts.CookieSecret = "xyzzyplugh"
+	opts.Validate()
+
+	proxy := NewOauthProxy(opts, func(string) bool { return true })
+	rw := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/robots.txt", nil)
+	proxy.ServeHTTP(rw, req)
+	assert.Equal(t, 200, rw.Code)
+	assert.Equal(t, "User-agent: *\nDisallow: /", rw.Body.String())
+}
+
 type TestProvider struct {
 	*providers.ProviderData
 	EmailAddress string