Provide a robots.txt that denies all crawlers
This commit is contained in:
		
							parent
							
								
									26170c56af
								
							
						
					
					
						commit
						5b07d9fcef
					
				|  | @ -149,6 +149,7 @@ The command line to run `google_auth_proxy` would look like this: | |||
| 
 | ||||
| Google Auth Proxy responds directly to the following endpoints. All other endpoints will be proxied upstream when authenticated. | ||||
| 
 | ||||
| * /robots.txt - returns a 200 OK response that disallows all User-agents from all paths; see [robotstxt.org](http://www.robotstxt.org/) for more info | ||||
| * /ping - returns an 200 OK response | ||||
| * /oauth2/sign_in - the login page, which also doubles as a sign out page (it clears cookies) | ||||
| * /oauth2/start - a URL that will redirect to start the OAuth cycle | ||||
|  |  | |||
|  | @ -21,6 +21,7 @@ import ( | |||
| 	"github.com/bitly/google_auth_proxy/providers" | ||||
| ) | ||||
| 
 | ||||
| const robotsPath = "/robots.txt" | ||||
| const pingPath = "/ping" | ||||
| const signInPath = "/oauth2/sign_in" | ||||
| const oauthStartPath = "/oauth2/start" | ||||
|  | @ -270,6 +271,11 @@ func (p *OauthProxy) SetCookie(rw http.ResponseWriter, req *http.Request, val st | |||
| 	http.SetCookie(rw, cookie) | ||||
| } | ||||
| 
 | ||||
| func (p *OauthProxy) RobotsTxt(rw http.ResponseWriter) { | ||||
| 	rw.WriteHeader(http.StatusOK) | ||||
| 	fmt.Fprintf(rw, "User-agent: *\nDisallow: /") | ||||
| } | ||||
| 
 | ||||
| func (p *OauthProxy) PingPage(rw http.ResponseWriter) { | ||||
| 	rw.WriteHeader(http.StatusOK) | ||||
| 	fmt.Fprintf(rw, "OK") | ||||
|  | @ -358,6 +364,11 @@ func (p *OauthProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { | |||
| 	var email string | ||||
| 	var access_token string | ||||
| 
 | ||||
| 	if req.URL.Path == robotsPath { | ||||
| 		p.RobotsTxt(rw) | ||||
| 		return | ||||
| 	} | ||||
| 
 | ||||
| 	if req.URL.Path == pingPath { | ||||
| 		p.PingPage(rw) | ||||
| 		return | ||||
|  |  | |||
|  | @ -67,6 +67,22 @@ func TestEncodedSlashes(t *testing.T) { | |||
| 	} | ||||
| } | ||||
| 
 | ||||
| func TestRobotsTxt(t *testing.T) { | ||||
| 	opts := NewOptions() | ||||
| 	opts.Upstreams = append(opts.Upstreams, "unused") | ||||
| 	opts.ClientID = "bazquux" | ||||
| 	opts.ClientSecret = "foobar" | ||||
| 	opts.CookieSecret = "xyzzyplugh" | ||||
| 	opts.Validate() | ||||
| 
 | ||||
| 	proxy := NewOauthProxy(opts, func(string) bool { return true }) | ||||
| 	rw := httptest.NewRecorder() | ||||
| 	req, _ := http.NewRequest("GET", "/robots.txt", nil) | ||||
| 	proxy.ServeHTTP(rw, req) | ||||
| 	assert.Equal(t, 200, rw.Code) | ||||
| 	assert.Equal(t, "User-agent: *\nDisallow: /", rw.Body.String()) | ||||
| } | ||||
| 
 | ||||
| type TestProvider struct { | ||||
| 	*providers.ProviderData | ||||
| 	EmailAddress string | ||||
|  |  | |||
		Loading…
	
		Reference in New Issue