From dc65ff800f3766d13e93665066872cd7fc585b73 Mon Sep 17 00:00:00 2001 From: Tanvir Alam Date: Mon, 20 Nov 2017 14:35:59 -0500 Subject: [PATCH 1/6] distribution: create sha256sum.txt file when creating binaries to allow validation of checksums. * update README.md to include instructions on how to verify prebuilt binaries for new releases. --- README.md | 5 +++++ dist.sh | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0e79061b..6acd3cc4 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,11 @@ to validate accounts by email, domain or group. ## Installation 1. Download [Prebuilt Binary](https://github.com/bitly/oauth2_proxy/releases) (current release is `v2.2`) or build with `$ go get github.com/bitly/oauth2_proxy` which will put the binary in `$GOROOT/bin` +Prebuilt binaries can be validated by extracting the file and verifying it against the `sha256sum.txt` checksum file provided for each release starting with version `v2.3`. +``` +sha256sum -c sha256sum.txt 2>&1 | grep OK +oauth2_proxy-2.3.linux-amd64: OK +``` 2. Select a Provider and Register an OAuth Application with a Provider 3. Configure OAuth2 Proxy using config file, command line options, or environment variables 4. Configure SSL or Deploy behind a SSL endpoint (example provided for Nginx) diff --git a/dist.sh b/dist.sh index 18c5d02e..90193df5 100755 --- a/dist.sh +++ b/dist.sh @@ -13,6 +13,7 @@ os=$(go env GOOS) arch=$(go env GOARCH) version=$(cat $DIR/version.go | grep "const VERSION" | awk '{print $NF}' | sed 's/"//g') goversion=$(go version | awk '{print $3}') +sha256sum=() echo "... running tests" ./test.sh @@ -25,10 +26,22 @@ for os in windows linux darwin; do fi BUILD=$(mktemp -d ${TMPDIR:-/tmp}/oauth2_proxy.XXXXXX) TARGET="oauth2_proxy-$version.$os-$arch.$goversion" + FILENAME="oauth2_proxy-$version.$os-$arch$EXT" GOOS=$os GOARCH=$arch CGO_ENABLED=0 \ - go build -ldflags="-s -w" -o $BUILD/$TARGET/oauth2_proxy$EXT || exit 1 - pushd $BUILD - tar czvf $TARGET.tar.gz $TARGET + go build -ldflags="-s -w" -o $BUILD/$TARGET/$FILENAME || exit 1 + pushd $BUILD/$TARGET + sha256sum+=("$(shasum -a 256 $FILENAME || exit 1)") + cd .. && tar czvf $TARGET.tar.gz $TARGET mv $TARGET.tar.gz $DIR/dist popd done + +checksum_file="sha256sum.txt" +cd $DIR/dist +if [ -f $checksum_file ]; then + rm $checksum_file +fi +touch $checksum_file +for checksum in "${sha256sum[@]}"; do + echo "$checksum" >> $checksum_file +done From 842a45b1dbe54f7fed22f96bbf9177d85b3ea8b0 Mon Sep 17 00:00:00 2001 From: Tanvir Alam Date: Mon, 4 Dec 2017 09:54:31 -0500 Subject: [PATCH 2/6] distribution: remove gpm references and update to use dep --- dist.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dist.sh b/dist.sh index 90193df5..a00318bb 100755 --- a/dist.sh +++ b/dist.sh @@ -5,9 +5,7 @@ set -e DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" echo "working dir $DIR" mkdir -p $DIR/dist -mkdir -p $DIR/.godeps -export GOPATH=$DIR/.godeps:$GOPATH -GOPATH=$DIR/.godeps gpm install +dep ensure || exit 1 os=$(go env GOOS) arch=$(go env GOARCH) From 9341dcbf79fe130bb06b8ba9cfbd3700080fac9e Mon Sep 17 00:00:00 2001 From: Paul Seiffert Date: Fri, 14 Jul 2017 13:08:34 +0200 Subject: [PATCH 3/6] Make request logging format configurable --- logging_handler.go | 78 +++++++++++++++++++++++++++++++--------------- main.go | 3 +- options.go | 38 +++++++++++----------- 3 files changed, 75 insertions(+), 44 deletions(-) diff --git a/logging_handler.go b/logging_handler.go index 17fca977..540b5409 100644 --- a/logging_handler.go +++ b/logging_handler.go @@ -9,9 +9,14 @@ import ( "net" "net/http" "net/url" + "text/template" "time" ) +const ( + defaultRequestLoggingFormat = "{{.Client}} - {{.Username}} [{{.Timestamp}}] {{.Host}} {{.RequestMethod}} {{.Upstream}} {{.RequestURI}} {{.Protocol}} {{.UserAgent}} {{.StatusCode}} {{.ResponseSize}} {{.RequestDuration}}" +) + // responseLogger is wrapper of http.ResponseWriter that keeps track of its HTTP status // code and body size type responseLogger struct { @@ -64,15 +69,38 @@ func (l *responseLogger) Size() int { return l.size } -// loggingHandler is the http.Handler implementation for LoggingHandlerTo and its friends -type loggingHandler struct { - writer io.Writer - handler http.Handler - enabled bool +// logMessageData is the container for all values that are available as variables in the request logging format. +// All values are pre-formatted strings so it is easy to use them in the format string. +type logMessageData struct { + Client, + Host, + Protocol, + RequestDuration, + RequestMethod, + RequestURI, + ResponseSize, + StatusCode, + Timestamp, + Upstream, + UserAgent, + Username string } -func LoggingHandler(out io.Writer, h http.Handler, v bool) http.Handler { - return loggingHandler{out, h, v} +// loggingHandler is the http.Handler implementation for LoggingHandlerTo and its friends +type loggingHandler struct { + writer io.Writer + handler http.Handler + enabled bool + logTemplate *template.Template +} + +func LoggingHandler(out io.Writer, h http.Handler, v bool, requestLoggingTpl string) http.Handler { + return loggingHandler{ + writer: out, + handler: h, + enabled: v, + logTemplate: template.Must(template.New("request-log").Parse(requestLoggingTpl)), + } } func (h loggingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { @@ -83,14 +111,13 @@ func (h loggingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { if !h.enabled { return } - logLine := buildLogLine(logger.authInfo, logger.upstream, req, url, t, logger.Status(), logger.Size()) - h.writer.Write(logLine) + h.writeLogLine(logger.authInfo, logger.upstream, req, url, t, logger.Status(), logger.Size()) } // Log entry for req similar to Apache Common Log Format. // ts is the timestamp with which the entry should be logged. // status, size are used to provide the response HTTP status and size. -func buildLogLine(username, upstream string, req *http.Request, url url.URL, ts time.Time, status int, size int) []byte { +func (h loggingHandler) writeLogLine(username, upstream string, req *http.Request, url url.URL, ts time.Time, status int, size int) { if username == "" { username = "-" } @@ -114,19 +141,20 @@ func buildLogLine(username, upstream string, req *http.Request, url url.URL, ts duration := float64(time.Now().Sub(ts)) / float64(time.Second) - logLine := fmt.Sprintf("%s - %s [%s] %s %s %s %q %s %q %d %d %0.3f\n", - client, - username, - ts.Format("02/Jan/2006:15:04:05 -0700"), - req.Host, - req.Method, - upstream, - url.RequestURI(), - req.Proto, - req.UserAgent(), - status, - size, - duration, - ) - return []byte(logLine) + h.logTemplate.Execute(h.writer, logMessageData{ + Client: client, + Host: req.Host, + Protocol: req.Proto, + RequestDuration: fmt.Sprintf("%0.3f", duration), + RequestMethod: req.Method, + RequestURI: fmt.Sprintf("%q", url.RequestURI()), + ResponseSize: fmt.Sprintf("%d", size), + StatusCode: fmt.Sprintf("%d", status), + Timestamp: ts.Format("02/Jan/2006:15:04:05 -0700"), + Upstream: upstream, + UserAgent: fmt.Sprintf("%q", req.UserAgent()), + Username: username, + }) + + h.writer.Write([]byte("\n")) } diff --git a/main.go b/main.go index ab0e4d35..114a2a83 100644 --- a/main.go +++ b/main.go @@ -67,6 +67,7 @@ func main() { flagSet.Bool("cookie-httponly", true, "set HttpOnly cookie flag") flagSet.Bool("request-logging", true, "Log requests to stdout") + flagSet.String("request-logging-format", defaultRequestLoggingFormat, "Template for log lines") flagSet.String("provider", "google", "OAuth provider") flagSet.String("login-url", "", "Authentication endpoint") @@ -124,7 +125,7 @@ func main() { } s := &Server{ - Handler: LoggingHandler(os.Stdout, oauthproxy, opts.RequestLogging), + Handler: LoggingHandler(os.Stdout, oauthproxy, opts.RequestLogging, opts.RequestLoggingFormat), Opts: opts, } s.ListenAndServe() diff --git a/options.go b/options.go index f1df9169..e8ec4ffa 100644 --- a/options.go +++ b/options.go @@ -71,7 +71,8 @@ type Options struct { Scope string `flag:"scope" cfg:"scope"` ApprovalPrompt string `flag:"approval-prompt" cfg:"approval_prompt"` - RequestLogging bool `flag:"request-logging" cfg:"request_logging"` + RequestLogging bool `flag:"request-logging" cfg:"request_logging"` + RequestLoggingFormat string `flag:"request-logging-format" cfg:"request_logging_format"` SignatureKey string `flag:"signature-key" cfg:"signature_key" env:"OAUTH2_PROXY_SIGNATURE_KEY"` @@ -90,23 +91,24 @@ type SignatureData struct { func NewOptions() *Options { return &Options{ - ProxyPrefix: "/oauth2", - HttpAddress: "127.0.0.1:4180", - HttpsAddress: ":443", - DisplayHtpasswdForm: true, - CookieName: "_oauth2_proxy", - CookieSecure: true, - CookieHttpOnly: true, - CookieExpire: time.Duration(168) * time.Hour, - CookieRefresh: time.Duration(0), - SetXAuthRequest: false, - SkipAuthPreflight: false, - PassBasicAuth: true, - PassUserHeaders: true, - PassAccessToken: false, - PassHostHeader: true, - ApprovalPrompt: "force", - RequestLogging: true, + ProxyPrefix: "/oauth2", + HttpAddress: "127.0.0.1:4180", + HttpsAddress: ":443", + DisplayHtpasswdForm: true, + CookieName: "_oauth2_proxy", + CookieSecure: true, + CookieHttpOnly: true, + CookieExpire: time.Duration(168) * time.Hour, + CookieRefresh: time.Duration(0), + SetXAuthRequest: false, + SkipAuthPreflight: false, + PassBasicAuth: true, + PassUserHeaders: true, + PassAccessToken: false, + PassHostHeader: true, + ApprovalPrompt: "force", + RequestLogging: true, + RequestLoggingFormat: defaultRequestLoggingFormat, } } From 69550cbb23b48bcd8ea4de8f388aa73185cb960a Mon Sep 17 00:00:00 2001 From: Paul Seiffert Date: Fri, 14 Jul 2017 13:14:18 +0200 Subject: [PATCH 4/6] Document request-logging-format option --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 85fd9201..11336694 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,7 @@ Usage of oauth2_proxy: -redeem-url string: Token redemption endpoint -redirect-url string: the OAuth Redirect URL. ie: "https://internalapp.yourcompany.com/oauth2/callback" -request-logging: Log requests to stdout (default true) + -request-logging-format: Template for request log lines (see "Logging Format" paragraph below) -resource string: The resource that is protected (Azure AD only) -scope string: OAuth scope specification -set-xauthrequest: set X-Auth-Request-User and X-Auth-Request-Email response headers (useful in Nginx auth_request mode) @@ -331,12 +332,21 @@ following: ## Logging Format -OAuth2 Proxy logs requests to stdout in a format similar to Apache Combined Log. +By default, OAuth2 Proxy logs requests to stdout in a format similar to Apache Combined Log. ``` - [19/Mar/2015:17:20:19 -0400] GET "/path/" HTTP/1.1 "" ``` +If you require a different format than that, you can configure it with the `-request-logging-format` flag. +The default format is configured as follows: + +``` +{{.Client}} - {{.Username}} [{{.Timestamp}}] {{.Host}} {{.RequestMethod}} {{.Upstream}} {{.RequestURI}} {{.Protocol}} {{.UserAgent}} {{.StatusCode}} {{.ResponseSize}} {{.RequestDuration}} +``` + +[See `logMessageData` in `logging_handler.go`](./logging_handler.go) for all available variables. + ## Adding a new Provider Follow the examples in the [`providers` package](providers/) to define a new From 1cefc96311f61e771e42c6765f91d0f5b5c14e14 Mon Sep 17 00:00:00 2001 From: Paul Seiffert Date: Fri, 14 Jul 2017 13:51:16 +0200 Subject: [PATCH 5/6] Test request logging --- logging_handler_test.go | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 logging_handler_test.go diff --git a/logging_handler_test.go b/logging_handler_test.go new file mode 100644 index 00000000..9717cd6e --- /dev/null +++ b/logging_handler_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestLoggingHandler_ServeHTTP(t *testing.T) { + ts := time.Now() + + tests := []struct { + Format, + ExpectedLogMessage string + }{ + {defaultRequestLoggingFormat, fmt.Sprintf("127.0.0.1 - - [%s] test-server GET - \"/foo/bar\" HTTP/1.1 \"\" 200 4 0.000\n", ts.Format("02/Jan/2006:15:04:05 -0700"))}, + {"{{.RequestMethod}}", "GET\n"}, + } + + for _, test := range tests { + buf := bytes.NewBuffer(nil) + handler := func(w http.ResponseWriter, req *http.Request) { + w.Write([]byte("test")) + } + + h := LoggingHandler(buf, http.HandlerFunc(handler), true, test.Format) + + r, _ := http.NewRequest("GET", "/foo/bar", nil) + r.RemoteAddr = "127.0.0.1" + r.Host = "test-server" + + h.ServeHTTP(httptest.NewRecorder(), r) + + actual := buf.String() + if actual != test.ExpectedLogMessage { + t.Errorf("Log message was\n%s\ninstead of expected \n%s", actual, test.ExpectedLogMessage) + } + } +} From 882fcf0a0108674002fdea2ba89c3d11598f8c2f Mon Sep 17 00:00:00 2001 From: Mark Maglana Date: Tue, 4 Jul 2017 14:36:21 -0700 Subject: [PATCH 6/6] providers: iterate across all pages from /user/orgs github endpoint. For some GHE instances where a user can have more than 100 organizations, traversing the other pages is important otherwise oauth2_proxy will consider the user unauthorized. This change traverses the list returned by the API to avoid that. Update github provider tests to include this case. --- providers/github.go | 70 ++++++++++++++++++++++++---------------- providers/github_test.go | 47 +++++++++++++++++++++------ 2 files changed, 80 insertions(+), 37 deletions(-) diff --git a/providers/github.go b/providers/github.go index f3af86fe..26526ce7 100644 --- a/providers/github.go +++ b/providers/github.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "path" + "strconv" "strings" ) @@ -61,36 +62,51 @@ func (p *GitHubProvider) hasOrg(accessToken string) (bool, error) { Login string `json:"login"` } - params := url.Values{ - "limit": {"100"}, + type orgsPage []struct { + Login string `json:"login"` } - endpoint := &url.URL{ - Scheme: p.ValidateURL.Scheme, - Host: p.ValidateURL.Host, - Path: path.Join(p.ValidateURL.Path, "/user/orgs"), - RawQuery: params.Encode(), - } - req, _ := http.NewRequest("GET", endpoint.String(), nil) - req.Header.Set("Accept", "application/vnd.github.v3+json") - req.Header.Set("Authorization", fmt.Sprintf("token %s", accessToken)) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return false, err - } + pn := 1 + for { + params := url.Values{ + "limit": {"200"}, + "page": {strconv.Itoa(pn)}, + } - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return false, err - } - if resp.StatusCode != 200 { - return false, fmt.Errorf( - "got %d from %q %s", resp.StatusCode, endpoint.String(), body) - } + endpoint := &url.URL{ + Scheme: p.ValidateURL.Scheme, + Host: p.ValidateURL.Host, + Path: path.Join(p.ValidateURL.Path, "/user/orgs"), + RawQuery: params.Encode(), + } + req, _ := http.NewRequest("GET", endpoint.String(), nil) + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("Authorization", fmt.Sprintf("token %s", accessToken)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, err + } - if err := json.Unmarshal(body, &orgs); err != nil { - return false, err + body, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return false, err + } + if resp.StatusCode != 200 { + return false, fmt.Errorf( + "got %d from %q %s", resp.StatusCode, endpoint.String(), body) + } + + var op orgsPage + if err := json.Unmarshal(body, &op); err != nil { + return false, err + } + if len(op) == 0 { + break + } + + orgs = append(orgs, op...) + pn += 1 } var presentOrgs []string @@ -118,7 +134,7 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) { } params := url.Values{ - "limit": {"100"}, + "limit": {"200"}, } endpoint := &url.URL{ diff --git a/providers/github_test.go b/providers/github_test.go index 8080525b..48101825 100644 --- a/providers/github_test.go +++ b/providers/github_test.go @@ -27,23 +27,32 @@ func testGitHubProvider(hostname string) *GitHubProvider { return p } -func testGitHubBackend(payload string) *httptest.Server { - pathToQueryMap := map[string]string{ - "/user": "", - "/user/emails": "", +func testGitHubBackend(payload []string) *httptest.Server { + pathToQueryMap := map[string][]string{ + "/user": []string{""}, + "/user/emails": []string{""}, + "/user/orgs": []string{"limit=200&page=1", "limit=200&page=2", "limit=200&page=3"}, } return httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { url := r.URL query, ok := pathToQueryMap[url.Path] + validQuery := false + index := 0 + for i, q := range query { + if q == url.RawQuery { + validQuery = true + index = i + } + } if !ok { w.WriteHeader(404) - } else if url.RawQuery != query { + } else if !validQuery { w.WriteHeader(404) } else { w.WriteHeader(200) - w.Write([]byte(payload)) + w.Write([]byte(payload[index])) } })) } @@ -89,7 +98,7 @@ func TestGitHubProviderOverrides(t *testing.T) { } func TestGitHubProviderGetEmailAddress(t *testing.T) { - b := testGitHubBackend(`[ {"email": "michael.bland@gsa.gov", "primary": true} ]`) + b := testGitHubBackend([]string{`[ {"email": "michael.bland@gsa.gov", "primary": true} ]`}) defer b.Close() bURL, _ := url.Parse(b.URL) @@ -101,10 +110,28 @@ func TestGitHubProviderGetEmailAddress(t *testing.T) { assert.Equal(t, "michael.bland@gsa.gov", email) } +func TestGitHubProviderGetEmailAddressWithOrg(t *testing.T) { + b := testGitHubBackend([]string{ + `[ {"email": "michael.bland@gsa.gov", "primary": true, "login":"testorg"} ]`, + `[ {"email": "michael.bland1@gsa.gov", "primary": true, "login":"testorg1"} ]`, + `[ ]`, + }) + defer b.Close() + + bURL, _ := url.Parse(b.URL) + p := testGitHubProvider(bURL.Host) + p.Org = "testorg1" + + session := &SessionState{AccessToken: "imaginary_access_token"} + email, err := p.GetEmailAddress(session) + assert.Equal(t, nil, err) + assert.Equal(t, "michael.bland@gsa.gov", email) +} + // Note that trying to trigger the "failed building request" case is not // practical, since the only way it can fail is if the URL fails to parse. func TestGitHubProviderGetEmailAddressFailedRequest(t *testing.T) { - b := testGitHubBackend("unused payload") + b := testGitHubBackend([]string{"unused payload"}) defer b.Close() bURL, _ := url.Parse(b.URL) @@ -120,7 +147,7 @@ func TestGitHubProviderGetEmailAddressFailedRequest(t *testing.T) { } func TestGitHubProviderGetEmailAddressEmailNotPresentInPayload(t *testing.T) { - b := testGitHubBackend("{\"foo\": \"bar\"}") + b := testGitHubBackend([]string{"{\"foo\": \"bar\"}"}) defer b.Close() bURL, _ := url.Parse(b.URL) @@ -133,7 +160,7 @@ func TestGitHubProviderGetEmailAddressEmailNotPresentInPayload(t *testing.T) { } func TestGitHubProviderGetUserName(t *testing.T) { - b := testGitHubBackend(`{"email": "michael.bland@gsa.gov", "login": "mbland"}`) + b := testGitHubBackend([]string{`{"email": "michael.bland@gsa.gov", "login": "mbland"}`}) defer b.Close() bURL, _ := url.Parse(b.URL)