From 857a764cef27c16e0a40dbb55f4e5ef85f55cc19 Mon Sep 17 00:00:00 2001 From: Orkhan Huseynli Date: Tue, 4 Aug 2026 11:04:03 +0400 Subject: [PATCH 1/2] feat(oidc): lazy OIDC discovery so startup survives an unreachable issuer Signed-off-by: Orkhan Huseynli --- CHANGELOG.md | 2 + docs/docs/configuration/alpha_config.md | 1 + docs/docs/configuration/overview.md | 1 + main_test.go | 1 + oauthproxy.go | 125 +++++++++++++++-- oauthproxy_lazy_test.go | 91 +++++++++++++ pkg/apis/options/legacy_options.go | 3 + pkg/apis/options/legacy_options_test.go | 1 + pkg/apis/options/providers.go | 18 +++ pkg/apis/options/providers_test.go | 16 +++ providers/lazy_provider.go | 173 ++++++++++++++++++++++++ providers/lazy_provider_test.go | 119 ++++++++++++++++ providers/providers.go | 37 ++++- 13 files changed, 574 insertions(+), 14 deletions(-) create mode 100644 oauthproxy_lazy_test.go create mode 100644 pkg/apis/options/providers_test.go create mode 100644 providers/lazy_provider.go create mode 100644 providers/lazy_provider_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 788e82c2..c40e0a90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ## Changes since v7.15.3 +- [#3487](https://github.com/oauth2-proxy/oauth2-proxy/issues/3487) Add opt-in `--oidc-lazy-discovery` (default off): when the OIDC issuer is unreachable at startup, oauth2-proxy starts anyway and performs OIDC discovery in the background (retrying with backoff), so Basic Auth via `--htpasswd-file` and other provider-independent features keep working while discovery is pending. The readiness endpoint reports not-ready and OAuth2 login returns 503 until discovery succeeds. + # V7.15.3 ## Release Highlights diff --git a/docs/docs/configuration/alpha_config.md b/docs/docs/configuration/alpha_config.md index 680741ba..bdc3928d 100644 --- a/docs/docs/configuration/alpha_config.md +++ b/docs/docs/configuration/alpha_config.md @@ -544,6 +544,7 @@ character. | `insecureSkipIssuerVerification` | _bool_ | InsecureSkipIssuerVerification skips verification of ID token issuers. When false, ID Token Issuers must match the OIDC discovery URL
default set to 'false' | | `insecureSkipNonce` | _bool_ | InsecureSkipNonce skips verifying the ID Token's nonce claim that must match
the random nonce sent in the initial OAuth flow. Otherwise, the nonce is checked
after the initial OAuth redeem & subsequent token refreshes.
default set to 'true'
Warning: In a future release, this will change to 'false' by default for enhanced security. | | `skipDiscovery` | _bool_ | SkipDiscovery allows to skip OIDC discovery and use manually supplied Endpoints
default set to 'false' | +| `lazyDiscovery` | _bool_ | LazyDiscovery allows oauth2-proxy to start even when the OIDC issuer is
unreachable. When enabled, OIDC discovery is performed in the background
(retrying with backoff) instead of blocking startup, so features that do
not depend on the provider - such as Basic Auth via htpasswdFile - remain
available while discovery is pending. The readiness endpoint reports
not-ready until discovery succeeds. Only applies when discovery is enabled
(SkipDiscovery is false). Note that a configuration error (rather than an
unreachable issuer) is retried indefinitely instead of failing startup.
default set to 'false' | | `jwksURL` | _string_ | JwksURL is the OpenID Connect JWKS URL
eg: https://www.googleapis.com/oauth2/v3/certs | | `publicKeyFiles` | _[]string_ | PublicKeyFiles is a list of paths pointing to public key files in PEM format to use
for verifying JWT tokens | | `emailClaim` | _string_ | EmailClaim indicates which claim contains the user email,
default set to 'email' | diff --git a/docs/docs/configuration/overview.md b/docs/docs/configuration/overview.md index 965953fa..d5ef7dc0 100644 --- a/docs/docs/configuration/overview.md +++ b/docs/docs/configuration/overview.md @@ -98,6 +98,7 @@ Provider specific options can be found on their respective subpages. | flag: `--oidc-extra-audience`
toml: `oidc_extra_audiences` | string \| list | additional audiences which are allowed to pass verification | `"[]"` | | flag: `--oidc-groups-claim`
toml: `oidc_groups_claim` | string | which OIDC claim contains the user groups | `"groups"` | | flag: `--oidc-issuer-url`
toml: `oidc_issuer_url` | string | the OpenID Connect issuer URL, e.g. `"https://accounts.google.com"` | | +| flag: `--oidc-lazy-discovery`
toml: `oidc_lazy_discovery` | bool | start oauth2-proxy even if the OIDC issuer is unreachable and perform discovery in the background (retrying with backoff) instead of failing startup. Provider-independent features such as Basic Auth via `--htpasswd-file` keep working while discovery is pending; the readiness endpoint reports not-ready and OAuth2 login returns `503` until it succeeds. Only applies when OIDC discovery is enabled. **Note:** a configuration error (e.g. issuer mismatch) is retried indefinitely instead of failing startup, so the provider never becomes ready - watch the readiness endpoint and logs | false | | flag: `--oidc-jwks-url`
toml: `oidc_jwks_url` | string | OIDC JWKS URI for token verification; required if OIDC discovery is disabled and public key files are not provided | | | flag: `--oidc-public-key-file`
toml: `oidc_public_key_files` | string | Path to public key file in PEM format to use for verifying JWT tokens (may be given multiple times). Required if OIDC discovery is disabled na JWKS URL isn't provided | | | flag: `--oidc-enabled-signing-alg`
toml: `oidc_enabled_signing_algs` | string \| list | List of allowed JWT signing algorithms. When oidc discovery is enabled, the effective set is the intersection between this list and the provider's discovered supported algorithms. | | diff --git a/main_test.go b/main_test.go index 58b8ae7e..4edb16fa 100644 --- a/main_test.go +++ b/main_test.go @@ -187,6 +187,7 @@ redirect_url="http://localhost:4180/oauth2/callback" InsecureAllowUnverifiedEmail: ptr.To(false), InsecureSkipIssuerVerification: ptr.To(false), SkipDiscovery: ptr.To(false), + LazyDiscovery: ptr.To(false), EnabledSigningAlgs: []string{}, }, MicrosoftEntraIDConfig: options.MicrosoftEntraIDOptions{ diff --git a/oauthproxy.go b/oauthproxy.go index f8dc5471..eea1b63c 100644 --- a/oauthproxy.go +++ b/oauthproxy.go @@ -30,6 +30,7 @@ import ( "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/encryption" "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/proxyhttp" "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/util" + "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/util/ptr" "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/version" "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/ip" @@ -89,12 +90,16 @@ type OAuthProxy struct { SignInPath string - allowedRoutes []allowedRoute - apiRoutes []apiRoute - redirectURL *url.URL // the url to receive requests at - relativeRedirectURL bool - whitelistDomains []string - provider providers.Provider + allowedRoutes []allowedRoute + apiRoutes []apiRoute + redirectURL *url.URL // the url to receive requests at + relativeRedirectURL bool + whitelistDomains []string + provider providers.Provider + // lazyProvider is non-nil only when the provider is being initialised in the + // background (see --oidc-lazy-discovery). Its background discovery loop is + // started in Start so it can be tied to the proxy's shutdown context. + lazyProvider *providers.LazyProvider sessionStore sessionsapi.SessionStore ProxyPrefix string basicAuthValidator basic.Validator @@ -137,10 +142,13 @@ func NewOAuthProxy(opts *options.Options, validator func(string) bool) (*OAuthPr } } - provider, err := providers.NewProvider(opts.Providers[0]) + provider, err := setupProvider(opts.Providers[0]) if err != nil { return nil, fmt.Errorf("error initialising provider: %v", err) } + // A LazyProvider needs its background discovery loop started; it is launched + // in Start so it can be cancelled on shutdown (see p.lazyProvider usage). + lazyProvider, _ := provider.(*providers.LazyProvider) pageWriter, err := pagewriter.NewWriter(pagewriter.Opts{ TemplatesPath: opts.Templates.Path, @@ -205,7 +213,8 @@ func NewOAuthProxy(opts *options.Options, validator func(string) bool) (*OAuthPr return nil, err } - preAuthChain, err := buildPreAuthChain(opts, sessionStore, trustedProxies) + readinessVerifiable := readinessVerifiers{sessionStore, providerReadiness{provider: provider}} + preAuthChain, err := buildPreAuthChain(opts, readinessVerifiable, trustedProxies) if err != nil { return nil, fmt.Errorf("could not build pre-auth chain: %v", err) } @@ -229,6 +238,7 @@ func NewOAuthProxy(opts *options.Options, validator func(string) bool) (*OAuthPr ProxyPrefix: opts.ProxyPrefix, provider: provider, + lazyProvider: lazyProvider, sessionStore: sessionStore, redirectURL: redirectURL, relativeRedirectURL: opts.RelativeRedirectURL, @@ -280,6 +290,12 @@ func (p *OAuthProxy) Start() error { cancel() // cancel the context }() + // When lazy OIDC discovery is enabled, perform discovery in the background + // and tie its lifetime to the proxy's shutdown context so it stops cleanly. + if p.lazyProvider != nil { + go p.lazyProvider.InitWithRetry(ctx) + } + return p.server.Start(ctx) } @@ -358,7 +374,7 @@ func (p *OAuthProxy) buildProxySubrouter(s *mux.Router) { // buildPreAuthChain constructs a chain that should process every request before // the OAuth2 Proxy authentication logic kicks in. // For example forcing HTTPS or health checks. -func buildPreAuthChain(opts *options.Options, sessionStore sessionsapi.SessionStore, trustedProxies *ip.NetSet) (alice.Chain, error) { +func buildPreAuthChain(opts *options.Options, readiness middleware.Verifiable, trustedProxies *ip.NetSet) (alice.Chain, error) { chain := alice.New(middleware.NewScope(opts.ReverseProxy, opts.Logging.RequestIDHeader, trustedProxies)) if opts.ForceHTTPS { @@ -382,14 +398,14 @@ func buildPreAuthChain(opts *options.Options, sessionStore sessionsapi.SessionSt if opts.Logging.SilencePing { chain = chain.Append( middleware.NewHealthCheck(healthCheckPaths, healthCheckUserAgents), - middleware.NewReadynessCheck(opts.ReadyPath, sessionStore), + middleware.NewReadynessCheck(opts.ReadyPath, readiness), middleware.NewRequestLogger(), ) } else { chain = chain.Append( middleware.NewRequestLogger(), middleware.NewHealthCheck(healthCheckPaths, healthCheckUserAgents), - middleware.NewReadynessCheck(opts.ReadyPath, sessionStore), + middleware.NewReadynessCheck(opts.ReadyPath, readiness), ) } @@ -478,6 +494,84 @@ func buildProviderName(p providers.Provider, override string) string { return p.Data().ProviderName } +// setupProvider constructs the identity provider. When OIDC discovery fails and +// lazy discovery is enabled for an OIDC-based provider, it returns a +// providers.LazyProvider that serves OIDC flows as "not ready" while background +// discovery is performed (started by Start) - so Basic Auth and other +// provider-independent features keep working. Otherwise a discovery failure is +// returned to the caller and aborts startup, preserving the historical +// behaviour. +func setupProvider(providerConfig options.Provider) (providers.Provider, error) { + provider, err := providers.NewProvider(providerConfig) + if err == nil { + return provider, nil + } + + lazyEnabled := ptr.Deref(providerConfig.OIDCConfig.LazyDiscovery, options.DefaultOIDCLazyDiscovery) + skipDiscovery := ptr.Deref(providerConfig.OIDCConfig.SkipDiscovery, options.DefaultSkipDiscovery) + needsVerifier, verifierErr := providers.ProviderRequiresOIDCProviderVerifier(providerConfig.Type) + if verifierErr != nil { + return nil, verifierErr + } + + // Only defer to lazy initialisation when OIDC discovery is the failing step: + // the user must have opted in, discovery must be enabled, and the provider + // must actually use OIDC discovery. + if !lazyEnabled || skipDiscovery || !needsVerifier { + return nil, err + } + + lazy, lazyErr := providers.NewLazyProvider(providerConfig) + if lazyErr != nil { + // If even the discovery-independent placeholder cannot be built, the + // failure is a genuine configuration error rather than an unreachable + // issuer, so surface the original error and abort startup. + return nil, err + } + + logger.Errorf("OIDC discovery failed at startup; continuing with lazy discovery: %v", err) + logger.Printf("WARNING: with --oidc-lazy-discovery, discovery is retried indefinitely in the background. " + + "If the failure above is a configuration error (e.g. issuer mismatch or no common signing algorithms) " + + "rather than an unreachable issuer, the provider will never become ready - watch the readiness endpoint and logs.") + return lazy, nil +} + +// providerReadiness reports the readiness of a LazyProvider for the /ready deep +// health check. Non-lazy providers are always considered ready. +type providerReadiness struct { + provider providers.Provider +} + +func (p providerReadiness) VerifyConnection(_ context.Context) error { + if lazy, ok := p.provider.(*providers.LazyProvider); ok && !lazy.Ready() { + return errors.New("provider not ready: OIDC discovery is still pending") + } + return nil +} + +// providerReady reports whether the identity provider can serve OAuth2 flows. +// A LazyProvider is only ready once background OIDC discovery has completed; all +// other providers are always ready. +func (p *OAuthProxy) providerReady() bool { + if p.lazyProvider != nil { + return p.lazyProvider.Ready() + } + return true +} + +// readinessVerifiers combines multiple Verifiable checks into one; VerifyConnection +// fails if any of them fail. +type readinessVerifiers []middleware.Verifiable + +func (rs readinessVerifiers) VerifyConnection(ctx context.Context) error { + for _, r := range rs { + if err := r.VerifyConnection(ctx); err != nil { + return err + } + } + return nil +} + // buildRoutesAllowlist builds an []allowedRoute list from either the legacy // SkipAuthRegex option (paths only support) or newer SkipAuthRoutes option // (method=path support) @@ -823,6 +917,15 @@ func (p *OAuthProxy) OAuthStart(rw http.ResponseWriter, req *http.Request) { } func (p *OAuthProxy) doOAuthStart(rw http.ResponseWriter, req *http.Request, overrides url.Values) { + // With lazy OIDC discovery the provider may not be ready yet. Returning an + // error here avoids a silent self-redirect loop (GetLoginURL would be empty) + // and keeps provider-independent auth (e.g. Basic Auth) usable meanwhile. + if !p.providerReady() { + logger.Errorf("cannot start OAuth2 login flow: identity provider is not ready (OIDC discovery is still pending)") + p.ErrorPage(rw, req, http.StatusServiceUnavailable, "The identity provider is not ready yet. Please try again shortly.") + return + } + extraParams := p.provider.Data().LoginURLParams(overrides) prepareNoCache(rw) diff --git a/oauthproxy_lazy_test.go b/oauthproxy_lazy_test.go new file mode 100644 index 00000000..a393d2e6 --- /dev/null +++ b/oauthproxy_lazy_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/apis/options" + "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/util/ptr" + "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/validation" + "github.com/oauth2-proxy/oauth2-proxy/v7/providers" + . "github.com/onsi/gomega" +) + +func unreachableOIDCProvider(lazy bool) options.Provider { + return options.Provider{ + ID: "test-provider", + Type: "oidc", + ClientID: "client-id", + OIDCConfig: options.OIDCOptions{ + IssuerURL: "http://127.0.0.1:1/realms/test", + SkipDiscovery: ptr.To(false), + LazyDiscovery: ptr.To(lazy), + }, + } +} + +func TestSetupProviderLazyFallback(t *testing.T) { + g := NewWithT(t) + + // setupProvider does not start the background loop (Start does), so there is + // no goroutine to cancel here. + provider, err := setupProvider(unreachableOIDCProvider(true)) + g.Expect(err).ToNot(HaveOccurred()) + + lazy, ok := provider.(*providers.LazyProvider) + g.Expect(ok).To(BeTrue(), "expected a LazyProvider when discovery fails and lazy discovery is enabled") + g.Expect(lazy.Ready()).To(BeFalse()) +} + +func TestSetupProviderWithoutLazyFailsFast(t *testing.T) { + g := NewWithT(t) + + _, err := setupProvider(unreachableOIDCProvider(false)) + g.Expect(err).To(HaveOccurred()) +} + +func TestProviderReadinessVerifier(t *testing.T) { + g := NewWithT(t) + + // A non-lazy provider (google needs no discovery) is always considered ready. + nonLazy, err := providers.NewProvider(options.Provider{ID: "g", Type: "google", ClientID: "client-id"}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(providerReadiness{provider: nonLazy}.VerifyConnection(context.Background())).To(Succeed()) + + // A lazy provider that has not completed discovery is not ready. + lazy, err := providers.NewLazyProvider(unreachableOIDCProvider(true)) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(providerReadiness{provider: lazy}.VerifyConnection(context.Background())).ToNot(Succeed()) +} + +// TestOAuthStartUnavailableWhenProviderNotReady ensures the OAuth2 login flow +// returns 503 (rather than a silent self-redirect loop) while lazy discovery is +// still pending. +func TestOAuthStartUnavailableWhenProviderNotReady(t *testing.T) { + g := NewWithT(t) + + opts := baseTestOptions() + opts.Providers[0].Type = "oidc" + opts.Providers[0].OIDCConfig = options.OIDCOptions{ + IssuerURL: "http://127.0.0.1:1/realms/test", + SkipDiscovery: ptr.To(false), + LazyDiscovery: ptr.To(true), + EmailClaim: options.OIDCEmailClaim, + AudienceClaims: []string{"aud"}, + } + g.Expect(validation.Validate(opts)).To(Succeed()) + + // Not calling Start, so background discovery never runs and the provider + // stays not-ready for the duration of the test. + proxy, err := NewOAuthProxy(opts, func(string) bool { return true }) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(proxy.providerReady()).To(BeFalse()) + + rw := httptest.NewRecorder() + req, _ := http.NewRequest(http.MethodGet, "/oauth2/start", nil) + proxy.ServeHTTP(rw, req) + + g.Expect(rw.Code).To(Equal(http.StatusServiceUnavailable)) +} diff --git a/pkg/apis/options/legacy_options.go b/pkg/apis/options/legacy_options.go index e53fd480..3e900df5 100644 --- a/pkg/apis/options/legacy_options.go +++ b/pkg/apis/options/legacy_options.go @@ -540,6 +540,7 @@ type LegacyProvider struct { InsecureOIDCSkipIssuerVerification bool `flag:"insecure-oidc-skip-issuer-verification" cfg:"insecure_oidc_skip_issuer_verification"` InsecureOIDCSkipNonce bool `flag:"insecure-oidc-skip-nonce" cfg:"insecure_oidc_skip_nonce"` SkipOIDCDiscovery bool `flag:"skip-oidc-discovery" cfg:"skip_oidc_discovery"` + OIDCLazyDiscovery bool `flag:"oidc-lazy-discovery" cfg:"oidc_lazy_discovery"` OIDCJwksURL string `flag:"oidc-jwks-url" cfg:"oidc_jwks_url"` OIDCEmailClaim string `flag:"oidc-email-claim" cfg:"oidc_email_claim"` OIDCGroupsClaim string `flag:"oidc-groups-claim" cfg:"oidc_groups_claim"` @@ -602,6 +603,7 @@ func legacyProviderFlagSet() *pflag.FlagSet { flagSet.Bool("insecure-oidc-skip-issuer-verification", false, "Do not verify if issuer matches OIDC discovery URL") flagSet.Bool("insecure-oidc-skip-nonce", true, "skip verifying the OIDC ID Token's nonce claim") flagSet.Bool("skip-oidc-discovery", false, "Skip OIDC discovery and use manually supplied Endpoints") + flagSet.Bool("oidc-lazy-discovery", false, "Start oauth2-proxy even if the OIDC issuer is unreachable and perform discovery in the background, retrying with backoff. Features that do not depend on the provider (e.g. Basic Auth via htpasswd-file) remain available while discovery is pending") flagSet.String("oidc-jwks-url", "", "OpenID Connect JWKS URL (ie: https://www.googleapis.com/oauth2/v3/certs)") flagSet.String("oidc-groups-claim", OIDCGroupsClaim, "which OIDC claim contains the user groups") flagSet.String("oidc-email-claim", OIDCEmailClaim, "which OIDC claim contains the user's email") @@ -723,6 +725,7 @@ func (l *LegacyProvider) convert() (Providers, error) { InsecureSkipIssuerVerification: &l.InsecureOIDCSkipIssuerVerification, InsecureSkipNonce: &l.InsecureOIDCSkipNonce, SkipDiscovery: &l.SkipOIDCDiscovery, + LazyDiscovery: &l.OIDCLazyDiscovery, JwksURL: l.OIDCJwksURL, UserIDClaim: l.UserIDClaim, EmailClaim: l.OIDCEmailClaim, diff --git a/pkg/apis/options/legacy_options_test.go b/pkg/apis/options/legacy_options_test.go index f6cbfb7c..29457776 100644 --- a/pkg/apis/options/legacy_options_test.go +++ b/pkg/apis/options/legacy_options_test.go @@ -951,6 +951,7 @@ var _ = Describe("Legacy Options", func() { defaultOIDCOptions := OIDCOptions{ SkipDiscovery: ptr.To(false), + LazyDiscovery: ptr.To(false), InsecureSkipNonce: ptr.To(false), InsecureAllowUnverifiedEmail: ptr.To(false), InsecureSkipIssuerVerification: ptr.To(false), diff --git a/pkg/apis/options/providers.go b/pkg/apis/options/providers.go index 6f115f8a..ab2cb76f 100644 --- a/pkg/apis/options/providers.go +++ b/pkg/apis/options/providers.go @@ -13,6 +13,10 @@ const ( // for OIDCOptions.SkipDiscovery DefaultSkipDiscovery bool = false + // DefaultOIDCLazyDiscovery is the default value for OIDCOptions.LazyDiscovery. + // When false, a failed OIDC discovery aborts startup (historical behaviour). + DefaultOIDCLazyDiscovery bool = false + // DefaultInsecureSkipNonce is the default value // for OIDCOptions.InsecureSkipNonce DefaultInsecureSkipNonce bool = true @@ -300,6 +304,16 @@ type OIDCOptions struct { // SkipDiscovery allows to skip OIDC discovery and use manually supplied Endpoints // default set to 'false' SkipDiscovery *bool `yaml:"skipDiscovery,omitempty"` + // LazyDiscovery allows oauth2-proxy to start even when the OIDC issuer is + // unreachable. When enabled, OIDC discovery is performed in the background + // (retrying with backoff) instead of blocking startup, so features that do + // not depend on the provider - such as Basic Auth via htpasswdFile - remain + // available while discovery is pending. The readiness endpoint reports + // not-ready until discovery succeeds. Only applies when discovery is enabled + // (SkipDiscovery is false). Note that a configuration error (rather than an + // unreachable issuer) is retried indefinitely instead of failing startup. + // default set to 'false' + LazyDiscovery *bool `yaml:"lazyDiscovery,omitempty"` // JwksURL is the OpenID Connect JWKS URL // eg: https://www.googleapis.com/oauth2/v3/certs JwksURL string `yaml:"jwksURL,omitempty"` @@ -349,6 +363,7 @@ func providerDefaults() Providers { InsecureAllowUnverifiedEmail: ptr.To(DefaultInsecureAllowUnverifiedEmail), InsecureSkipNonce: ptr.To(DefaultInsecureSkipNonce), SkipDiscovery: ptr.To(DefaultSkipDiscovery), + LazyDiscovery: ptr.To(DefaultOIDCLazyDiscovery), UserIDClaim: OIDCEmailClaim, // Deprecated: Use OIDCEmailClaim EmailClaim: OIDCEmailClaim, GroupsClaim: OIDCGroupsClaim, @@ -394,6 +409,9 @@ func (o *OIDCOptions) EnsureDefaults() { if o.SkipDiscovery == nil { o.SkipDiscovery = ptr.To(DefaultSkipDiscovery) } + if o.LazyDiscovery == nil { + o.LazyDiscovery = ptr.To(DefaultOIDCLazyDiscovery) + } if o.UserIDClaim == "" { o.UserIDClaim = OIDCEmailClaim } diff --git a/pkg/apis/options/providers_test.go b/pkg/apis/options/providers_test.go new file mode 100644 index 00000000..494e50da --- /dev/null +++ b/pkg/apis/options/providers_test.go @@ -0,0 +1,16 @@ +package options + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("OIDCOptions EnsureDefaults", func() { + It("defaults LazyDiscovery to false when unset", func() { + o := &OIDCOptions{} + o.EnsureDefaults() + + Expect(o.LazyDiscovery).ToNot(BeNil()) + Expect(*o.LazyDiscovery).To(Equal(DefaultOIDCLazyDiscovery)) + }) +}) diff --git a/providers/lazy_provider.go b/providers/lazy_provider.go new file mode 100644 index 00000000..3a8c69be --- /dev/null +++ b/providers/lazy_provider.go @@ -0,0 +1,173 @@ +package providers + +import ( + "context" + "errors" + "net/url" + "sync" + "time" + + "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/apis/options" + "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/apis/sessions" + "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/logger" +) + +// ErrProviderNotReady is returned by a LazyProvider's methods while background +// OIDC discovery has not yet completed. Provider-independent features (such as +// Basic Auth via htpasswd-file) do not go through these methods and therefore +// keep working while discovery is pending. +var ErrProviderNotReady = errors.New("provider not ready: OIDC discovery has not completed yet") + +// Background discovery backoff bounds. +const ( + lazyDiscoveryInitialInterval = 3 * time.Second + lazyDiscoveryMaxInterval = 30 * time.Second +) + +// LazyProvider wraps a Provider whose construction depends on OIDC discovery. +// It starts out not-ready, delegating only Data() to a discovery-independent +// placeholder, and returns ErrProviderNotReady from OAuth flow methods. Once +// background discovery succeeds, the real provider is swapped in atomically and +// all methods delegate to it. +type LazyProvider struct { + providerConfig options.Provider + placeholder Provider + + mu sync.RWMutex + inner Provider +} + +var _ Provider = (*LazyProvider)(nil) + +// NewLazyProvider builds a LazyProvider for the given configuration. The +// placeholder provider is constructed without performing OIDC discovery so it +// cannot fail on an unreachable issuer. Call InitWithRetry (typically in a +// goroutine) to perform discovery in the background. +func NewLazyProvider(providerConfig options.Provider) (*LazyProvider, error) { + placeholder, err := newPlaceholderProvider(providerConfig) + if err != nil { + return nil, err + } + return &LazyProvider{ + providerConfig: providerConfig, + placeholder: placeholder, + }, nil +} + +// current returns the real provider if discovery has completed, otherwise nil. +func (l *LazyProvider) current() Provider { + l.mu.RLock() + defer l.mu.RUnlock() + return l.inner +} + +// Ready reports whether background discovery has completed and the real +// provider is available. +func (l *LazyProvider) Ready() bool { + return l.current() != nil +} + +func (l *LazyProvider) setInner(p Provider) { + l.mu.Lock() + l.inner = p + l.mu.Unlock() +} + +// InitWithRetry repeatedly attempts to construct the real provider (performing +// OIDC discovery) until it succeeds or ctx is cancelled. Once construction +// succeeds the real provider is swapped in and the LazyProvider becomes ready. +// It is intended to be run in a goroutine. +func (l *LazyProvider) InitWithRetry(ctx context.Context) { + interval := lazyDiscoveryInitialInterval + attempt := 0 + for { + attempt++ + provider, err := NewProvider(l.providerConfig) + if err == nil { + l.setInner(provider) + logger.Printf("OIDC discovery succeeded after %d attempt(s); provider is now ready", attempt) + return + } + logger.Errorf("lazy OIDC discovery attempt %d failed, will retry in %s: %v", attempt, interval, err) + + timer := time.NewTimer(interval) + select { + case <-ctx.Done(): + timer.Stop() + logger.Errorf("stopping lazy OIDC discovery: %v", ctx.Err()) + return + case <-timer.C: + } + + interval *= 2 + if interval > lazyDiscoveryMaxInterval { + interval = lazyDiscoveryMaxInterval + } + } +} + +// Data returns the real provider's data once ready, otherwise the placeholder's. +func (l *LazyProvider) Data() *ProviderData { + if p := l.current(); p != nil { + return p.Data() + } + return l.placeholder.Data() +} + +// GetLoginURL returns an empty string until the provider is ready. +func (l *LazyProvider) GetLoginURL(redirectURI, finalRedirect, nonce string, extraParams url.Values) string { + if p := l.current(); p != nil { + return p.GetLoginURL(redirectURI, finalRedirect, nonce, extraParams) + } + return "" +} + +func (l *LazyProvider) Redeem(ctx context.Context, redirectURI, code, codeVerifier string) (*sessions.SessionState, error) { + if p := l.current(); p != nil { + return p.Redeem(ctx, redirectURI, code, codeVerifier) + } + return nil, ErrProviderNotReady +} + +func (l *LazyProvider) GetEmailAddress(ctx context.Context, s *sessions.SessionState) (string, error) { + if p := l.current(); p != nil { + return p.GetEmailAddress(ctx, s) + } + return "", ErrProviderNotReady +} + +func (l *LazyProvider) EnrichSession(ctx context.Context, s *sessions.SessionState) error { + if p := l.current(); p != nil { + return p.EnrichSession(ctx, s) + } + return ErrProviderNotReady +} + +func (l *LazyProvider) Authorize(ctx context.Context, s *sessions.SessionState) (bool, error) { + if p := l.current(); p != nil { + return p.Authorize(ctx, s) + } + return false, ErrProviderNotReady +} + +// ValidateSession returns false until the provider is ready. +func (l *LazyProvider) ValidateSession(ctx context.Context, s *sessions.SessionState) bool { + if p := l.current(); p != nil { + return p.ValidateSession(ctx, s) + } + return false +} + +func (l *LazyProvider) RefreshSession(ctx context.Context, s *sessions.SessionState) (bool, error) { + if p := l.current(); p != nil { + return p.RefreshSession(ctx, s) + } + return false, ErrProviderNotReady +} + +func (l *LazyProvider) CreateSessionFromToken(ctx context.Context, token string) (*sessions.SessionState, error) { + if p := l.current(); p != nil { + return p.CreateSessionFromToken(ctx, token) + } + return nil, ErrProviderNotReady +} diff --git a/providers/lazy_provider_test.go b/providers/lazy_provider_test.go new file mode 100644 index 00000000..7aa00feb --- /dev/null +++ b/providers/lazy_provider_test.go @@ -0,0 +1,119 @@ +package providers + +import ( + "context" + "net/url" + "testing" + + "github.com/oauth2-proxy/mockoidc" + "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/apis/options" + "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/util/ptr" + . "github.com/onsi/gomega" +) + +// unreachableLazyConfig builds an OIDC provider config pointing at an +// unreachable issuer, with lazy discovery enabled. +func unreachableLazyConfig() options.Provider { + return options.Provider{ + ID: providerID, + Type: "oidc", + ClientID: clientID, + OIDCConfig: options.OIDCOptions{ + // Port 1 is not listenable, so discovery fails fast. + IssuerURL: "http://127.0.0.1:1/realms/test", + SkipDiscovery: ptr.To(false), + LazyDiscovery: ptr.To(true), + AudienceClaims: []string{"aud"}, + }, + } +} + +func TestLazyProviderNotReadyGating(t *testing.T) { + g := NewWithT(t) + + lazy, err := NewLazyProvider(unreachableLazyConfig()) + g.Expect(err).ToNot(HaveOccurred()) + + // Not ready until background discovery completes. + g.Expect(lazy.Ready()).To(BeFalse()) + + // Data() serves the discovery-independent placeholder. + g.Expect(lazy.Data()).ToNot(BeNil()) + g.Expect(lazy.Data().ProviderName).To(Equal("OpenID Connect")) + + // OAuth-flow methods report not-ready rather than panicking. + g.Expect(lazy.GetLoginURL("https://rd", "", "", url.Values{})).To(BeEmpty()) + g.Expect(lazy.ValidateSession(context.Background(), nil)).To(BeFalse()) + + _, err = lazy.Redeem(context.Background(), "https://rd", "code", "") + g.Expect(err).To(MatchError(ErrProviderNotReady)) + + _, err = lazy.GetEmailAddress(context.Background(), nil) + g.Expect(err).To(MatchError(ErrProviderNotReady)) + + err = lazy.EnrichSession(context.Background(), nil) + g.Expect(err).To(MatchError(ErrProviderNotReady)) + + _, err = lazy.Authorize(context.Background(), nil) + g.Expect(err).To(MatchError(ErrProviderNotReady)) + + _, err = lazy.RefreshSession(context.Background(), nil) + g.Expect(err).To(MatchError(ErrProviderNotReady)) + + _, err = lazy.CreateSessionFromToken(context.Background(), "token") + g.Expect(err).To(MatchError(ErrProviderNotReady)) +} + +func TestLazyProviderBecomesReady(t *testing.T) { + g := NewWithT(t) + + m, err := mockoidc.Run() + g.Expect(err).ToNot(HaveOccurred()) + defer func() { + g.Expect(m.Shutdown()).To(Succeed()) + }() + + providerConfig := options.Provider{ + ID: providerID, + Type: "oidc", + ClientID: m.Config().ClientID, + OIDCConfig: options.OIDCOptions{ + IssuerURL: m.Issuer(), + SkipDiscovery: ptr.To(false), + LazyDiscovery: ptr.To(true), + AudienceClaims: []string{"aud"}, + }, + } + + lazy, err := NewLazyProvider(providerConfig) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(lazy.Ready()).To(BeFalse()) + + // The issuer is reachable, so discovery succeeds on the first attempt and + // InitWithRetry returns promptly. + lazy.InitWithRetry(context.Background()) + + g.Expect(lazy.Ready()).To(BeTrue()) + // Once ready, delegation to the real provider works. + g.Expect(lazy.GetLoginURL("https://rd", "", "nonce", url.Values{})).ToNot(BeEmpty()) +} + +func TestLazyProviderInitWithRetryStopsOnContextCancel(t *testing.T) { + g := NewWithT(t) + + lazy, err := NewLazyProvider(unreachableLazyConfig()) + g.Expect(err).ToNot(HaveOccurred()) + + ctx, cancel := context.WithCancel(context.Background()) + + done := make(chan struct{}) + go func() { + lazy.InitWithRetry(ctx) + close(done) + }() + + cancel() + + g.Eventually(done, "2s").Should(BeClosed()) + g.Expect(lazy.Ready()).To(BeFalse()) +} diff --git a/providers/providers.go b/providers/providers.go index f87d26a2..2e8c3c39 100644 --- a/providers/providers.go +++ b/providers/providers.go @@ -37,6 +37,25 @@ func NewProvider(providerConfig options.Provider) (Provider, error) { if err != nil { return nil, fmt.Errorf("could not create provider data: %v", err) } + return providerFromData(providerConfig, providerData) +} + +// newPlaceholderProvider constructs a Provider without performing OIDC +// discovery. The resulting provider has the correct display name and defaults +// but no verifier or discovered endpoints, so it must not be used to serve OIDC +// flows. It is used as the initial value of a LazyProvider until background +// discovery completes. +func newPlaceholderProvider(providerConfig options.Provider) (Provider, error) { + providerData, err := buildProviderData(providerConfig, true) + if err != nil { + return nil, fmt.Errorf("could not create provider data: %v", err) + } + return providerFromData(providerConfig, providerData) +} + +// providerFromData constructs the concrete Provider implementation for the +// configured provider type from an already-built ProviderData. +func providerFromData(providerConfig options.Provider, providerData *ProviderData) (Provider, error) { switch providerConfig.Type { case options.ADFSProvider: return NewADFSProvider(providerData, providerConfig), nil @@ -78,6 +97,15 @@ func NewProvider(providerConfig options.Provider) (Provider, error) { } func newProviderDataFromConfig(providerConfig options.Provider) (*ProviderData, error) { + return buildProviderData(providerConfig, false) +} + +// buildProviderData builds the ProviderData for the given configuration. When +// skipDiscovery is true, the OIDC discovery step (which reaches out to the +// issuer over the network) is skipped, leaving the Verifier and discovered +// endpoints unset. This is used to build a placeholder provider for lazy +// initialisation, so oauth2-proxy can start before the issuer is reachable. +func buildProviderData(providerConfig options.Provider, skipDiscovery bool) (*ProviderData, error) { p := &ProviderData{ Scope: providerConfig.Scope, ClientID: providerConfig.ClientID, @@ -87,12 +115,12 @@ func newProviderDataFromConfig(providerConfig options.Provider) (*ProviderData, AdditionalClaims: providerConfig.AdditionalClaims, } - needsVerifier, err := providerRequiresOIDCProviderVerifier(providerConfig.Type) + needsVerifier, err := ProviderRequiresOIDCProviderVerifier(providerConfig.Type) if err != nil { return nil, err } - if needsVerifier { + if needsVerifier && !skipDiscovery { pv, err := internaloidc.NewProviderVerifier(context.TODO(), internaloidc.ProviderVerifierOptions{ AudienceClaims: providerConfig.OIDCConfig.AudienceClaims, ClientID: providerConfig.ClientID, @@ -187,7 +215,10 @@ func parseCodeChallengeMethod(providerConfig options.Provider) string { } } -func providerRequiresOIDCProviderVerifier(providerType options.ProviderType) (bool, error) { +// ProviderRequiresOIDCProviderVerifier reports whether the given provider type +// relies on the OIDC ProviderVerifier (and therefore on OIDC discovery when it +// is enabled). +func ProviderRequiresOIDCProviderVerifier(providerType options.ProviderType) (bool, error) { switch providerType { case options.BitbucketProvider, options.DigitalOceanProvider, options.FacebookProvider, options.GitHubProvider, options.GoogleProvider, options.KeycloakProvider, options.LinkedInProvider, options.LoginGovProvider, From c57b2c62824c6a42f02b233439a6ddb386cc9b57 Mon Sep 17 00:00:00 2001 From: Orkhan Huseynli Date: Wed, 12 Aug 2026 08:46:48 +0400 Subject: [PATCH 2/2] feat(oidc): keep /ready healthy under lazy discovery and make backoff configurable Signed-off-by: Orkhan Huseynli --- CHANGELOG.md | 2 +- docs/docs/configuration/alpha_config.md | 4 +- docs/docs/configuration/overview.md | 4 +- main_test.go | 2 + oauthproxy.go | 39 +++----------- oauthproxy_lazy_test.go | 58 +++++++++++---------- pkg/apis/options/legacy_options.go | 68 ++++++++++++++----------- pkg/apis/options/load_test.go | 3 ++ pkg/apis/options/providers.go | 42 +++++++++++---- providers/lazy_provider.go | 21 ++++---- 10 files changed, 132 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c40e0a90..724eb653 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ ## Changes since v7.15.3 -- [#3487](https://github.com/oauth2-proxy/oauth2-proxy/issues/3487) Add opt-in `--oidc-lazy-discovery` (default off): when the OIDC issuer is unreachable at startup, oauth2-proxy starts anyway and performs OIDC discovery in the background (retrying with backoff), so Basic Auth via `--htpasswd-file` and other provider-independent features keep working while discovery is pending. The readiness endpoint reports not-ready and OAuth2 login returns 503 until discovery succeeds. +- [#3487](https://github.com/oauth2-proxy/oauth2-proxy/issues/3487) Add opt-in `--oidc-lazy-discovery` (default off): when the OIDC issuer is unreachable at startup, oauth2-proxy starts anyway and performs OIDC discovery in the background (retrying with backoff), so Basic Auth via `--htpasswd-file` and other provider-independent features keep working while discovery is pending. The readiness endpoint stays healthy (keeping the pod in load-balancer rotation) while OAuth2 login returns 503 until discovery succeeds. The background backoff is configurable via `--oidc-lazy-discovery-initial-interval` (default `3s`) and `--oidc-lazy-discovery-max-interval` (default `30s`). # V7.15.3 diff --git a/docs/docs/configuration/alpha_config.md b/docs/docs/configuration/alpha_config.md index bdc3928d..4160f26a 100644 --- a/docs/docs/configuration/alpha_config.md +++ b/docs/docs/configuration/alpha_config.md @@ -544,7 +544,9 @@ character. | `insecureSkipIssuerVerification` | _bool_ | InsecureSkipIssuerVerification skips verification of ID token issuers. When false, ID Token Issuers must match the OIDC discovery URL
default set to 'false' | | `insecureSkipNonce` | _bool_ | InsecureSkipNonce skips verifying the ID Token's nonce claim that must match
the random nonce sent in the initial OAuth flow. Otherwise, the nonce is checked
after the initial OAuth redeem & subsequent token refreshes.
default set to 'true'
Warning: In a future release, this will change to 'false' by default for enhanced security. | | `skipDiscovery` | _bool_ | SkipDiscovery allows to skip OIDC discovery and use manually supplied Endpoints
default set to 'false' | -| `lazyDiscovery` | _bool_ | LazyDiscovery allows oauth2-proxy to start even when the OIDC issuer is
unreachable. When enabled, OIDC discovery is performed in the background
(retrying with backoff) instead of blocking startup, so features that do
not depend on the provider - such as Basic Auth via htpasswdFile - remain
available while discovery is pending. The readiness endpoint reports
not-ready until discovery succeeds. Only applies when discovery is enabled
(SkipDiscovery is false). Note that a configuration error (rather than an
unreachable issuer) is retried indefinitely instead of failing startup.
default set to 'false' | +| `lazyDiscovery` | _bool_ | LazyDiscovery lets oauth2-proxy start when the OIDC issuer is unreachable,
retrying discovery in the background instead of failing startup. OAuth2
login returns 503 until discovery succeeds; other features (e.g. Basic Auth)
keep working. default set to 'false' | +| `lazyDiscoveryInitialInterval` | _duration_ | LazyDiscoveryInitialInterval is the backoff before the first background
discovery retry, doubling up to LazyDiscoveryMaxInterval. default set to '3s' | +| `lazyDiscoveryMaxInterval` | _duration_ | LazyDiscoveryMaxInterval is the maximum backoff between background
discovery retries. default set to '30s' | | `jwksURL` | _string_ | JwksURL is the OpenID Connect JWKS URL
eg: https://www.googleapis.com/oauth2/v3/certs | | `publicKeyFiles` | _[]string_ | PublicKeyFiles is a list of paths pointing to public key files in PEM format to use
for verifying JWT tokens | | `emailClaim` | _string_ | EmailClaim indicates which claim contains the user email,
default set to 'email' | diff --git a/docs/docs/configuration/overview.md b/docs/docs/configuration/overview.md index d5ef7dc0..a6252f13 100644 --- a/docs/docs/configuration/overview.md +++ b/docs/docs/configuration/overview.md @@ -98,7 +98,9 @@ Provider specific options can be found on their respective subpages. | flag: `--oidc-extra-audience`
toml: `oidc_extra_audiences` | string \| list | additional audiences which are allowed to pass verification | `"[]"` | | flag: `--oidc-groups-claim`
toml: `oidc_groups_claim` | string | which OIDC claim contains the user groups | `"groups"` | | flag: `--oidc-issuer-url`
toml: `oidc_issuer_url` | string | the OpenID Connect issuer URL, e.g. `"https://accounts.google.com"` | | -| flag: `--oidc-lazy-discovery`
toml: `oidc_lazy_discovery` | bool | start oauth2-proxy even if the OIDC issuer is unreachable and perform discovery in the background (retrying with backoff) instead of failing startup. Provider-independent features such as Basic Auth via `--htpasswd-file` keep working while discovery is pending; the readiness endpoint reports not-ready and OAuth2 login returns `503` until it succeeds. Only applies when OIDC discovery is enabled. **Note:** a configuration error (e.g. issuer mismatch) is retried indefinitely instead of failing startup, so the provider never becomes ready - watch the readiness endpoint and logs | false | +| flag: `--oidc-lazy-discovery`
toml: `oidc_lazy_discovery` | bool | start oauth2-proxy even when the OIDC issuer is unreachable, retrying discovery in the background instead of failing startup. OAuth2 login returns `503` until discovery succeeds; other features (e.g. Basic Auth) keep working | false | +| flag: `--oidc-lazy-discovery-initial-interval`
toml: `oidc_lazy_discovery_initial_interval` | duration | backoff before the first background discovery retry, doubling up to `--oidc-lazy-discovery-max-interval` | `3s` | +| flag: `--oidc-lazy-discovery-max-interval`
toml: `oidc_lazy_discovery_max_interval` | duration | maximum backoff between background discovery retries | `30s` | | flag: `--oidc-jwks-url`
toml: `oidc_jwks_url` | string | OIDC JWKS URI for token verification; required if OIDC discovery is disabled and public key files are not provided | | | flag: `--oidc-public-key-file`
toml: `oidc_public_key_files` | string | Path to public key file in PEM format to use for verifying JWT tokens (may be given multiple times). Required if OIDC discovery is disabled na JWKS URL isn't provided | | | flag: `--oidc-enabled-signing-alg`
toml: `oidc_enabled_signing_algs` | string \| list | List of allowed JWT signing algorithms. When oidc discovery is enabled, the effective set is the intersection between this list and the provider's discovered supported algorithms. | | diff --git a/main_test.go b/main_test.go index 4edb16fa..8218437f 100644 --- a/main_test.go +++ b/main_test.go @@ -188,6 +188,8 @@ redirect_url="http://localhost:4180/oauth2/callback" InsecureSkipIssuerVerification: ptr.To(false), SkipDiscovery: ptr.To(false), LazyDiscovery: ptr.To(false), + LazyDiscoveryInitialInterval: options.DefaultOIDCLazyDiscoveryInitialInterval, + LazyDiscoveryMaxInterval: options.DefaultOIDCLazyDiscoveryMaxInterval, EnabledSigningAlgs: []string{}, }, MicrosoftEntraIDConfig: options.MicrosoftEntraIDOptions{ diff --git a/oauthproxy.go b/oauthproxy.go index eea1b63c..8558e2ba 100644 --- a/oauthproxy.go +++ b/oauthproxy.go @@ -213,8 +213,7 @@ func NewOAuthProxy(opts *options.Options, validator func(string) bool) (*OAuthPr return nil, err } - readinessVerifiable := readinessVerifiers{sessionStore, providerReadiness{provider: provider}} - preAuthChain, err := buildPreAuthChain(opts, readinessVerifiable, trustedProxies) + preAuthChain, err := buildPreAuthChain(opts, sessionStore, trustedProxies) if err != nil { return nil, fmt.Errorf("could not build pre-auth chain: %v", err) } @@ -374,7 +373,7 @@ func (p *OAuthProxy) buildProxySubrouter(s *mux.Router) { // buildPreAuthChain constructs a chain that should process every request before // the OAuth2 Proxy authentication logic kicks in. // For example forcing HTTPS or health checks. -func buildPreAuthChain(opts *options.Options, readiness middleware.Verifiable, trustedProxies *ip.NetSet) (alice.Chain, error) { +func buildPreAuthChain(opts *options.Options, sessionStore sessionsapi.SessionStore, trustedProxies *ip.NetSet) (alice.Chain, error) { chain := alice.New(middleware.NewScope(opts.ReverseProxy, opts.Logging.RequestIDHeader, trustedProxies)) if opts.ForceHTTPS { @@ -398,14 +397,14 @@ func buildPreAuthChain(opts *options.Options, readiness middleware.Verifiable, t if opts.Logging.SilencePing { chain = chain.Append( middleware.NewHealthCheck(healthCheckPaths, healthCheckUserAgents), - middleware.NewReadynessCheck(opts.ReadyPath, readiness), + middleware.NewReadynessCheck(opts.ReadyPath, sessionStore), middleware.NewRequestLogger(), ) } else { chain = chain.Append( middleware.NewRequestLogger(), middleware.NewHealthCheck(healthCheckPaths, healthCheckUserAgents), - middleware.NewReadynessCheck(opts.ReadyPath, readiness), + middleware.NewReadynessCheck(opts.ReadyPath, sessionStore), ) } @@ -536,22 +535,11 @@ func setupProvider(providerConfig options.Provider) (providers.Provider, error) return lazy, nil } -// providerReadiness reports the readiness of a LazyProvider for the /ready deep -// health check. Non-lazy providers are always considered ready. -type providerReadiness struct { - provider providers.Provider -} - -func (p providerReadiness) VerifyConnection(_ context.Context) error { - if lazy, ok := p.provider.(*providers.LazyProvider); ok && !lazy.Ready() { - return errors.New("provider not ready: OIDC discovery is still pending") - } - return nil -} - // providerReady reports whether the identity provider can serve OAuth2 flows. // A LazyProvider is only ready once background OIDC discovery has completed; all -// other providers are always ready. +// other providers are always ready. Note this does not gate the /ready endpoint +// (which stays healthy under lazy discovery so the pod remains in load-balancer +// rotation); it only gates the OAuth2 login flow. func (p *OAuthProxy) providerReady() bool { if p.lazyProvider != nil { return p.lazyProvider.Ready() @@ -559,19 +547,6 @@ func (p *OAuthProxy) providerReady() bool { return true } -// readinessVerifiers combines multiple Verifiable checks into one; VerifyConnection -// fails if any of them fail. -type readinessVerifiers []middleware.Verifiable - -func (rs readinessVerifiers) VerifyConnection(ctx context.Context) error { - for _, r := range rs { - if err := r.VerifyConnection(ctx); err != nil { - return err - } - } - return nil -} - // buildRoutesAllowlist builds an []allowedRoute list from either the legacy // SkipAuthRegex option (paths only support) or newer SkipAuthRoutes option // (method=path support) diff --git a/oauthproxy_lazy_test.go b/oauthproxy_lazy_test.go index a393d2e6..c83cb9ac 100644 --- a/oauthproxy_lazy_test.go +++ b/oauthproxy_lazy_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "net/http" "net/http/httptest" "testing" @@ -13,6 +12,27 @@ import ( . "github.com/onsi/gomega" ) +// newLazyPendingProxy builds a proxy whose OIDC issuer is unreachable with lazy +// discovery enabled. Start is not called, so background discovery never runs and +// the provider stays not-ready for the duration of the test. +func newLazyPendingProxy(g *WithT) *OAuthProxy { + opts := baseTestOptions() + opts.Providers[0].Type = "oidc" + opts.Providers[0].OIDCConfig = options.OIDCOptions{ + IssuerURL: "http://127.0.0.1:1/realms/test", + SkipDiscovery: ptr.To(false), + LazyDiscovery: ptr.To(true), + EmailClaim: options.OIDCEmailClaim, + AudienceClaims: []string{"aud"}, + } + g.Expect(validation.Validate(opts)).To(Succeed()) + + proxy, err := NewOAuthProxy(opts, func(string) bool { return true }) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(proxy.providerReady()).To(BeFalse()) + return proxy +} + func unreachableOIDCProvider(lazy bool) options.Provider { return options.Provider{ ID: "test-provider", @@ -46,18 +66,19 @@ func TestSetupProviderWithoutLazyFailsFast(t *testing.T) { g.Expect(err).To(HaveOccurred()) } -func TestProviderReadinessVerifier(t *testing.T) { +// TestReadyEndpointOKWhenProviderNotReady ensures /ready stays healthy under +// lazy discovery so the pod remains in load-balancer rotation while discovery +// is still pending. +func TestReadyEndpointOKWhenProviderNotReady(t *testing.T) { g := NewWithT(t) - // A non-lazy provider (google needs no discovery) is always considered ready. - nonLazy, err := providers.NewProvider(options.Provider{ID: "g", Type: "google", ClientID: "client-id"}) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(providerReadiness{provider: nonLazy}.VerifyConnection(context.Background())).To(Succeed()) + proxy := newLazyPendingProxy(g) - // A lazy provider that has not completed discovery is not ready. - lazy, err := providers.NewLazyProvider(unreachableOIDCProvider(true)) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(providerReadiness{provider: lazy}.VerifyConnection(context.Background())).ToNot(Succeed()) + rw := httptest.NewRecorder() + req, _ := http.NewRequest(http.MethodGet, "/ready", nil) + proxy.ServeHTTP(rw, req) + + g.Expect(rw.Code).To(Equal(http.StatusOK)) } // TestOAuthStartUnavailableWhenProviderNotReady ensures the OAuth2 login flow @@ -66,22 +87,7 @@ func TestProviderReadinessVerifier(t *testing.T) { func TestOAuthStartUnavailableWhenProviderNotReady(t *testing.T) { g := NewWithT(t) - opts := baseTestOptions() - opts.Providers[0].Type = "oidc" - opts.Providers[0].OIDCConfig = options.OIDCOptions{ - IssuerURL: "http://127.0.0.1:1/realms/test", - SkipDiscovery: ptr.To(false), - LazyDiscovery: ptr.To(true), - EmailClaim: options.OIDCEmailClaim, - AudienceClaims: []string{"aud"}, - } - g.Expect(validation.Validate(opts)).To(Succeed()) - - // Not calling Start, so background discovery never runs and the provider - // stays not-ready for the duration of the test. - proxy, err := NewOAuthProxy(opts, func(string) bool { return true }) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(proxy.providerReady()).To(BeFalse()) + proxy := newLazyPendingProxy(g) rw := httptest.NewRecorder() req, _ := http.NewRequest(http.MethodGet, "/oauth2/start", nil) diff --git a/pkg/apis/options/legacy_options.go b/pkg/apis/options/legacy_options.go index 3e900df5..452a3084 100644 --- a/pkg/apis/options/legacy_options.go +++ b/pkg/apis/options/legacy_options.go @@ -531,37 +531,39 @@ type LegacyProvider struct { // These options allow for other providers besides Google, with // potential overrides. - ProviderType string `flag:"provider" cfg:"provider"` - ProviderName string `flag:"provider-display-name" cfg:"provider_display_name"` - ProviderCAFiles []string `flag:"provider-ca-file" cfg:"provider_ca_files"` - UseSystemTrustStore bool `flag:"use-system-trust-store" cfg:"use_system_trust_store"` - OIDCIssuerURL string `flag:"oidc-issuer-url" cfg:"oidc_issuer_url"` - InsecureOIDCAllowUnverifiedEmail bool `flag:"insecure-oidc-allow-unverified-email" cfg:"insecure_oidc_allow_unverified_email"` - InsecureOIDCSkipIssuerVerification bool `flag:"insecure-oidc-skip-issuer-verification" cfg:"insecure_oidc_skip_issuer_verification"` - InsecureOIDCSkipNonce bool `flag:"insecure-oidc-skip-nonce" cfg:"insecure_oidc_skip_nonce"` - SkipOIDCDiscovery bool `flag:"skip-oidc-discovery" cfg:"skip_oidc_discovery"` - OIDCLazyDiscovery bool `flag:"oidc-lazy-discovery" cfg:"oidc_lazy_discovery"` - OIDCJwksURL string `flag:"oidc-jwks-url" cfg:"oidc_jwks_url"` - OIDCEmailClaim string `flag:"oidc-email-claim" cfg:"oidc_email_claim"` - OIDCGroupsClaim string `flag:"oidc-groups-claim" cfg:"oidc_groups_claim"` - OIDCAudienceClaims []string `flag:"oidc-audience-claim" cfg:"oidc_audience_claims"` - OIDCExtraAudiences []string `flag:"oidc-extra-audience" cfg:"oidc_extra_audiences"` - OIDCPublicKeyFiles []string `flag:"oidc-public-key-file" cfg:"oidc_public_key_files"` - OIDCEnabledSigningAlgs []string `flag:"oidc-enabled-signing-alg" cfg:"oidc_enabled_signing_algs"` - LoginURL string `flag:"login-url" cfg:"login_url"` - AuthRequestResponseMode string `flag:"auth-request-response-mode" cfg:"auth_request_response_mode"` - RedeemURL string `flag:"redeem-url" cfg:"redeem_url"` - ProfileURL string `flag:"profile-url" cfg:"profile_url"` - SkipClaimsFromProfileURL bool `flag:"skip-claims-from-profile-url" cfg:"skip_claims_from_profile_url"` - ProtectedResource string `flag:"resource" cfg:"resource"` - ValidateURL string `flag:"validate-url" cfg:"validate_url"` - Scope string `flag:"scope" cfg:"scope"` - Prompt string `flag:"prompt" cfg:"prompt"` - ApprovalPrompt string `flag:"approval-prompt" cfg:"approval_prompt"` // Deprecated by OIDC 1.0 - UserIDClaim string `flag:"user-id-claim" cfg:"user_id_claim"` - AllowedGroups []string `flag:"allowed-group" cfg:"allowed_groups"` - AllowedRoles []string `flag:"allowed-role" cfg:"allowed_roles"` - BackendLogoutURL string `flag:"backend-logout-url" cfg:"backend_logout_url"` + ProviderType string `flag:"provider" cfg:"provider"` + ProviderName string `flag:"provider-display-name" cfg:"provider_display_name"` + ProviderCAFiles []string `flag:"provider-ca-file" cfg:"provider_ca_files"` + UseSystemTrustStore bool `flag:"use-system-trust-store" cfg:"use_system_trust_store"` + OIDCIssuerURL string `flag:"oidc-issuer-url" cfg:"oidc_issuer_url"` + InsecureOIDCAllowUnverifiedEmail bool `flag:"insecure-oidc-allow-unverified-email" cfg:"insecure_oidc_allow_unverified_email"` + InsecureOIDCSkipIssuerVerification bool `flag:"insecure-oidc-skip-issuer-verification" cfg:"insecure_oidc_skip_issuer_verification"` + InsecureOIDCSkipNonce bool `flag:"insecure-oidc-skip-nonce" cfg:"insecure_oidc_skip_nonce"` + SkipOIDCDiscovery bool `flag:"skip-oidc-discovery" cfg:"skip_oidc_discovery"` + OIDCLazyDiscovery bool `flag:"oidc-lazy-discovery" cfg:"oidc_lazy_discovery"` + OIDCLazyDiscoveryInitialInterval time.Duration `flag:"oidc-lazy-discovery-initial-interval" cfg:"oidc_lazy_discovery_initial_interval"` + OIDCLazyDiscoveryMaxInterval time.Duration `flag:"oidc-lazy-discovery-max-interval" cfg:"oidc_lazy_discovery_max_interval"` + OIDCJwksURL string `flag:"oidc-jwks-url" cfg:"oidc_jwks_url"` + OIDCEmailClaim string `flag:"oidc-email-claim" cfg:"oidc_email_claim"` + OIDCGroupsClaim string `flag:"oidc-groups-claim" cfg:"oidc_groups_claim"` + OIDCAudienceClaims []string `flag:"oidc-audience-claim" cfg:"oidc_audience_claims"` + OIDCExtraAudiences []string `flag:"oidc-extra-audience" cfg:"oidc_extra_audiences"` + OIDCPublicKeyFiles []string `flag:"oidc-public-key-file" cfg:"oidc_public_key_files"` + OIDCEnabledSigningAlgs []string `flag:"oidc-enabled-signing-alg" cfg:"oidc_enabled_signing_algs"` + LoginURL string `flag:"login-url" cfg:"login_url"` + AuthRequestResponseMode string `flag:"auth-request-response-mode" cfg:"auth_request_response_mode"` + RedeemURL string `flag:"redeem-url" cfg:"redeem_url"` + ProfileURL string `flag:"profile-url" cfg:"profile_url"` + SkipClaimsFromProfileURL bool `flag:"skip-claims-from-profile-url" cfg:"skip_claims_from_profile_url"` + ProtectedResource string `flag:"resource" cfg:"resource"` + ValidateURL string `flag:"validate-url" cfg:"validate_url"` + Scope string `flag:"scope" cfg:"scope"` + Prompt string `flag:"prompt" cfg:"prompt"` + ApprovalPrompt string `flag:"approval-prompt" cfg:"approval_prompt"` // Deprecated by OIDC 1.0 + UserIDClaim string `flag:"user-id-claim" cfg:"user_id_claim"` + AllowedGroups []string `flag:"allowed-group" cfg:"allowed_groups"` + AllowedRoles []string `flag:"allowed-role" cfg:"allowed_roles"` + BackendLogoutURL string `flag:"backend-logout-url" cfg:"backend_logout_url"` AcrValues string `flag:"acr-values" cfg:"acr_values"` JWTKey string `flag:"jwt-key" cfg:"jwt_key"` @@ -604,6 +606,8 @@ func legacyProviderFlagSet() *pflag.FlagSet { flagSet.Bool("insecure-oidc-skip-nonce", true, "skip verifying the OIDC ID Token's nonce claim") flagSet.Bool("skip-oidc-discovery", false, "Skip OIDC discovery and use manually supplied Endpoints") flagSet.Bool("oidc-lazy-discovery", false, "Start oauth2-proxy even if the OIDC issuer is unreachable and perform discovery in the background, retrying with backoff. Features that do not depend on the provider (e.g. Basic Auth via htpasswd-file) remain available while discovery is pending") + flagSet.Duration("oidc-lazy-discovery-initial-interval", DefaultOIDCLazyDiscoveryInitialInterval, "Initial backoff before the first background OIDC discovery retry when oidc-lazy-discovery is enabled; doubles after each attempt up to oidc-lazy-discovery-max-interval") + flagSet.Duration("oidc-lazy-discovery-max-interval", DefaultOIDCLazyDiscoveryMaxInterval, "Maximum backoff between background OIDC discovery retries when oidc-lazy-discovery is enabled") flagSet.String("oidc-jwks-url", "", "OpenID Connect JWKS URL (ie: https://www.googleapis.com/oauth2/v3/certs)") flagSet.String("oidc-groups-claim", OIDCGroupsClaim, "which OIDC claim contains the user groups") flagSet.String("oidc-email-claim", OIDCEmailClaim, "which OIDC claim contains the user's email") @@ -726,6 +730,8 @@ func (l *LegacyProvider) convert() (Providers, error) { InsecureSkipNonce: &l.InsecureOIDCSkipNonce, SkipDiscovery: &l.SkipOIDCDiscovery, LazyDiscovery: &l.OIDCLazyDiscovery, + LazyDiscoveryInitialInterval: l.OIDCLazyDiscoveryInitialInterval, + LazyDiscoveryMaxInterval: l.OIDCLazyDiscoveryMaxInterval, JwksURL: l.OIDCJwksURL, UserIDClaim: l.UserIDClaim, EmailClaim: l.OIDCEmailClaim, diff --git a/pkg/apis/options/load_test.go b/pkg/apis/options/load_test.go index 40f9a725..4d581e2b 100644 --- a/pkg/apis/options/load_test.go +++ b/pkg/apis/options/load_test.go @@ -44,6 +44,9 @@ var _ = Describe("Load", func() { OIDCGroupsClaim: "groups", OIDCAudienceClaims: []string{"aud"}, InsecureOIDCSkipNonce: true, + + OIDCLazyDiscoveryInitialInterval: DefaultOIDCLazyDiscoveryInitialInterval, + OIDCLazyDiscoveryMaxInterval: DefaultOIDCLazyDiscoveryMaxInterval, }, Options: Options{ diff --git a/pkg/apis/options/providers.go b/pkg/apis/options/providers.go index ab2cb76f..d0ab0685 100644 --- a/pkg/apis/options/providers.go +++ b/pkg/apis/options/providers.go @@ -1,6 +1,10 @@ package options -import "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/util/ptr" +import ( + "time" + + "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/util/ptr" +) const ( // OIDCEmailClaim is the generic email claim used by the OIDC provider. @@ -17,6 +21,14 @@ const ( // When false, a failed OIDC discovery aborts startup (historical behaviour). DefaultOIDCLazyDiscovery bool = false + // DefaultOIDCLazyDiscoveryInitialInterval is the default value for + // OIDCOptions.LazyDiscoveryInitialInterval. + DefaultOIDCLazyDiscoveryInitialInterval time.Duration = 3 * time.Second + + // DefaultOIDCLazyDiscoveryMaxInterval is the default value for + // OIDCOptions.LazyDiscoveryMaxInterval. + DefaultOIDCLazyDiscoveryMaxInterval time.Duration = 30 * time.Second + // DefaultInsecureSkipNonce is the default value // for OIDCOptions.InsecureSkipNonce DefaultInsecureSkipNonce bool = true @@ -304,16 +316,17 @@ type OIDCOptions struct { // SkipDiscovery allows to skip OIDC discovery and use manually supplied Endpoints // default set to 'false' SkipDiscovery *bool `yaml:"skipDiscovery,omitempty"` - // LazyDiscovery allows oauth2-proxy to start even when the OIDC issuer is - // unreachable. When enabled, OIDC discovery is performed in the background - // (retrying with backoff) instead of blocking startup, so features that do - // not depend on the provider - such as Basic Auth via htpasswdFile - remain - // available while discovery is pending. The readiness endpoint reports - // not-ready until discovery succeeds. Only applies when discovery is enabled - // (SkipDiscovery is false). Note that a configuration error (rather than an - // unreachable issuer) is retried indefinitely instead of failing startup. - // default set to 'false' + // LazyDiscovery lets oauth2-proxy start when the OIDC issuer is unreachable, + // retrying discovery in the background instead of failing startup. OAuth2 + // login returns 503 until discovery succeeds; other features (e.g. Basic Auth) + // keep working. default set to 'false' LazyDiscovery *bool `yaml:"lazyDiscovery,omitempty"` + // LazyDiscoveryInitialInterval is the backoff before the first background + // discovery retry, doubling up to LazyDiscoveryMaxInterval. default set to '3s' + LazyDiscoveryInitialInterval time.Duration `yaml:"lazyDiscoveryInitialInterval,omitempty"` + // LazyDiscoveryMaxInterval is the maximum backoff between background + // discovery retries. default set to '30s' + LazyDiscoveryMaxInterval time.Duration `yaml:"lazyDiscoveryMaxInterval,omitempty"` // JwksURL is the OpenID Connect JWKS URL // eg: https://www.googleapis.com/oauth2/v3/certs JwksURL string `yaml:"jwksURL,omitempty"` @@ -412,6 +425,15 @@ func (o *OIDCOptions) EnsureDefaults() { if o.LazyDiscovery == nil { o.LazyDiscovery = ptr.To(DefaultOIDCLazyDiscovery) } + if o.LazyDiscoveryInitialInterval <= 0 { + o.LazyDiscoveryInitialInterval = DefaultOIDCLazyDiscoveryInitialInterval + } + if o.LazyDiscoveryMaxInterval <= 0 { + o.LazyDiscoveryMaxInterval = DefaultOIDCLazyDiscoveryMaxInterval + } + if o.LazyDiscoveryMaxInterval < o.LazyDiscoveryInitialInterval { + o.LazyDiscoveryMaxInterval = o.LazyDiscoveryInitialInterval + } if o.UserIDClaim == "" { o.UserIDClaim = OIDCEmailClaim } diff --git a/providers/lazy_provider.go b/providers/lazy_provider.go index 3a8c69be..4d20d92a 100644 --- a/providers/lazy_provider.go +++ b/providers/lazy_provider.go @@ -18,12 +18,6 @@ import ( // keep working while discovery is pending. var ErrProviderNotReady = errors.New("provider not ready: OIDC discovery has not completed yet") -// Background discovery backoff bounds. -const ( - lazyDiscoveryInitialInterval = 3 * time.Second - lazyDiscoveryMaxInterval = 30 * time.Second -) - // LazyProvider wraps a Provider whose construction depends on OIDC discovery. // It starts out not-ready, delegating only Data() to a discovery-independent // placeholder, and returns ErrProviderNotReady from OAuth flow methods. Once @@ -78,7 +72,16 @@ func (l *LazyProvider) setInner(p Provider) { // succeeds the real provider is swapped in and the LazyProvider becomes ready. // It is intended to be run in a goroutine. func (l *LazyProvider) InitWithRetry(ctx context.Context) { - interval := lazyDiscoveryInitialInterval + initialInterval := l.providerConfig.OIDCConfig.LazyDiscoveryInitialInterval + if initialInterval <= 0 { + initialInterval = options.DefaultOIDCLazyDiscoveryInitialInterval + } + maxInterval := l.providerConfig.OIDCConfig.LazyDiscoveryMaxInterval + if maxInterval < initialInterval { + maxInterval = initialInterval + } + + interval := initialInterval attempt := 0 for { attempt++ @@ -100,8 +103,8 @@ func (l *LazyProvider) InitWithRetry(ctx context.Context) { } interval *= 2 - if interval > lazyDiscoveryMaxInterval { - interval = lazyDiscoveryMaxInterval + if interval > maxInterval { + interval = maxInterval } } }