diff --git a/oauthproxy_test.go b/oauthproxy_test.go
index d31d62b6..7c72069a 100644
--- a/oauthproxy_test.go
+++ b/oauthproxy_test.go
@@ -1747,7 +1747,7 @@ func TestGetJwtSession(t *testing.T) {
 	verifier := oidc.NewVerifier("https://issuer.example.com", keyset,
 		&oidc.Config{ClientID: "https://test.myapp.com", SkipExpiryCheck: true,
 			SkipClientIDCheck: true})
-	verificationOptions := &internaloidc.IDTokenVerificationOptions{
+	verificationOptions := internaloidc.IDTokenVerificationOptions{
 		AudienceClaims: []string{"aud"},
 		ClientID:       "https://test.myapp.com",
 		ExtraAudiences: []string{},
diff --git a/pkg/apis/options/options.go b/pkg/apis/options/options.go
index 77492765..8641a430 100644
--- a/pkg/apis/options/options.go
+++ b/pkg/apis/options/options.go
@@ -68,26 +68,26 @@ type Options struct {
 	// internal values that are set after config validation
 	redirectURL        *url.URL
 	signatureData      *SignatureData
-	oidcVerifier       *internaloidc.IDTokenVerifier
-	jwtBearerVerifiers []*internaloidc.IDTokenVerifier
+	oidcVerifier       internaloidc.IDTokenVerifier
+	jwtBearerVerifiers []internaloidc.IDTokenVerifier
 	realClientIPParser ipapi.RealClientIPParser
 }
 
 // Options for Getting internal values
-func (o *Options) GetRedirectURL() *url.URL                       { return o.redirectURL }
-func (o *Options) GetSignatureData() *SignatureData               { return o.signatureData }
-func (o *Options) GetOIDCVerifier() *internaloidc.IDTokenVerifier { return o.oidcVerifier }
-func (o *Options) GetJWTBearerVerifiers() []*internaloidc.IDTokenVerifier {
+func (o *Options) GetRedirectURL() *url.URL                      { return o.redirectURL }
+func (o *Options) GetSignatureData() *SignatureData              { return o.signatureData }
+func (o *Options) GetOIDCVerifier() internaloidc.IDTokenVerifier { return o.oidcVerifier }
+func (o *Options) GetJWTBearerVerifiers() []internaloidc.IDTokenVerifier {
 	return o.jwtBearerVerifiers
 }
 func (o *Options) GetRealClientIPParser() ipapi.RealClientIPParser { return o.realClientIPParser }
 
 // Options for Setting internal values
-func (o *Options) SetRedirectURL(s *url.URL)                               { o.redirectURL = s }
-func (o *Options) SetSignatureData(s *SignatureData)                       { o.signatureData = s }
-func (o *Options) SetOIDCVerifier(s *internaloidc.IDTokenVerifier)         { o.oidcVerifier = s }
-func (o *Options) SetJWTBearerVerifiers(s []*internaloidc.IDTokenVerifier) { o.jwtBearerVerifiers = s }
-func (o *Options) SetRealClientIPParser(s ipapi.RealClientIPParser)        { o.realClientIPParser = s }
+func (o *Options) SetRedirectURL(s *url.URL)                              { o.redirectURL = s }
+func (o *Options) SetSignatureData(s *SignatureData)                      { o.signatureData = s }
+func (o *Options) SetOIDCVerifier(s internaloidc.IDTokenVerifier)         { o.oidcVerifier = s }
+func (o *Options) SetJWTBearerVerifiers(s []internaloidc.IDTokenVerifier) { o.jwtBearerVerifiers = s }
+func (o *Options) SetRealClientIPParser(s ipapi.RealClientIPParser)       { o.realClientIPParser = s }
 
 // NewOptions constructs a new Options with defaulted values
 func NewOptions() *Options {
diff --git a/pkg/validation/options.go b/pkg/validation/options.go
index 1d83a0aa..cd8f24f9 100644
--- a/pkg/validation/options.go
+++ b/pkg/validation/options.go
@@ -8,13 +8,11 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/coreos/go-oidc/v3/oidc"
 	"github.com/mbland/hmacauth"
 	"github.com/oauth2-proxy/oauth2-proxy/v7/pkg/apis/options"
 	"github.com/oauth2-proxy/oauth2-proxy/v7/pkg/ip"
 	"github.com/oauth2-proxy/oauth2-proxy/v7/pkg/logger"
 	internaloidc "github.com/oauth2-proxy/oauth2-proxy/v7/pkg/providers/oidc"
-	"github.com/oauth2-proxy/oauth2-proxy/v7/pkg/requests"
 	"github.com/oauth2-proxy/oauth2-proxy/v7/pkg/util"
 )
 
@@ -148,32 +146,27 @@ func parseJwtIssuers(issuers []string, msgs []string) ([]jwtIssuer, []string) {
 
 // newVerifierFromJwtIssuer takes in issuer information in jwtIssuer info and returns
 // a verifier for that issuer.
-func newVerifierFromJwtIssuer(audienceClaims []string, extraAudiences []string, jwtIssuer jwtIssuer) (*internaloidc.IDTokenVerifier, error) {
-	config := &oidc.Config{
-		ClientID:          jwtIssuer.audience,
-		SkipClientIDCheck: true, // client id check is done within oauth2-proxy: IDTokenVerifier.Verify
-	}
-	// Try as an OpenID Connect Provider first
-	var verifier *oidc.IDTokenVerifier
-	provider, err := oidc.NewProvider(context.Background(), jwtIssuer.issuerURI)
-	if err != nil {
-		// Try as JWKS URI
-		jwksURI := strings.TrimSuffix(jwtIssuer.issuerURI, "/") + "/.well-known/jwks.json"
-		if err := requests.New(jwksURI).Do().Error(); err != nil {
-			return nil, err
-		}
-
-		verifier = oidc.NewVerifier(jwtIssuer.issuerURI, oidc.NewRemoteKeySet(context.Background(), jwksURI), config)
-	} else {
-		verifier = provider.Verifier(config)
-	}
-	verificationOptions := &internaloidc.IDTokenVerificationOptions{
+func newVerifierFromJwtIssuer(audienceClaims []string, extraAudiences []string, jwtIssuer jwtIssuer) (internaloidc.IDTokenVerifier, error) {
+	pvOpts := internaloidc.ProviderVerifierOptions{
 		AudienceClaims: audienceClaims,
 		ClientID:       jwtIssuer.audience,
 		ExtraAudiences: extraAudiences,
-		// ExtraAudiences: o.Providers[0].OIDCConfig.ExtraAudiences,
+		IssuerURL:      jwtIssuer.issuerURI,
 	}
-	return internaloidc.NewVerifier(verifier, verificationOptions), nil
+
+	pv, err := internaloidc.NewProviderVerifier(context.TODO(), pvOpts)
+	if err != nil {
+		// If the discovery didn't work, try again without discovery
+		pvOpts.JWKsURL = strings.TrimSuffix(jwtIssuer.issuerURI, "/") + "/.well-known/jwks.json"
+		pvOpts.SkipDiscovery = true
+
+		pv, err = internaloidc.NewProviderVerifier(context.TODO(), pvOpts)
+		if err != nil {
+			return nil, fmt.Errorf("could not construct provider verifier for JWT Issuer: %v", err)
+		}
+	}
+
+	return pv.Verifier(), nil
 }
 
 // jwtIssuer hold parsed JWT issuer info that's used to construct a verifier.