diff --git a/proxyconfig/proxyconfig.go b/proxyconfig/proxyconfig.go
index a2c93c0d..a706fa08 100644
--- a/proxyconfig/proxyconfig.go
+++ b/proxyconfig/proxyconfig.go
@@ -21,19 +21,23 @@ func (pc *ProxyConfig) Validate() error {
 	}
 
 	if pc.HTTP != nil {
-		_, err := url.Parse(pc.HTTP.URL)
+		_, err := url.ParseRequestURI(pc.HTTP.URL)
 		if err != nil {
 			return fmt.Errorf("proxy http set with invalid url: %v", err)
 		}
 	}
 	if pc.HTTPS != nil {
-		_, err := url.Parse(pc.HTTPS.URL)
+		_, err := url.ParseRequestURI(pc.HTTPS.URL)
 		if err != nil {
 			return fmt.Errorf("proxy https set with invalid url: %v", err)
 		}
 	}
 
-	// TODO: maybe validate noproxy?
+	for _, u := range pc.NoProxy {
+		if _, err := url.ParseRequestURI(u); err != nil {
+			return fmt.Errorf("proxy no_proxy set with invalid url: %v", err)
+		}
+	}
 	return nil
 }
 
diff --git a/vault/azurekeyvault/config.go b/vault/azurekeyvault/config.go
index 8cbfc90a..a7349b42 100644
--- a/vault/azurekeyvault/config.go
+++ b/vault/azurekeyvault/config.go
@@ -32,14 +32,18 @@ func (c *Config) Validate() error {
 	if c.ClientID == "" {
 		return errors.New("client_id is not set")
 	}
-	if _, err := url.Parse(c.URL); err != nil {
+	if _, err := url.ParseRequestURI(c.URL); err != nil {
 		return fmt.Errorf("failed to parse url: %v", err)
 	}
 
-	if c.CertPath != "" {
+	if c.CertPath == "" {
 		return errors.New("cert path must be provided")
 	}
 
+	if _, err := os.Stat(c.CertPath); err != nil {
+		return fmt.Errorf("cert path %q does not exist: %v", c.CertPath, err)
+	}
+
 	if err := c.Proxy.Validate(); err != nil {
 		return fmt.Errorf("proxy validation failed: %v", err)
 	}
diff --git a/vault/azurekeyvault/config_test.go b/vault/azurekeyvault/config_test.go
index bb7cd5db..b24835fb 100644
--- a/vault/azurekeyvault/config_test.go
+++ b/vault/azurekeyvault/config_test.go
@@ -2,6 +2,7 @@ package azurekeyvault
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/actions/actions-runner-controller/proxyconfig"
@@ -98,16 +99,6 @@ func TestValidate_valid(t *testing.T) {
 	clientID := "clientID"
 	url := "https://example.com"
 
-	cp, err := os.CreateTemp("", "")
-	require.NoError(t, err)
-	err = cp.Close()
-	require.NoError(t, err)
-	certPath := cp.Name()
-
-	t.Cleanup(func() {
-		os.Remove(certPath)
-	})
-
 	proxy := &proxyconfig.ProxyConfig{
 		HTTP: &proxyconfig.ProxyServerConfig{
 			URL:      "http://httpconfig.com",
@@ -124,15 +115,10 @@ func TestValidate_valid(t *testing.T) {
 		},
 	}
 
+	certPath, err := filepath.Abs("testdata/server.crt")
+	require.NoError(t, err)
+
 	tt := map[string]*Config{
-		"with jwt": {
-			TenantID:     tenantID,
-			ClientID:     clientID,
-			URL:          url,
-			CertPath:     "",
-			CertPassword: "",
-			Proxy:        proxy,
-		},
 		"with cert": {
 			TenantID:     tenantID,
 			ClientID:     clientID,
diff --git a/vault/azurekeyvault/testdata/server.crt b/vault/azurekeyvault/testdata/server.crt
new file mode 100644
index 00000000..60930054
--- /dev/null
+++ b/vault/azurekeyvault/testdata/server.crt
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDOjCCAiKgAwIBAgIUQr7R8yN5+2and6ucUOPF6oIbD48wDQYJKoZIhvcNAQEL
+BQAwFzEVMBMGA1UEAwwMVGVzdCBSb290IENBMB4XDTI1MDIyODEyMDEzMFoXDTI2
+MDcxMzEyMDEzMFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEA4oL2hAPQlDVaNJru5fIstkpoVSuam0vpswC7ciRc
+XQRjF3q8kjtIA7+jdySsKJqOLGnybDX3awvRyKMEjq11IfnZLjZc+FzTlA+x4z0h
+MHb0GiBFXKNzrExGI9F0KEPtFxcMIqZ119LY2ReexxWkZBQYlgTepaevp71za4c2
+n4Zy1+0iS5+uklZ4ANKMTBGlN76Qgt530VnpNiIeUbiUzY58Vx4q7kFcUv/oSz8p
+rbXr+/GGpAjrOc6/JsezRE8YK2po60dvV80TJ2Jt6pduvF7OSQnq/v4mJl1xuXKl
+Byo9HLbeu3BuVRWQs2/EwEzx5kX3Ugysl9Bm44K2yKe9/QIDAQABo4GAMH4wHwYD
+VR0jBBgwFoAUfd/q0BY4fkVBV3X+HWzXH0toW08wCQYDVR0TBAIwADALBgNVHQ8E
+BAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0RBAgwBocEfwAAATAdBgNV
+HQ4EFgQUe0rTTfWjho3hgeLTnajTCpddo2MwDQYJKoZIhvcNAQELBQADggEBAIR2
+5zkA7rPnddxCunsz8Jjq3wyhR/KiAFz+RGeFeiXDkF2fWr7QIQ9KbFbv8tpfXR7P
+B75bY0sXwutHMB2sZDi92cH5sthNBfp19fI35cxcU4oTPxp4UZJKEiA3Qx8y73CX
+NJu1009nPdOJNlIboDGAFdZ5SH6RCh+YcQZ68kjHPWBIpXxLbs9FN3QmpbAvtLh1
+PoPaSy7IjKmxm1u+Lf6tyIn2IiB3MiynaB3OKvbkLCseM/5SZKMk6WKSDWopOCJr
+xciPOc+yeLz5I2Omn0uViOIIciqjlgxncWAyNtDgvJcecwqB2cPiIhk6GY0QZ1uM
+e7KoqGzWXvWLqJ13a9U=
+-----END CERTIFICATE-----
diff --git a/vault/vault_test.go b/vault/vault_test.go
deleted file mode 100644
index 98d6c3fb..00000000
--- a/vault/vault_test.go
+++ /dev/null
@@ -1,34 +0,0 @@
-package vault_test
-
-import (
-	"os"
-	"testing"
-
-	"github.com/actions/actions-runner-controller/vault"
-	"github.com/actions/actions-runner-controller/vault/azurekeyvault"
-	"github.com/stretchr/testify/require"
-)
-
-func TestInitAll_AzureKeyVault(t *testing.T) {
-	os.Clearenv()
-	os.Setenv("LISTENER_AZURE_KEY_VAULT_TENANT_ID", "tenantID")
-	os.Setenv("LISTENER_AZURE_KEY_VAULT_CLIENT_ID", "clientID")
-	os.Setenv("LISTENER_AZURE_KEY_VAULT_URL", "https://example.com")
-	os.Setenv("LISTENER_AZURE_KEY_VAULT_CERT_PATH", "/path/to/cert")
-	os.Setenv("LISTENER_AZURE_KEY_VAULT_CERT_PASSWORD", "password")
-	os.Setenv("LISTENER_AZURE_KEY_VAULT_PROXY_HTTP_URL", "http://proxy.example.com")
-	os.Setenv("LISTENER_AZURE_KEY_VAULT_PROXY_HTTP_USERNAME", "username")
-	os.Setenv("LISTENER_AZURE_KEY_VAULT_PROXY_HTTP_PASSWORD", "password")
-	os.Setenv("LISTENER_AZURE_KEY_VAULT_PROXY_HTTPS_URL", "https://proxy.example.com")
-	os.Setenv("LISTENER_AZURE_KEY_VAULT_PROXY_HTTPS_USERNAME", "username")
-	os.Setenv("LISTENER_AZURE_KEY_VAULT_PROXY_HTTPS_PASSWORD", "password")
-	os.Setenv("LISTENER_AZURE_KEY_VAULT_PROXY_NO_PROXY", "temp.com")
-
-	vaults, err := vault.InitAll("LISTENER_")
-	require.NoError(t, err)
-	require.Len(t, vaults, 1)
-	require.Contains(t, vaults, vault.VaultTypeAzureKeyVault)
-	akv, ok := vaults[vault.VaultTypeAzureKeyVault].(*azurekeyvault.AzureKeyVault)
-	require.True(t, ok)
-	require.NotNil(t, akv)
-}