public_git
/
helmfile
mirror of https://github.com/helmfile/helmfile.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: added in oci repository flag and added helm methods to pull and export charts (#1629)

This commit is contained in:
Chris Mellard 2021-01-28 13:02:00 +13:00 committed by GitHub
parent 33880dab77
commit 2a71640095
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 265 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -37,7 +37,7 @@ RUN set -x & \
chmod +x kubectl && \
mv kubectl /usr/local/bin/kubectl
RUN ["helm", "init", "--client-only"]
RUN ["helm", "init", "--client-only", "--stable-repo-url", "https://charts.helm.sh/stable"]
RUN helm plugin install https://github.com/databus23/helm-diff && \
helm plugin install https://github.com/futuresimple/helm-secrets && \
helm plugin install https://github.com/hypnoglow/helm-s3.git && \

32
README.md
View File

@ -54,13 +54,14 @@ repositories:
# helm-git powered repository: You can treat any Git repository as a charts repository
- name: polaris
url: git+https://github.com/reactiveops/polaris@deploy/helm?ref=master
# Advanced configuration: You can setup basic or tls auth
# Advanced configuration: You can setup basic or tls auth and optionally enable helm OCI integration
- name: roboll
url: http://roboll.io/charts
certFile: optional_client_cert
keyFile: optional_client_key
username: optional_username
password: optional_password
oci: true
# Advanced configuration: You can use a ca bundle to use an https repo
# with a self-signed certificate
- name: insecure
@ -1308,6 +1309,35 @@ repositories:
url: https://<MyRegistry>.azurecr.io/helm/v1/repo
```
## OCI Registries
In order to use OCI chart registries firstly they must be marked in the repository list as OCI enabled, e.g.
```yaml
repositories:
- name: myOCIRegistry
url: https://myregistry.azurecr.io
oci: true
```
Secondly the credentials for the OCI registry can either be specified within `helmfile.yaml` similar to
```yaml
repositories:
- name: myOCIRegistry
url: https://myregistry.azurecr.io
oci: true
username: spongebob
password: squarepants
```
or for CI scenarios these can be sourced from the environment with the format `<registryName>_USERNAME` and `<registryName_PASSWORD>`, e.g.
```shell
export MYOCIREGISTRY_USERNAME=spongebob
export MYOCIREGISTRY_PASSWORD=squarepants
```
## Attribution
We use:

4
go.sum
View File

@ -626,10 +626,6 @@ github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ=
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/variantdev/chartify v0.4.9 h1:06foIMnJj31q/l1JZ+54anDLwqtP8zAOv5qVEn2IQhM=
github.com/variantdev/chartify v0.4.9/go.mod h1:jqlUJIzcrIVSfg8FC4g+IoC5WB83TBl8rnVVEv6g8MQ=
github.com/variantdev/chartify v0.5.0 h1:I6T6oobjLfYmwZ4dUjRsO9AdGKPCMtfzt0CXR0ovl9k=
github.com/variantdev/chartify v0.5.0/go.mod h1:jqlUJIzcrIVSfg8FC4g+IoC5WB83TBl8rnVVEv6g8MQ=
github.com/variantdev/chartify v0.6.0 h1:QQ00a8Vtuhk6F9jeTZJEXV2g0zRXhYG43xovWZrc3ac=
github.com/variantdev/chartify v0.6.0/go.mod h1:qF4XzQlkfH/6k2jAi1hLas+lK4zSCa8kY+r5JdmLA68=
github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363 h1:KrfQBEUn+wEOQ/6UIfoqRDvn+Q/wZridQ7t0G1vQqKE=

15
pkg/app/app_test.go
View File

@ -2404,6 +2404,10 @@ type mockRunner struct {
err error
}
func (mock *mockRunner) ExecuteStdIn(cmd string, args []string, env map[string]string, stdin io.Reader) ([]byte, error) {
return []byte{}, nil
}
func (mock *mockRunner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) {
return []byte{}, nil
}
@ -2441,6 +2445,14 @@ func (helm *mockHelmExec) TemplateRelease(name, chart string, flags ...string) e
return nil
}
func (helm *mockHelmExec) ChartPull(chart string, flags ...string) error {
return nil
}
func (helm *mockHelmExec) ChartExport(chart string, path string, flags ...string) error {
return nil
}
func (helm *mockHelmExec) UpdateDeps(chart string) error {
return nil
}
@ -2462,6 +2474,9 @@ func (helm *mockHelmExec) AddRepo(name, repository, cafile, certfile, keyfile, u
func (helm *mockHelmExec) UpdateRepo() error {
return nil
}
func (helm *mockHelmExec) RegistryLogin(name string, username string, password string) error {
return nil
}
func (helm *mockHelmExec) SyncRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
return nil
}

13
pkg/app/mocks_test.go
View File

@ -22,7 +22,14 @@ func (helm *noCallHelmExec) TemplateRelease(name, chart string, flags ...string)
helm.doPanic()
return nil
}
func (helm *noCallHelmExec) ChartPull(chart string, flags ...string) error {
helm.doPanic()
return nil
}
func (helm *noCallHelmExec) ChartExport(chart string, path string, flags ...string) error {
helm.doPanic()
return nil
}
func (helm *noCallHelmExec) UpdateDeps(chart string) error {
helm.doPanic()
return nil
@ -49,6 +56,10 @@ func (helm *noCallHelmExec) UpdateRepo() error {
helm.doPanic()
return nil
}
func (helm *noCallHelmExec) RegistryLogin(name string, username string, password string) error {
helm.doPanic()
return nil
}
func (helm *noCallHelmExec) SyncRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
helm.doPanic()
return nil

5
pkg/event/bus_test.go
View File

@ -2,6 +2,7 @@ package event
import (
"fmt"
"io"
"os"
"testing"
@ -16,6 +17,10 @@ var logger = helmexec.NewLogger(os.Stdout, "warn")
type runner struct {
}
func (r *runner) ExecuteStdIn(cmd string, args []string, env map[string]string, stdin io.Reader) ([]byte, error) {
return []byte(""), nil
}
func (r *runner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) {
if cmd == "ng" {
return nil, fmt.Errorf("cmd failed due to invalid cmd: %s", cmd)

10
pkg/exectest/helm.go
View File

@ -89,6 +89,9 @@ func (helm *Helm) AddRepo(name, repository, cafile, certfile, keyfile, username,
func (helm *Helm) UpdateRepo() error {
return nil
}
func (helm *Helm) RegistryLogin(name string, username string, password string) error {
return nil
}
func (helm *Helm) SyncRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
if strings.Contains(name, "error") {
return errors.New("error")
@ -158,7 +161,12 @@ func (helm *Helm) Lint(name, chart string, flags ...string) error {
func (helm *Helm) TemplateRelease(name, chart string, flags ...string) error {
return nil
}
func (helm *Helm) ChartPull(chart string, flags ...string) error {
return nil
}
func (helm *Helm) ChartExport(chart string, path string, flags ...string) error {
return nil
}
func (helm *Helm) IsHelm3() bool {
return false
}

63
pkg/helmexec/exec.go
View File

@ -1,6 +1,7 @@
package helmexec
import (
"bytes"
"fmt"
"io"
"io/ioutil"
@ -72,13 +73,13 @@ func parseHelmVersion(versionStr string) (semver.Version, error) {
func getHelmVersion(helmBinary string, runner Runner) (semver.Version, error) {
// Autodetect from `helm verison`
bytes, err := runner.Execute(helmBinary, []string{"version", "--client", "--short"}, nil)
// Autodetect from `helm version`
outBytes, err := runner.Execute(helmBinary, []string{"version", "--client", "--short"}, nil)
if err != nil {
return semver.Version{}, fmt.Errorf("error determining helm version: %w", err)
}
return parseHelmVersion(string(bytes))
return parseHelmVersion(string(outBytes))
}
// New for running helm commands
@ -157,6 +158,24 @@ func (helm *execer) UpdateRepo() error {
return err
}
func (helm *execer) RegistryLogin(repository string, username string, password string) error {
helm.logger.Info("Logging in to registry")
args := []string{
"registry",
"login",
repository,
"--username",
username,
"--password",
password,
}
buffer := bytes.Buffer{}
buffer.Write([]byte(fmt.Sprintf("%s\n", password)))
out, err := helm.execStdIn(args, map[string]string{"HELM_EXPERIMENTAL_OCI": "1"}, &buffer)
helm.info(out)
return err
}
func (helm *execer) BuildDeps(name, chart string) error {
helm.logger.Infof("Building dependency release=%v, chart=%v", name, chart)
out, err := helm.exec([]string{"dependency", "build", chart}, map[string]string{})
@ -369,6 +388,20 @@ func (helm *execer) Fetch(chart string, flags ...string) error {
return err
}
func (helm *execer) ChartPull(chart string, flags ...string) error {
helm.logger.Infof("Pulling %v", chart)
out, err := helm.exec(append([]string{"chart", "pull", chart}, flags...), map[string]string{"HELM_EXPERIMENTAL_OCI": "1"})
helm.info(out)
return err
}
func (helm *execer) ChartExport(chart string, path string, flags ...string) error {
helm.logger.Infof("Exporting %v", chart)
out, err := helm.exec(append([]string{"chart", "export", chart, "--destination", path}, flags...), map[string]string{"HELM_EXPERIMENTAL_OCI": "1"})
helm.info(out)
return err
}
func (helm *execer) DeleteRelease(context HelmContext, name string, flags ...string) error {
helm.logger.Infof("Deleting %v", name)
preArgs := context.GetTillerlessArgs(helm)
@ -398,17 +431,31 @@ func (helm *execer) exec(args []string, env map[string]string) ([]byte, error) {
}
cmd := fmt.Sprintf("exec: %s %s", helm.helmBinary, strings.Join(cmdargs, " "))
helm.logger.Debug(cmd)
bytes, err := helm.runner.Execute(helm.helmBinary, cmdargs, env)
return bytes, err
outBytes, err := helm.runner.Execute(helm.helmBinary, cmdargs, env)
return outBytes, err
}
func (helm *execer) execStdIn(args []string, env map[string]string, stdin io.Reader) ([]byte, error) {
cmdargs := args
if len(helm.extra) > 0 {
cmdargs = append(cmdargs, helm.extra...)
}
if helm.kubeContext != "" {
cmdargs = append([]string{"--kube-context", helm.kubeContext}, cmdargs...)
}
cmd := fmt.Sprintf("exec: %s %s", helm.helmBinary, strings.Join(cmdargs, " "))
helm.logger.Debug(cmd)
outBytes, err := helm.runner.ExecuteStdIn(helm.helmBinary, cmdargs, env, stdin)
return outBytes, err
}
func (helm *execer) azcli(name string) ([]byte, error) {
cmdargs := append(strings.Split("acr helm repo add --name", " "), name)
cmd := fmt.Sprintf("exec: az %s", strings.Join(cmdargs, " "))
helm.logger.Debug(cmd)
bytes, err := helm.runner.Execute("az", cmdargs, map[string]string{})
helm.logger.Debugf("%s: %s", cmd, bytes)
return bytes, err
outBytes, err := helm.runner.Execute("az", cmdargs, map[string]string{})
helm.logger.Debugf("%s: %s", cmd, outBytes)
return outBytes, err
}
func (helm *execer) info(out []byte) {

5
pkg/helmexec/exec_test.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"github.com/google/go-cmp/cmp"
"io"
"os"
"path"
"path/filepath"
@ -21,6 +22,10 @@ type mockRunner struct {
err error
}
func (mock *mockRunner) ExecuteStdIn(cmd string, args []string, env map[string]string, stdin io.Reader) ([]byte, error) {
return mock.output, mock.err
}
func (mock *mockRunner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) {
return mock.output, mock.err
}

3
pkg/helmexec/helmexec.go
View File

@ -14,12 +14,15 @@ type Interface interface {
AddRepo(name, repository, cafile, certfile, keyfile, username, password string, managed string) error
UpdateRepo() error
RegistryLogin(name string, username string, password string) error
BuildDeps(name, chart string) error
UpdateDeps(chart string) error
SyncRelease(context HelmContext, name, chart string, flags ...string) error
DiffRelease(context HelmContext, name, chart string, suppressDiff bool, flags ...string) error
TemplateRelease(name, chart string, flags ...string) error
Fetch(chart string, flags ...string) error
ChartPull(chart string, flags ...string) error
ChartExport(chart string, path string, flags ...string) error
Lint(name, chart string, flags ...string) error
ReleaseStatus(context HelmContext, name string, flags ...string) error
DeleteRelease(context HelmContext, name string, flags ...string) error

12
pkg/helmexec/runner.go
View File

@ -22,6 +22,7 @@ const (
// Runner interface for shell commands
type Runner interface {
Execute(cmd string, args []string, env map[string]string) ([]byte, error)
ExecuteStdIn(cmd string, args []string, env map[string]string, stdin io.Reader) ([]byte, error)
}
// ShellRunner implemention for shell commands
@ -41,6 +42,17 @@ func (shell ShellRunner) Execute(cmd string, args []string, env map[string]strin
})
}
// Execute a shell command
func (shell ShellRunner) ExecuteStdIn(cmd string, args []string, env map[string]string, stdin io.Reader) ([]byte, error) {
preparedCmd := exec.Command(cmd, args...)
preparedCmd.Dir = shell.Dir
preparedCmd.Env = mergeEnv(os.Environ(), env)
preparedCmd.Stdin = stdin
return Output(preparedCmd, &logWriterGenerator{
log: shell.Logger,
})
}
func Output(c *exec.Cmd, logWriterGenerators ...*logWriterGenerator) ([]byte, error) {
if c.Stdout != nil {
return nil, errors.New("exec: Stdout already set")

121
pkg/state/state.go
View File

@ -18,6 +18,7 @@ import (
"sync"
"text/template"
"github.com/hashicorp/go-getter/helper/url"
"github.com/imdario/mergo"
"github.com/variantdev/chartify"
@ -162,6 +163,7 @@ type RepositorySpec struct {
Username string `yaml:"username,omitempty"`
Password string `yaml:"password,omitempty"`
Managed string `yaml:"managed,omitempty"`
OCI bool `yaml:"oci,omitempty"`
}
// ReleaseSpec defines the structure of a helm release
@ -336,6 +338,7 @@ func (st *HelmState) ApplyOverrides(spec *ReleaseSpec) {
type RepoUpdater interface {
AddRepo(name, repository, cafile, certfile, keyfile, username, password string, managed string) error
UpdateRepo() error
RegistryLogin(name string, username string, password string) error
}
// getRepositoriesToSync returns the names of repositories to be updated
@ -375,8 +378,18 @@ func (st *HelmState) SyncRepos(helm RepoUpdater, shouldSkip map[string]bool) ([]
if shouldSkip[repo.Name] {
continue
}
var err error
if repo.OCI {
username, password := gatherOCIUsernamePassword(repo.Name, repo.Username, repo.Password)
if username == "" || password == "" {
return nil, fmt.Errorf("username and password are required fields for logging in to OCI registries with helm")
}
err = helm.RegistryLogin(repo.URL, username, password)
} else {
err = helm.AddRepo(repo.Name, repo.URL, repo.CaFile, repo.CertFile, repo.KeyFile, repo.Username, repo.Password, repo.Managed)
}
if err := helm.AddRepo(repo.Name, repo.URL, repo.CaFile, repo.CertFile, repo.KeyFile, repo.Username, repo.Password, repo.Managed); err != nil {
if err != nil {
return nil, err
}
@ -386,6 +399,24 @@ func (st *HelmState) SyncRepos(helm RepoUpdater, shouldSkip map[string]bool) ([]
return updated, nil
}
func gatherOCIUsernamePassword(repoName string, username string, password string) (string, string) {
var user, pass string
if username != "" {
user = username
} else if u := os.Getenv(fmt.Sprintf("%s_USERNAME", strings.ToUpper(repoName))); u != "" {
user = u
}
if password != "" {
pass = password
} else if p := os.Getenv(fmt.Sprintf("%s_PASSWORD", strings.ToUpper(repoName))); p != "" {
pass = p
}
return user, pass
}
type syncResult struct {
errors []*ReleaseError
}
@ -857,6 +888,20 @@ type chartPrepareResult struct {
chartFetchedByGoGetter bool
}
func (st *HelmState) GetRepositoryAndNameFromChartName(chartName string) (*RepositorySpec, string) {
chart := strings.Split(chartName, "/")
if len(chart) == 1 {
return nil, chartName
}
repo := chart[0]
for _, r := range st.Repositories {
if r.Name == repo {
return &r, strings.Join(chart[1:], "/")
}
}
return nil, chartName
}
// PrepareCharts creates temporary directories of charts.
//
// Each resulting "chart" can be one of the followings:
@ -935,16 +980,21 @@ func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurre
chartName := release.Chart
isLocal := st.directoryExistsAt(normalizeChart(st.basePath, chartName))
chartPath, err := st.goGetterChart(chartName, release.Directory, release.ForceGoGetter)
if err != nil {
results <- &chartPrepareResult{err: fmt.Errorf("release %q: %w", release.Name, err)}
return
}
chartFetchedByGoGetter := chartPath != chartName
isOCI, chartPath, err := st.getOCIChart(release, dir, helm)
if err != nil {
results <- &chartPrepareResult{err: fmt.Errorf("release %q: %w", release.Name, err)}
return
}
isLocal := st.directoryExistsAt(normalizeChart(st.basePath, chartName))
chartification, clean, err := st.PrepareChartify(helm, release, chartPath, workerIndex)
defer clean()
if err != nil {
@ -957,7 +1007,7 @@ func (st *HelmState) PrepareCharts(helm helmexec.Interface, dir string, concurre
skipDepsGlobal := opts.SkipDeps
skipDepsRelease := release.SkipDeps != nil && *release.SkipDeps
skipDepsDefault := release.SkipDeps == nil && st.HelmDefaults.SkipDeps
skipDeps := !isLocal || skipDepsGlobal || skipDepsRelease || skipDepsDefault
skipDeps := !isLocal || skipDepsGlobal || skipDepsRelease || skipDepsDefault || !isOCI
if chartification != nil {
c := chartify.New(
@ -2928,3 +2978,64 @@ func (st *HelmState) Reverse() {
st.Helmfiles[i], st.Helmfiles[j] = st.Helmfiles[j], st.Helmfiles[i]
}
}
func (st *HelmState) getOCIChart(release *ReleaseSpec, tempDir string, helm helmexec.Interface) (bool, string, error) {
isOCI := false
repo, name := st.GetRepositoryAndNameFromChartName(release.Chart)
if repo == nil {
return false, release.Chart, nil
}
if repo.OCI {
isOCI = true
}
if !isOCI {
return isOCI, release.Chart, nil
}
repoUrl, err := url.Parse(repo.URL)
if err != nil {
return isOCI, release.Chart, err
}
if repoUrl.Scheme == "" {
return isOCI, release.Chart, fmt.Errorf("unable to detect scheme - a valid url must be supplied for OCI registry %s", repo.URL)
}
chartVersion := "latest"
if release.Version != "" {
chartVersion = release.Version
}
qualifiedChartName := fmt.Sprintf("%s/%s:%s", repoUrl.Host, name, chartVersion)
err = helm.ChartPull(qualifiedChartName)
if err != nil {
return isOCI, release.Chart, err
}
pathElems := []string{
tempDir,
}
if release.Namespace != "" {
pathElems = append(pathElems, release.Namespace)
}
if release.KubeContext != "" {
pathElems = append(pathElems, release.KubeContext)
}
pathElems = append(pathElems, release.Name, name, chartVersion)
dir := filepath.Join(pathElems...)
err = helm.ChartExport(qualifiedChartName, dir)
if err != nil {
return isOCI, release.Chart, err
}
return isOCI, filepath.Join(dir, name), nil
}

2
test/integration/run.sh
View File

@ -49,7 +49,7 @@ info "Using namespace: ${test_ns}"
if helm version --client 2>/dev/null | grep '"v2\.'; then
helm_major_version=2
info "Using Helm version: $(helm version --short --client | grep -o v.*$)"
${helm} init --wait --override spec.template.spec.automountServiceAccountToken=true
${helm} init --stable-repo-url https://charts.helm.sh/stable --wait --override spec.template.spec.automountServiceAccountToken=true
# helm v3
else
helm_major_version=3