Provide `--cache-repo` as OCI image layout path (#2250)

* Adds the ability to provide `--cache-repo` as an OCI image layout path

- Adds cache.LayoutCache to implement cache.LayerCache interface
- When opts.CacheRepo has "oci:" prefix, instantiates a LayoutCache

Signed-off-by: Natalie Arellano <narellano@vmware.com>

* Add integration test for layout cache

Signed-off-by: Natalie Arellano <narellano@vmware.com>

* Updates from PR review

Signed-off-by: Natalie Arellano <narellano@vmware.com>

Signed-off-by: Natalie Arellano <narellano@vmware.com>

cmd/executor/cmd/root.go

@@ -219,7 +219,7 @@ func addKanikoOptionsFlags() {
 	RootCmd.PersistentFlags().StringVarP(&opts.Target, "target", "", "", "Set the target build stage to build")
 	RootCmd.PersistentFlags().BoolVarP(&opts.NoPush, "no-push", "", false, "Do not push the image to the registry")
 	RootCmd.PersistentFlags().BoolVarP(&opts.NoPushCache, "no-push-cache", "", false, "Do not push the cache layers to the registry")
-	RootCmd.PersistentFlags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided")
+	RootCmd.PersistentFlags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided; when prefixed with 'oci:' the repository will be written in OCI image layout format at the path provided")
 	RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "", "/cache", "Specify a local directory to use as a cache.")
 	RootCmd.PersistentFlags().StringVarP(&opts.DigestFile, "digest-file", "", "", "Specify a file to save the digest of the built image to.")
 	RootCmd.PersistentFlags().StringVarP(&opts.ImageNameDigestFile, "image-name-with-digest-file", "", "", "Specify a file to save the image name w/ digest of the built image to.")

integration/dockerfiles/Dockerfile_test_cache_copy_oci

@@ -0,0 +1,3 @@
+FROM google/cloud-sdk:256.0.0-alpine
+
+COPY context/foo /usr/bin

integration/dockerfiles/Dockerfile_test_cache_install_oci

@@ -0,0 +1,23 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test to make sure the cache works properly
+# /date should be the same regardless of when this image is built
+# if the cache is implemented correctly
+
+FROM debian:9.11
+WORKDIR /foo
+RUN apt-get update && apt-get install -y make
+COPY context/bar /context
+RUN echo "hey" > foo

integration/dockerfiles/Dockerfile_test_cache_oci

@@ -0,0 +1,22 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test to make sure the cache works properly
+# If the image is built twice, /date should be the same in both images
+# if the cache is implemented correctly
+
+FROM debian:9.11
+RUN date > /date
+COPY context/foo /foo
+RUN echo hey

integration/dockerfiles/Dockerfile_test_cache_perm_oci

@@ -0,0 +1,8 @@
+# Test to make sure the cache works with special file permissions properly.
+# If the image is built twice, directory foo should have the sticky bit,
+# and file bar should have the setuid and setgid bits.
+
+FROM busybox
+
+RUN mkdir foo && chmod +t foo
+RUN touch bar && chmod u+s,g+s bar

integration/images.go

@@ -189,6 +189,7 @@ type DockerFileBuilder struct {
 	filesBuilt              map[string]struct{}
 	DockerfilesToIgnore     map[string]struct{}
 	TestCacheDockerfiles    map[string]struct{}
+	TestOCICacheDockerfiles map[string]struct{}
 }
 
 type logger func(string, ...interface{})
@@ -216,6 +217,12 @@ func NewDockerFileBuilder() *DockerFileBuilder {
 		"Dockerfile_test_cache_perm":    {},
 		"Dockerfile_test_cache_copy":    {},
 	}
+	d.TestOCICacheDockerfiles = map[string]struct{}{
+		"Dockerfile_test_cache_oci":         {},
+		"Dockerfile_test_cache_install_oci": {},
+		"Dockerfile_test_cache_perm_oci":    {},
+		"Dockerfile_test_cache_copy_oci":    {},
+	}
 	return &d
 }
 
@@ -355,15 +362,14 @@ func populateVolumeCache() error {
 	return nil
 }
 
-// buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built
+// buildCachedImage builds the image for testing caching via kaniko where version is the nth time this image has been built
-func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cacheRepo, dockerfilesPath string, version int, args []string) error {
+func (d *DockerFileBuilder) buildCachedImage(config *integrationTestConfig, cacheRepo, dockerfilesPath, dockerfile string, version int, args []string) error {
 	imageRepo, serviceAccount := config.imageRepo, config.serviceAccount
 	_, ex, _, _ := runtime.Caller(0)
 	cwd := filepath.Dir(ex)
 
 	cacheFlag := "--cache=true"
 
-	for dockerfile := range d.TestCacheDockerfiles {
 	benchmarkEnv := "BENCHMARK_FILE=false"
 	if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err == nil && b {
 		os.Mkdir("benchmarks", 0755)
@@ -391,7 +397,6 @@ func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cac
 	if err != nil {
 		return fmt.Errorf("Failed to build cached image %s with kaniko command \"%s\": %w", kanikoImage, kanikoCmd.Args, err)
 	}
-	}
 	return nil
 }
 

integration/integration_test.go

@@ -559,22 +559,44 @@ func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder
 // Build each image with kaniko twice, and then make sure they're exactly the same
 func TestCache(t *testing.T) {
 	populateVolumeCache()
+
+	// Build dockerfiles with registry cache
 	for dockerfile := range imageBuilder.TestCacheDockerfiles {
-		args := []string{}
-		if dockerfile == "Dockerfile_test_cache_copy" {
-			args = append(args, "--cache-copy-layers=true")
-		}
 		t.Run("test_cache_"+dockerfile, func(t *testing.T) {
 			dockerfile := dockerfile
-			t.Parallel()
-
 			cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano()))
+			t.Parallel()
+			verifyBuildWith(t, cache, dockerfile)
+		})
+	}
+
+	// Build dockerfiles with layout cache
+	for dockerfile := range imageBuilder.TestOCICacheDockerfiles {
+		t.Run("test_oci_cache_"+dockerfile, func(t *testing.T) {
+			dockerfile := dockerfile
+			cache := filepath.Join("oci:", cacheDir, "cached", fmt.Sprintf("%v", time.Now().UnixNano()))
+			t.Parallel()
+			verifyBuildWith(t, cache, dockerfile)
+		})
+	}
+
+	if err := logBenchmarks("benchmark_cache"); err != nil {
+		t.Logf("Failed to create benchmark file: %v", err)
+	}
+}
+
+func verifyBuildWith(t *testing.T, cache, dockerfile string) {
+	args := []string{}
+	if strings.HasPrefix(dockerfile, "Dockerfile_test_cache_copy") {
+		args = append(args, "--cache-copy-layers=true")
+	}
+
 	// Build the initial image which will cache layers
-			if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 0, args); err != nil {
+	if err := imageBuilder.buildCachedImage(config, cache, dockerfilesPath, dockerfile, 0, args); err != nil {
 		t.Fatalf("error building cached image for the first time: %v", err)
 	}
 	// Build the second image which should pull from the cache
-			if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 1, args); err != nil {
+	if err := imageBuilder.buildCachedImage(config, cache, dockerfilesPath, dockerfile, 1, args); err != nil {
 		t.Fatalf("error building cached image for the second time: %v", err)
 	}
 	// Make sure both images are the same
@@ -585,12 +607,6 @@ func TestCache(t *testing.T) {
 
 	expected := fmt.Sprintf(emptyContainerDiff, kanikoVersion0, kanikoVersion1, kanikoVersion0, kanikoVersion1)
 	checkContainerDiffOutput(t, diff, expected)
-		})
-	}
-
-	if err := logBenchmarks("benchmark_cache"); err != nil {
-		t.Logf("Failed to create benchmark file: %v", err)
-	}
 }
 
 func TestRelativePaths(t *testing.T) {

pkg/cache/cache.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
@@ -28,6 +29,7 @@ import (
 	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/pkg/errors"
@@ -73,25 +75,82 @@ func (rc *RegistryCache) RetrieveLayer(ck string) (v1.Image, error) {
 		return nil, err
 	}
 
+	if err = verifyImage(img, rc.Opts.CacheTTL, cache); err != nil {
+		return nil, err
+	}
+	return img, nil
+}
+
+func verifyImage(img v1.Image, cacheTTL time.Duration, cache string) error {
 	cf, err := img.ConfigFile()
 	if err != nil {
-		return nil, errors.Wrap(err, fmt.Sprintf("retrieving config file for %s", cache))
+		return errors.Wrap(err, fmt.Sprintf("retrieving config file for %s", cache))
 	}
 
-	expiry := cf.Created.Add(rc.Opts.CacheTTL)
+	expiry := cf.Created.Add(cacheTTL)
 	// Layer is stale, rebuild it.
 	if expiry.Before(time.Now()) {
 		logrus.Infof("Cache entry expired: %s", cache)
-		return nil, fmt.Errorf("Cache entry expired: %s", cache)
+		return fmt.Errorf("Cache entry expired: %s", cache)
 	}
 
 	// Force the manifest to be populated
 	if _, err := img.RawManifest(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// LayoutCache is the OCI image layout cache
+type LayoutCache struct {
+	Opts *config.KanikoOptions
+}
+
+func (lc *LayoutCache) RetrieveLayer(ck string) (v1.Image, error) {
+	cache, err := Destination(lc.Opts, ck)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting cache destination")
+	}
+	logrus.Infof("Checking for cached layer %s...", cache)
+
+	var img v1.Image
+	if img, err = locateImage(strings.TrimPrefix(cache, "oci:")); err != nil {
+		return nil, errors.Wrap(err, "locating cache image")
+	}
+
+	if err = verifyImage(img, lc.Opts.CacheTTL, cache); err != nil {
 		return nil, err
 	}
 	return img, nil
 }
 
+func locateImage(path string) (v1.Image, error) {
+	var img v1.Image
+	layoutPath, err := layout.FromPath(path)
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("constructing layout path from %s", path))
+	}
+	index, err := layoutPath.ImageIndex()
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("retrieving index file for %s", layoutPath))
+	}
+	manifest, err := index.IndexManifest()
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("retrieving manifest file for %s", layoutPath))
+	}
+	for _, m := range manifest.Manifests {
+		// assume there is only one image
+		img, err = layoutPath.Image(m.Digest)
+		if err != nil {
+			return nil, errors.Wrap(err, fmt.Sprintf("initializing image with digest %s", m.Digest.String()))
+		}
+	}
+	if img == nil {
+		return nil, fmt.Errorf("path contains no images")
+	}
+	return img, nil
+}
+
 // Destination returns the repo where the layer should be stored
 // If no cache is specified, one is inferred from the destination provided
 func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) {

pkg/executor/build.go

@@ -127,9 +127,7 @@ func newStageBuilder(args *dockerfile.BuildArgs, opts *config.KanikoOptions, sta
 		crossStageDeps:   crossStageDeps,
 		digestToCacheKey: dcm,
 		stageIdxToDigest: sid,
-		layerCache: &cache.RegistryCache{
+		layerCache:       newLayerCache(opts),
-			Opts: opts,
-		},
 		pushLayerToCache: pushLayerToCache,
 	}
 
@@ -184,6 +182,21 @@ func initConfig(img partial.WithConfigFile, opts *config.KanikoOptions) (*v1.Con
 	return imageConfig, nil
 }
 
+func newLayerCache(opts *config.KanikoOptions) cache.LayerCache {
+	if isOCILayout(opts.CacheRepo) {
+		return &cache.LayoutCache{
+			Opts: opts,
+		}
+	}
+	return &cache.RegistryCache{
+		Opts: opts,
+	}
+}
+
+func isOCILayout(path string) bool {
+	return strings.HasPrefix(path, "oci:")
+}
+
 func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string, compositeKey CompositeCache, args *dockerfile.BuildArgs, env []string) (CompositeCache, error) {
 	// First replace all the environment variables or args in the command
 	replacementEnvs := args.ReplacementEnvs(env)

pkg/executor/build_test.go

@@ -28,6 +28,7 @@ import (
 	"strconv"
 	"testing"
 
+	"github.com/GoogleContainerTools/kaniko/pkg/cache"
 	"github.com/GoogleContainerTools/kaniko/pkg/commands"
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
@@ -522,6 +523,36 @@ func TestInitializeConfig(t *testing.T) {
 	}
 }
 
+func Test_newLayerCache_defaultCache(t *testing.T) {
+	t.Run("default layer cache is registry cache", func(t *testing.T) {
+		layerCache := newLayerCache(&config.KanikoOptions{CacheRepo: "some-cache-repo"})
+		foundCache, ok := layerCache.(*cache.RegistryCache)
+		if !ok {
+			t.Error("expected layer cache to be a registry cache")
+		}
+		if foundCache.Opts.CacheRepo != "some-cache-repo" {
+			t.Errorf(
+				"expected cache repo to be 'some-cache-repo'; got %q", foundCache.Opts.CacheRepo,
+			)
+		}
+	})
+}
+
+func Test_newLayerCache_layoutCache(t *testing.T) {
+	t.Run("when cache repo has 'oci:' prefix layer cache is layout cache", func(t *testing.T) {
+		layerCache := newLayerCache(&config.KanikoOptions{CacheRepo: "oci:/some-cache-repo"})
+		foundCache, ok := layerCache.(*cache.LayoutCache)
+		if !ok {
+			t.Error("expected layer cache to be a layout cache")
+		}
+		if foundCache.Opts.CacheRepo != "oci:/some-cache-repo" {
+			t.Errorf(
+				"expected cache repo to be 'oci:/some-cache-repo'; got %q", foundCache.Opts.CacheRepo,
+			)
+		}
+	})
+}
+
 func Test_stageBuilder_optimize(t *testing.T) {
 	testCases := []struct {
 		opts     *config.KanikoOptions

pkg/executor/push.go

@@ -84,8 +84,12 @@ func CheckPushPermissions(opts *config.KanikoOptions) error {
 	} else if opts.NoPush && !opts.NoPushCache {
 		// When no push is set, we want to check permissions for the cache repo
 		// instead of the destinations
+		if isOCILayout(opts.CacheRepo) {
+			targets = []string{} // no need to check push permissions if we're just writing to disk
+		} else {
 			targets = []string{opts.CacheRepo}
 		}
+	}
 
 	checked := map[string]bool{}
 	for _, destination := range targets {
@@ -289,8 +293,8 @@ func writeImageOutputs(image v1.Image, destRefs []name.Tag) error {
 	return nil
 }
 
-// pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache
+// pushLayerToCache pushes layer (tagged with cacheKey) to opts.CacheRepo
-// if opts.Cache doesn't exist, infer the cache from the given destination
+// if opts.CacheRepo doesn't exist, infer the cache from the given destination
 func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath string, createdBy string) error {
 	var layer v1.Layer
 	var err error
@@ -331,5 +335,9 @@ func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath strin
 	cacheOpts.Destinations = []string{cache}
 	cacheOpts.InsecureRegistries = opts.InsecureRegistries
 	cacheOpts.SkipTLSVerifyRegistries = opts.SkipTLSVerifyRegistries
+	if isOCILayout(cache) {
+		cacheOpts.OCILayoutPath = strings.TrimPrefix(cache, "oci:")
+		cacheOpts.NoPush = true
+	}
 	return DoPush(empty, &cacheOpts)
 }

pkg/executor/push_test.go

@@ -252,74 +252,99 @@ func TestImageNameTagDigestFile(t *testing.T) {
 	testutil.CheckErrorAndDeepEqual(t, false, err, want, got)
 }
 
-var calledExecCommand = []bool{}
+var checkPushPermsCallCount = 0
-var calledCheckPushPermission = false
 
-func setCalledFalse() {
+func resetCalledCount() {
-	calledExecCommand = []bool{}
+	checkPushPermsCallCount = 0
-	calledCheckPushPermission = false
 }
 
 func fakeCheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error {
-	calledCheckPushPermission = true
+	checkPushPermsCallCount++
 	return nil
 }
 
 func TestCheckPushPermissions(t *testing.T) {
 	tests := []struct {
 		description                     string
-		Destination           []string
+		cacheRepo                       string
-		ShouldCallExecCommand []bool
+		checkPushPermsExpectedCallCount int
-		ExistingConfig        bool
+		destinations                    []string
+		existingConfig                  bool
+		noPush                          bool
+		noPushCache                     bool
 	}{
-		{"a gcr image without config", []string{"gcr.io/test-image"}, []bool{true}, false},
+		{description: "a gcr image without config", destinations: []string{"gcr.io/test-image"}, checkPushPermsExpectedCallCount: 1},
-		{"a gcr image with config", []string{"gcr.io/test-image"}, []bool{false}, true},
+		{description: "a gcr image with config", destinations: []string{"gcr.io/test-image"}, existingConfig: true, checkPushPermsExpectedCallCount: 1},
-		{"a pkg.dev image without config", []string{"us-docker.pkg.dev/test-image"}, []bool{true}, false},
+		{description: "a pkg.dev image without config", destinations: []string{"us-docker.pkg.dev/test-image"}, checkPushPermsExpectedCallCount: 1},
-		{"a pkg.dev image with config", []string{"us-docker.pkg.dev/test-image"}, []bool{false}, true},
+		{description: "a pkg.dev image with config", destinations: []string{"us-docker.pkg.dev/test-image"}, existingConfig: true, checkPushPermsExpectedCallCount: 1},
-		{"localhost registry with config", []string{"localhost:5000/test-image"}, []bool{false}, false},
+		{description: "localhost registry without config", destinations: []string{"localhost:5000/test-image"}, checkPushPermsExpectedCallCount: 1},
-		{"localhost registry without config", []string{"localhost:5000/test-image"}, []bool{false}, true},
+		{description: "localhost registry with config", destinations: []string{"localhost:5000/test-image"}, existingConfig: true, checkPushPermsExpectedCallCount: 1},
-		{"any other registry", []string{"notgcr.io/test-image"}, []bool{false}, false},
+		{description: "any other registry", destinations: []string{"notgcr.io/test-image"}, checkPushPermsExpectedCallCount: 1},
-		{"multiple destinations pushed to different registry",
+		{
-			[]string{
+			description: "multiple destinations pushed to different registry",
+			destinations: []string{
 				"us-central1-docker.pkg.dev/prj/test-image",
 				"us-west-docker.pkg.dev/prj/test-image",
 			},
-			[]bool{true, true}, false,
+			checkPushPermsExpectedCallCount: 2,
 		},
-		{"same image names with different tags",
+		{
-			[]string{
+			description: "same image names with different tags",
+			destinations: []string{
 				"us-central1-docker.pkg.dev/prj/test-image:tag1",
 				"us-central1-docker.pkg.dev/prj/test-image:tag2",
 			},
-			[]bool{true, true}, false,
+			checkPushPermsExpectedCallCount: 1,
 		},
-		{"same destination image multiple times",
+		{
-			[]string{
+			description: "same destination image multiple times",
+			destinations: []string{
 				"us-central1-docker.pkg.dev/prj/test-image",
 				"us-central1-docker.pkg.dev/prj/test-image",
 			},
-			[]bool{true, false}, false,
+			checkPushPermsExpectedCallCount: 1,
+		},
+		{
+			description:                     "no push and no push cache",
+			destinations:                    []string{"us-central1-docker.pkg.dev/prj/test-image"},
+			checkPushPermsExpectedCallCount: 0,
+			noPush:                          true,
+			noPushCache:                     true,
+		},
+		{
+			description:                     "no push and push cache",
+			destinations:                    []string{"us-central1-docker.pkg.dev/prj/test-image"},
+			cacheRepo:                       "us-central1-docker.pkg.dev/prj/cache-image",
+			checkPushPermsExpectedCallCount: 1,
+			noPush:                          true,
+		},
+		{
+			description:                     "no push and cache repo is OCI image layout",
+			destinations:                    []string{"us-central1-docker.pkg.dev/prj/test-image"},
+			cacheRepo:                       "oci:/some-layout-path",
+			checkPushPermsExpectedCallCount: 0,
+			noPush:                          true,
 		},
 	}
 
 	checkRemotePushPermission = fakeCheckPushPermission
 	for _, test := range tests {
 		t.Run(test.description, func(t *testing.T) {
-			setCalledFalse()
+			resetCalledCount()
 			fs = afero.NewMemMapFs()
 			opts := config.KanikoOptions{
-				Destinations: test.Destination,
+				CacheRepo:    test.cacheRepo,
+				Destinations: test.destinations,
+				NoPush:       test.noPush,
+				NoPushCache:  test.noPushCache,
 			}
-			if test.ExistingConfig {
+			if test.existingConfig {
 				afero.WriteFile(fs, util.DockerConfLocation(), []byte(""), os.FileMode(0644))
 				defer fs.Remove(util.DockerConfLocation())
 			}
 			CheckPushPermissions(&opts)
-			for i, shdCall := range test.ShouldCallExecCommand {
+			if checkPushPermsCallCount != test.checkPushPermsExpectedCallCount {
-				if i < len(calledExecCommand) && shdCall != calledExecCommand[i] {
+				t.Errorf("expected check push permissions call count to be %d but it was %d", test.checkPushPermsExpectedCallCount, checkPushPermsCallCount)
-					t.Errorf("Expected calledExecCommand to be %v however it was %v",
-						calledExecCommand, shdCall)
-				}
 			}
 		})
 	}