Merge pull request #1190 from dani29/onbuildcp

Multistage ONBUILD COPY Support
This commit is contained in:
Dani Raznikov 2020-04-15 21:29:58 +03:00 committed by GitHub
commit 1534f90c93
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 315 additions and 168 deletions

View File

@ -22,9 +22,9 @@ import (
"io/ioutil"
"net/http"
"regexp"
"strconv"
"strings"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/sirupsen/logrus"
"github.com/GoogleContainerTools/kaniko/pkg/config"
@ -34,15 +34,14 @@ import (
"github.com/pkg/errors"
)
// Stages parses a Dockerfile and returns an array of KanikoStage
func Stages(opts *config.KanikoOptions) ([]config.KanikoStage, error) {
func ParseStages(opts *config.KanikoOptions) ([]instructions.Stage, []instructions.ArgCommand, error) {
var err error
var d []uint8
match, _ := regexp.MatchString("^https?://", opts.DockerfilePath)
if match {
response, e := http.Get(opts.DockerfilePath)
if e != nil {
return nil, e
return nil, nil, e
}
d, err = ioutil.ReadAll(response.Body)
} else {
@ -50,66 +49,15 @@ func Stages(opts *config.KanikoOptions) ([]config.KanikoStage, error) {
}
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("reading dockerfile at path %s", opts.DockerfilePath))
return nil, nil, errors.Wrap(err, fmt.Sprintf("reading dockerfile at path %s", opts.DockerfilePath))
}
stages, metaArgs, err := Parse(d)
if err != nil {
return nil, errors.Wrap(err, "parsing dockerfile")
}
targetStage, err := targetStage(stages, opts.Target)
if err != nil {
return nil, err
}
resolveStages(stages)
args := unifyArgs(metaArgs, opts.BuildArgs)
if err := resolveStagesArgs(stages, args); err != nil {
return nil, errors.Wrap(err, "resolving args")
}
var kanikoStages []config.KanikoStage
for index, stage := range stages {
if len(stage.Name) > 0 {
logrus.Infof("Resolved base name %s to %s", stage.BaseName, stage.Name)
}
baseImageIndex := baseImageIndex(index, stages)
kanikoStages = append(kanikoStages, config.KanikoStage{
Stage: stage,
BaseImageIndex: baseImageIndex,
BaseImageStoredLocally: (baseImageIndex != -1),
SaveStage: saveStage(index, stages),
Final: index == targetStage,
MetaArgs: metaArgs,
Index: index,
})
if index == targetStage {
break
}
return nil, nil, errors.Wrap(err, "parsing dockerfile")
}
return kanikoStages, nil
}
// unifyArgs returns the unified args between metaArgs and --build-arg
// by default --build-arg overrides metaArgs except when --build-arg is empty
func unifyArgs(metaArgs []instructions.ArgCommand, buildArgs []string) []string {
argsMap := make(map[string]string)
for _, a := range metaArgs {
if a.Value != nil {
argsMap[a.Key] = *a.Value
}
}
splitter := "="
for _, a := range buildArgs {
s := strings.Split(a, splitter)
if len(s) > 1 && s[1] != "" {
argsMap[s[0]] = s[1]
}
}
var args []string
for k, v := range argsMap {
args = append(args, fmt.Sprintf("%s=%s", k, v))
}
return args
return stages, metaArgs, nil
}
// baseImageIndex returns the index of the stage the current stage is built off
@ -229,44 +177,6 @@ func targetStage(stages []instructions.Stage, target string) (int, error) {
return -1, fmt.Errorf("%s is not a valid target build stage", target)
}
// resolveStages resolves any calls to previous stages with names to indices
// Ex. --from=second_stage should be --from=1 for easier processing later on
// As third party library lowers stage name in FROM instruction, this function resolves stage case insensitively.
func resolveStages(stages []instructions.Stage) {
nameToIndex := make(map[string]string)
for i, stage := range stages {
index := strconv.Itoa(i)
if stage.Name != index {
nameToIndex[stage.Name] = index
}
for _, cmd := range stage.Commands {
switch c := cmd.(type) {
case *instructions.CopyCommand:
if c.From != "" {
if val, ok := nameToIndex[strings.ToLower(c.From)]; ok {
c.From = val
}
}
}
}
}
}
// resolveStagesArgs resolves all the args from list of stages
func resolveStagesArgs(stages []instructions.Stage, args []string) error {
for i, s := range stages {
resolvedBaseName, err := util.ResolveEnvironmentReplacement(s.BaseName, args, false)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("resolving base name %s", s.BaseName))
}
if s.BaseName != resolvedBaseName {
stages[i].BaseName = resolvedBaseName
}
}
return nil
}
// ParseCommands parses an array of commands into an array of instructions.Command; used for onbuild
func ParseCommands(cmdArray []string) ([]instructions.Command, error) {
var cmds []instructions.Command
@ -303,3 +213,102 @@ func saveStage(index int, stages []instructions.Stage) bool {
return false
}
// ResolveCrossStageCommands resolves any calls to previous stages with names to indices
// Ex. --from=secondStage should be --from=1 for easier processing later on
// As third party library lowers stage name in FROM instruction, this function resolves stage case insensitively.
func ResolveCrossStageCommands(cmds []instructions.Command, stageNameToIdx map[string]string) {
for _, cmd := range cmds {
switch c := cmd.(type) {
case *instructions.CopyCommand:
if c.From != "" {
if val, ok := stageNameToIdx[strings.ToLower(c.From)]; ok {
c.From = val
}
}
}
}
}
// resolveStagesArgs resolves all the args from list of stages
func resolveStagesArgs(stages []instructions.Stage, args []string) error {
for i, s := range stages {
resolvedBaseName, err := util.ResolveEnvironmentReplacement(s.BaseName, args, false)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("resolving base name %s", s.BaseName))
}
if s.BaseName != resolvedBaseName {
stages[i].BaseName = resolvedBaseName
}
}
return nil
}
func MakeKanikoStages(opts *config.KanikoOptions, stages []instructions.Stage, metaArgs []instructions.ArgCommand) ([]config.KanikoStage, error) {
targetStage, err := targetStage(stages, opts.Target)
if err != nil {
return nil, errors.Wrap(err, "Error finding target stage")
}
args := unifyArgs(metaArgs, opts.BuildArgs)
if err := resolveStagesArgs(stages, args); err != nil {
return nil, errors.Wrap(err, "resolving args")
}
var kanikoStages []config.KanikoStage
for index, stage := range stages {
if len(stage.Name) > 0 {
logrus.Infof("Resolved base name %s to %s", stage.BaseName, stage.Name)
}
baseImageIndex := baseImageIndex(index, stages)
kanikoStages = append(kanikoStages, config.KanikoStage{
Stage: stage,
BaseImageIndex: baseImageIndex,
BaseImageStoredLocally: (baseImageIndex != -1),
SaveStage: saveStage(index, stages),
Final: index == targetStage,
MetaArgs: metaArgs,
Index: index,
})
if index == targetStage {
break
}
}
return kanikoStages, nil
}
func GetOnBuildInstructions(config *v1.Config, stageNameToIdx map[string]string) ([]instructions.Command, error) {
if config.OnBuild == nil || len(config.OnBuild) == 0 {
return nil, nil
}
cmds, err := ParseCommands(config.OnBuild)
if err != nil {
return nil, err
}
// Iterate over commands and replace references to other stages with their index
ResolveCrossStageCommands(cmds, stageNameToIdx)
return cmds, nil
}
// unifyArgs returns the unified args between metaArgs and --build-arg
// by default --build-arg overrides metaArgs except when --build-arg is empty
func unifyArgs(metaArgs []instructions.ArgCommand, buildArgs []string) []string {
argsMap := make(map[string]string)
for _, a := range metaArgs {
if a.Value != nil {
argsMap[a.Key] = *a.Value
}
}
splitter := "="
for _, a := range buildArgs {
s := strings.Split(a, splitter)
if len(s) > 1 && s[1] != "" {
argsMap[s[0]] = s[1]
}
}
var args []string
for k, v := range argsMap {
args = append(args, fmt.Sprintf("%s=%s", k, v))
}
return args
}

View File

@ -20,17 +20,19 @@ import (
"fmt"
"io/ioutil"
"os"
"strconv"
"reflect"
"testing"
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/testutil"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
)
func Test_Stages_ArgValueWithQuotes(t *testing.T) {
func Test_ParseStages_ArgValueWithQuotes(t *testing.T) {
dockerfile := `
ARG IMAGE="ubuntu:16.04"
ARG FOO=bar
FROM ${IMAGE}
RUN echo hi > /hi
@ -54,25 +56,23 @@ func Test_Stages_ArgValueWithQuotes(t *testing.T) {
t.Fatal(err)
}
stages, err := Stages(&config.KanikoOptions{DockerfilePath: tmpfile.Name()})
stages, metaArgs, err := ParseStages(&config.KanikoOptions{DockerfilePath: tmpfile.Name()})
if err != nil {
t.Fatal(err)
}
if len(stages) == 0 {
t.Fatal("length of stages expected to be greater than zero, but was zero")
}
if len(stages[0].MetaArgs) == 0 {
t.Fatal("length of stage[0] meta args expected to be greater than zero, but was zero")
if len(metaArgs) != 2 {
t.Fatalf("length of stage meta args expected to be 2, but was %d", len(metaArgs))
}
expectedVal := "ubuntu:16.04"
arg := stages[0].MetaArgs[0]
if arg.ValueString() != expectedVal {
t.Fatalf("expected stages[0].MetaArgs[0] val to be %s but was %s", expectedVal, arg.ValueString())
for i, expectedVal := range []string{"ubuntu:16.04", "bar"} {
if metaArgs[i].ValueString() != expectedVal {
t.Fatalf("expected metaArg %d val to be %s but was %s", i, expectedVal, metaArgs[i].ValueString())
}
}
}
@ -190,40 +190,63 @@ func Test_stripEnclosingQuotes(t *testing.T) {
}
}
func Test_resolveStages(t *testing.T) {
dockerfile := `
FROM scratch
RUN echo hi > /hi
FROM scratch AS second
COPY --from=0 /hi /hi2
FROM scratch AS tHiRd
COPY --from=second /hi2 /hi3
COPY --from=1 /hi2 /hi3
FROM scratch
COPY --from=thIrD /hi3 /hi4
COPY --from=third /hi3 /hi4
COPY --from=2 /hi3 /hi4
`
stages, _, err := Parse([]byte(dockerfile))
if err != nil {
t.Fatal(err)
func Test_GetOnBuildInstructions(t *testing.T) {
type testCase struct {
name string
cfg *v1.Config
stageToIdx map[string]string
expCommands []instructions.Command
}
resolveStages(stages)
for index, stage := range stages {
if index == 0 {
continue
}
expectedStage := strconv.Itoa(index - 1)
for _, command := range stage.Commands {
copyCmd := command.(*instructions.CopyCommand)
if copyCmd.From != expectedStage {
t.Fatalf("unexpected copy command: %s resolved to stage %s, expected %s", copyCmd.String(), copyCmd.From, expectedStage)
}
}
tests := []testCase{
{name: "no on-build on config",
cfg: &v1.Config{},
stageToIdx: map[string]string{"builder": "0"},
expCommands: nil,
},
{name: "onBuild on config, nothing to resolve",
cfg: &v1.Config{OnBuild: []string{"WORKDIR /app"}},
stageToIdx: map[string]string{"builder": "0", "temp": "1"},
expCommands: []instructions.Command{&instructions.WorkdirCommand{Path: "/app"}},
},
{name: "onBuild on config, resolve multiple stages",
cfg: &v1.Config{OnBuild: []string{"COPY --from=builder a.txt b.txt", "COPY --from=temp /app /app"}},
stageToIdx: map[string]string{"builder": "0", "temp": "1"},
expCommands: []instructions.Command{
&instructions.CopyCommand{
SourcesAndDest: []string{"a.txt b.txt"},
From: "0",
},
&instructions.CopyCommand{
SourcesAndDest: []string{"/app /app"},
From: "1",
},
}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cmds, err := GetOnBuildInstructions(test.cfg, test.stageToIdx)
if err != nil {
t.Fatalf("Failed to parse config for on-build instructions")
}
if len(cmds) != len(test.expCommands) {
t.Fatalf("Expected %d commands, got %d", len(test.expCommands), len(cmds))
}
for i, cmd := range cmds {
if reflect.TypeOf(cmd) != reflect.TypeOf(test.expCommands[i]) {
t.Fatalf("Got command %s, expected %s", cmd, test.expCommands[i])
}
switch c := cmd.(type) {
case *instructions.CopyCommand:
{
exp := test.expCommands[i].(*instructions.CopyCommand)
testutil.CheckDeepEqual(t, exp.From, c.From)
}
}
}
})
}
}
@ -398,8 +421,6 @@ func Test_ResolveStagesArgs(t *testing.T) {
t.Fatal(err)
}
stagesLen := len(stages)
resolveStages(stages)
args := unifyArgs(metaArgs, buildArgs)
if err := resolveStagesArgs(stages, args); err != nil {
t.Fatalf("fail to resolves args %v: %v", buildArgs, err)

View File

@ -52,6 +52,11 @@ import (
// This is the size of an empty tar in Go
const emptyTarSize = 1024
// for testing
var (
initializeConfig = initConfig
)
type cachePusher func(*config.KanikoOptions, string, string, string) error
type snapShotter interface {
Init() error
@ -78,7 +83,7 @@ type stageBuilder struct {
}
// newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage
func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string, dcm map[string]string, sid map[string]string) (*stageBuilder, error) {
func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, crossStageDeps map[int][]string, dcm map[string]string, sid map[string]string, stageNameToIdx map[string]string) (*stageBuilder, error) {
sourceImage, err := util.RetrieveSourceImage(stage, opts)
if err != nil {
return nil, err
@ -89,7 +94,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross
return nil, err
}
if err := resolveOnBuild(&stage, &imageConfig.Config); err != nil {
if err := resolveOnBuild(&stage, &imageConfig.Config, stageNameToIdx); err != nil {
return nil, err
}
@ -136,7 +141,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross
return s, nil
}
func initializeConfig(img partial.WithConfigFile, opts *config.KanikoOptions) (*v1.ConfigFile, error) {
func initConfig(img partial.WithConfigFile, opts *config.KanikoOptions) (*v1.ConfigFile, error) {
imageConfig, err := img.ConfigFile()
if err != nil {
return nil, err
@ -487,7 +492,7 @@ func (s *stageBuilder) saveLayerToImage(layer v1.Layer, createdBy string) error
return err
}
func CalculateDependencies(stages []config.KanikoStage, opts *config.KanikoOptions) (map[int][]string, error) {
func CalculateDependencies(stages []config.KanikoStage, opts *config.KanikoOptions, stageNameToIdx map[string]string) (map[int][]string, error) {
images := []v1.Image{}
depGraph := map[int][]string{}
for _, s := range stages {
@ -509,7 +514,11 @@ func CalculateDependencies(stages []config.KanikoStage, opts *config.KanikoOptio
if err != nil {
return nil, err
}
for _, c := range s.Commands {
cmds, err := dockerfile.GetOnBuildInstructions(&cfg.Config, stageNameToIdx)
cmds = append(cmds, s.Commands...)
for _, c := range cmds {
switch cmd := c.(type) {
case *instructions.CopyCommand:
if cmd.From != "" {
@ -550,8 +559,13 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
digestToCacheKey := make(map[string]string)
stageIdxToDigest := make(map[string]string)
// Parse dockerfile
stages, err := dockerfile.Stages(opts)
stages, metaArgs, err := dockerfile.ParseStages(opts)
if err != nil {
return nil, err
}
stageNameToIdx := ResolveCrossStageInstructions(stages)
kanikoStages, err := dockerfile.MakeKanikoStages(opts, stages, metaArgs)
if err != nil {
return nil, err
}
@ -561,17 +575,17 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) {
}
// Some stages may refer to other random images, not previous stages
if err := fetchExtraStages(stages, opts); err != nil {
if err := fetchExtraStages(kanikoStages, opts); err != nil {
return nil, err
}
crossStageDependencies, err := CalculateDependencies(stages, opts)
crossStageDependencies, err := CalculateDependencies(kanikoStages, opts, stageNameToIdx)
if err != nil {
return nil, err
}
logrus.Infof("Built cross stage deps: %v", crossStageDependencies)
for index, stage := range stages {
sb, err := newStageBuilder(opts, stage, crossStageDependencies, digestToCacheKey, stageIdxToDigest)
for index, stage := range kanikoStages {
sb, err := newStageBuilder(opts, stage, crossStageDependencies, digestToCacheKey, stageIdxToDigest, stageNameToIdx)
if err != nil {
return nil, err
}
@ -771,15 +785,12 @@ func getHasher(snapshotMode string) (func(string) (string, error), error) {
return nil, fmt.Errorf("%s is not a valid snapshot mode", snapshotMode)
}
func resolveOnBuild(stage *config.KanikoStage, config *v1.Config) error {
if config.OnBuild == nil || len(config.OnBuild) == 0 {
return nil
}
// Otherwise, parse into commands
cmds, err := dockerfile.ParseCommands(config.OnBuild)
func resolveOnBuild(stage *config.KanikoStage, config *v1.Config, stageNameToIdx map[string]string) error {
cmds, err := dockerfile.GetOnBuildInstructions(config, stageNameToIdx)
if err != nil {
return err
}
// Append to the beginning of the commands in the stage
stage.Commands = append(cmds, stage.Commands...)
logrus.Infof("Executing %v build triggers", len(cmds))
@ -808,3 +819,19 @@ func reviewConfig(stage config.KanikoStage, config *v1.Config) {
config.Cmd = nil
}
}
// iterates over a list of stages and resolves instructions referring to earlier stages
// returns a mapping of stage name to stage id, f.e - ["first": "0", "second": "1", "target": "2"]
func ResolveCrossStageInstructions(stages []instructions.Stage) map[string]string {
nameToIndex := make(map[string]string)
for i, stage := range stages {
index := strconv.Itoa(i)
if stage.Name != "" {
nameToIndex[stage.Name] = index
}
dockerfile.ResolveCrossStageCommands(stage.Commands, nameToIndex)
}
logrus.Debugf("Built stage name to index map: %v", nameToIndex)
return nameToIndex
}

View File

@ -25,6 +25,7 @@ import (
"path/filepath"
"reflect"
"sort"
"strconv"
"testing"
"github.com/GoogleContainerTools/kaniko/pkg/commands"
@ -35,6 +36,7 @@ import (
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
)
@ -194,7 +196,8 @@ func Test_stageBuilder_shouldTakeSnapshot(t *testing.T) {
func TestCalculateDependencies(t *testing.T) {
type args struct {
dockerfile string
dockerfile string
mockInitConfig func(partial.WithConfigFile, *config.KanikoOptions) (*v1.ConfigFile, error)
}
tests := []struct {
name string
@ -314,19 +317,58 @@ COPY --from=stage2 /bar /bat
1: {"/bar"},
},
},
{
name: "one image has onbuild config",
args: args{
mockInitConfig: func(img partial.WithConfigFile, opts *config.KanikoOptions) (*v1.ConfigFile, error) {
cfg, err := img.ConfigFile()
// if image is "alpine" then add ONBUILD to its config
if cfg != nil && cfg.Architecture != "" {
cfg.Config.OnBuild = []string{"COPY --from=builder /app /app"}
}
return cfg, err
},
dockerfile: `
FROM scratch as builder
RUN foo
FROM alpine as second
# This image has an ONBUILD command so it will be executed
COPY --from=builder /foo /bar
FROM scratch as target
COPY --from=second /bar /bat
`,
},
want: map[int][]string{
0: {"/app", "/foo"},
1: {"/bar"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.args.mockInitConfig != nil {
original := initializeConfig
defer func() { initializeConfig = original }()
initializeConfig = tt.args.mockInitConfig
}
f, _ := ioutil.TempFile("", "")
ioutil.WriteFile(f.Name(), []byte(tt.args.dockerfile), 0755)
opts := &config.KanikoOptions{
DockerfilePath: f.Name(),
}
testStages, err := dockerfile.Stages(opts)
testStages, metaArgs, err := dockerfile.ParseStages(opts)
if err != nil {
t.Errorf("Failed to parse test dockerfile to stages: %s", err)
}
got, err := CalculateDependencies(testStages, opts)
stageNameToIdx := ResolveCrossStageInstructions(testStages)
kanikoStages, err := dockerfile.MakeKanikoStages(opts, testStages, metaArgs)
if err != nil {
t.Errorf("Failed to parse stages to Kaniko Stages: %s", err)
}
got, err := CalculateDependencies(kanikoStages, opts, stageNameToIdx)
if err != nil {
t.Errorf("got error: %s,", err)
}
@ -870,12 +912,16 @@ COPY %s bar.txt
DockerfilePath: f.Name(),
}
stages, err := dockerfile.Stages(opts)
testStages, metaArgs, err := dockerfile.ParseStages(opts)
if err != nil {
t.Errorf("could not parse test dockerfile")
t.Errorf("Failed to parse test dockerfile to stages: %s", err)
}
stage := stages[0]
_ = ResolveCrossStageInstructions(testStages)
kanikoStages, err := dockerfile.MakeKanikoStages(opts, testStages, metaArgs)
if err != nil {
t.Errorf("Failed to parse stages to Kaniko Stages: %s", err)
}
stage := kanikoStages[0]
cmds := stage.Commands
return testcase{
@ -941,12 +987,17 @@ COPY %s bar.txt
DockerfilePath: f.Name(),
}
stages, err := dockerfile.Stages(opts)
testStages, metaArgs, err := dockerfile.ParseStages(opts)
if err != nil {
t.Errorf("could not parse test dockerfile")
t.Errorf("Failed to parse test dockerfile to stages: %s", err)
}
_ = ResolveCrossStageInstructions(testStages)
kanikoStages, err := dockerfile.MakeKanikoStages(opts, testStages, metaArgs)
if err != nil {
t.Errorf("Failed to parse stages to Kaniko Stages: %s", err)
}
stage := stages[0]
stage := kanikoStages[0]
cmds := stage.Commands
return testcase{
@ -1247,3 +1298,42 @@ func hashCompositeKeys(t *testing.T, ck1 CompositeCache, ck2 CompositeCache) (st
}
return key1, key2
}
func Test_ResolveCrossStageInstructions(t *testing.T) {
df := `
FROM scratch
RUN echo hi > /hi
FROM scratch AS second
COPY --from=0 /hi /hi2
FROM scratch AS tHiRd
COPY --from=second /hi2 /hi3
COPY --from=1 /hi2 /hi3
FROM scratch
COPY --from=thIrD /hi3 /hi4
COPY --from=third /hi3 /hi4
COPY --from=2 /hi3 /hi4
`
stages, _, err := dockerfile.Parse([]byte(df))
if err != nil {
t.Fatal(err)
}
stageToIdx := ResolveCrossStageInstructions(stages)
for index, stage := range stages {
if index == 0 {
continue
}
expectedStage := strconv.Itoa(index - 1)
for _, command := range stage.Commands {
copyCmd := command.(*instructions.CopyCommand)
if copyCmd.From != expectedStage {
t.Fatalf("unexpected copy command: %s resolved to stage %s, expected %s", copyCmd.String(), copyCmd.From, expectedStage)
}
}
expectedMap := map[string]string{"second": "1", "third": "2"}
testutil.CheckDeepEqual(t, expectedMap, stageToIdx)
}
}