diff --git a/IntegrationGuide.md b/IntegrationGuide.md new file mode 100644 index 0000000..cb6b0c8 --- /dev/null +++ b/IntegrationGuide.md @@ -0,0 +1,16 @@ +# Integrating with Orchard + +Orchard has a REST API that follows [OpenAPI specification](https://swagger.io/specification/) and is described in `api/openapi.yaml`. + +You can run `orchard dev` locally and navigate to http://127.0.0.1:6120/v1/ for interactive documentation. + +![](docs/orchard-api-documentation-browser.png) + +## Resource management + +Some resources, such as `Worker` and `VM`, have a `resource` field which is a dictionary that maps between resource names and their amounts (amount requested or amount provided, depending on the resource) and is useful for scheduling. + +Well-known resources: + +* `org.cirruslabs.tart-vms` — number of Tart VM slots available on the machine or requested by the VM + * this number is `2` for workers and `1` for VMs by default diff --git a/api/api.go b/api/api.go new file mode 100644 index 0000000..59217ca --- /dev/null +++ b/api/api.go @@ -0,0 +1,6 @@ +package api + +import _ "embed" + +//go:embed openapi.yaml +var Spec []byte diff --git a/api/openapi.yaml b/api/openapi.yaml index 558cd58..1d33865 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -258,6 +258,14 @@ components: name: type: string description: Node name + resources: + type: object + description: | + Dictionary that maps the resource name to the + amount of this resource provided by the worker + for running VMs. + additionalProperties: + type: integer VM: title: Virtual Machine type: object @@ -265,6 +273,14 @@ components: name: type: string description: VM name + resources: + type: object + description: | + Dictionary that maps the resource name to the + amount of this resource required on the worker + for this VM to be scheduled + additionalProperties: + type: integer Events: title: Events type: object diff --git a/docs/orchard-api-documentation-browser.png b/docs/orchard-api-documentation-browser.png new file mode 100644 index 0000000..cbe1f6d Binary files /dev/null and b/docs/orchard-api-documentation-browser.png differ diff --git a/go.mod b/go.mod index f635b45..d7d1882 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/dgraph-io/badger/v3 v3.2103.5 github.com/dustin/go-humanize v1.0.0 github.com/gin-gonic/gin v1.8.2 + github.com/go-openapi/runtime v0.25.0 github.com/gofrs/flock v0.8.1 github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.3.0 @@ -26,11 +27,15 @@ require ( golang.org/x/term v0.5.0 google.golang.org/grpc v1.53.0 google.golang.org/protobuf v1.28.1 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 howett.net/plist v1.0.0 ) require ( + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect @@ -38,9 +43,19 @@ require ( github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/fatih/color v1.13.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-openapi/analysis v0.21.2 // indirect + github.com/go-openapi/errors v0.20.2 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/loads v0.21.1 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/strfmt v0.21.2 // indirect + github.com/go-openapi/swag v0.21.1 // indirect + github.com/go-openapi/validate v0.21.0 // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-playground/validator/v10 v10.11.1 // indirect + github.com/go-stack/stack v1.8.1 // indirect github.com/goccy/go-json v0.9.11 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.0.0 // indirect @@ -49,26 +64,30 @@ require ( github.com/google/flatbuffers v1.12.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.12.3 // indirect + github.com/klauspost/compress v1.13.6 // indirect github.com/leodido/go-urn v1.2.1 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.9 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oklog/ulid v1.3.1 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/ugorji/go/codec v1.2.7 // indirect + go.mongodb.org/mongo-driver v1.8.3 // indirect go.opencensus.io v0.22.5 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index e6cde5b..b60d7db 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/avast/retry-go/v4 v4.3.3 h1:G56Bp6mU0b5HE1SkaoVjscZjlQb0oy4mezwY/cGH19w= @@ -51,6 +58,33 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.8.2 h1:UzKToD9/PoFj/V4rvlKqTRKnQYyz8Sc1MJlv4JHPtvY= github.com/gin-gonic/gin v1.8.2/go.mod h1:qw5AYuDrzRTnhvusDsrov+fDIxp9Dleuu12h8nfB398= +github.com/go-openapi/analysis v0.21.2 h1:hXFrOYFHUAMQdu6zwAiKKJHJQ8kqZs1ux/ru1P1wLJU= +github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= +github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.20.2 h1:dxy7PGTqEh94zj2E3h1cUmQQWiM1+aeCROfAr02EmK8= +github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/loads v0.21.1 h1:Wb3nVZpdEzDTcly8S4HMkey6fjARRzb7iEaySimlDW0= +github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= +github.com/go-openapi/runtime v0.25.0 h1:7yQTCdRbWhX8vnIjdzU8S00tBYf7Sg71EBeorlPHvhc= +github.com/go-openapi/runtime v0.25.0/go.mod h1:Ux6fikcHXyyob6LNWxtE96hWwjBPYF0DXgVFuMTneOs= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= +github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= +github.com/go-openapi/strfmt v0.21.2 h1:5NDNgadiX1Vhemth/TH4gCGopWSTdDjxl60H3B7f+os= +github.com/go-openapi/strfmt v0.21.2/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/validate v0.21.0 h1:+Wqk39yKOhfpLqNLEC0/eViCkzM5FVXVqrvt526+wcI= +github.com/go-openapi/validate v0.21.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -59,6 +93,33 @@ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/j github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= @@ -79,16 +140,19 @@ github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaW github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= @@ -104,12 +168,20 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -121,8 +193,15 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -135,15 +214,26 @@ github.com/mitchellh/go-grpc-net-conn v0.0.0-20200427190222-eb030e4876f0 h1:oZue github.com/mitchellh/go-grpc-net-conn v0.0.0-20200427190222-eb030e4876f0/go.mod h1:ZCzL0JMR6qfm7VrDC8HGwVtPA8D2Ijc/edUSBw58x94= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -151,11 +241,17 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -163,6 +259,7 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= @@ -172,6 +269,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -184,13 +282,23 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= +go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= +go.mongodb.org/mongo-driver v1.8.3 h1:TDKlTkGDKm9kkJVUOAXDK5/fkqKHJVwYQSpoRfB43R4= +go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= @@ -200,10 +308,14 @@ go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= @@ -221,6 +333,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -228,19 +341,27 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -249,11 +370,13 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= @@ -262,7 +385,11 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -291,6 +418,7 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -298,9 +426,12 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/command/create/vm.go b/internal/command/create/vm.go index 7a5e204..4d07597 100644 --- a/internal/command/create/vm.go +++ b/internal/command/create/vm.go @@ -1,16 +1,21 @@ package create import ( + "errors" + "fmt" "github.com/cirruslabs/orchard/pkg/client" v1 "github.com/cirruslabs/orchard/pkg/resource/v1" "github.com/spf13/cobra" ) +var ErrVMFailed = errors.New("failed to create VM") + var image string var cpu uint64 var memory uint64 var softnet bool var headless bool +var stringToStringResources map[string]string func newCreateVMCommand() *cobra.Command { command := &cobra.Command{ @@ -25,6 +30,8 @@ func newCreateVMCommand() *cobra.Command { command.PersistentFlags().Uint64Var(&memory, "memory", 8*1024, "megabytes of memory to use") command.PersistentFlags().BoolVar(&softnet, "softnet", false, "whether to use Softnet network isolation") command.PersistentFlags().BoolVar(&headless, "headless", true, "whether to run without graphics") + command.PersistentFlags().StringToStringVar(&stringToStringResources, "resources", map[string]string{}, + "resources to request for this VM") return command } @@ -32,6 +39,12 @@ func newCreateVMCommand() *cobra.Command { func runCreateVM(cmd *cobra.Command, args []string) error { name := args[0] + // Convert resources + resources, err := v1.NewResourcesFromStringToString(stringToStringResources) + if err != nil { + return fmt.Errorf("%w: %v", ErrVMFailed, err) + } + client, err := client.New() if err != nil { return err @@ -41,10 +54,11 @@ func runCreateVM(cmd *cobra.Command, args []string) error { Meta: v1.Meta{ Name: name, }, - Image: image, - CPU: cpu, - Memory: memory, - Softnet: softnet, - Headless: headless, + Image: image, + CPU: cpu, + Memory: memory, + Softnet: softnet, + Headless: headless, + Resources: resources, }) } diff --git a/internal/command/dev/dev.go b/internal/command/dev/dev.go index f3cdc5e..b9fdaa1 100644 --- a/internal/command/dev/dev.go +++ b/internal/command/dev/dev.go @@ -1,10 +1,14 @@ package dev import ( + "errors" + "fmt" "github.com/cirruslabs/orchard/internal/config" "github.com/cirruslabs/orchard/internal/controller" + "github.com/cirruslabs/orchard/internal/netconstants" "github.com/cirruslabs/orchard/internal/worker" "github.com/cirruslabs/orchard/pkg/client" + v1 "github.com/cirruslabs/orchard/pkg/resource/v1" "github.com/spf13/cobra" "go.uber.org/zap" "os" @@ -12,7 +16,10 @@ import ( "path/filepath" ) +var ErrFailed = errors.New("failed to run development controller and worker") + var devDataDirPath string +var stringToStringResources map[string]string func NewCommand() *cobra.Command { command := &cobra.Command{ @@ -23,6 +30,8 @@ func NewCommand() *cobra.Command { command.PersistentFlags().StringVarP(&devDataDirPath, "data-dir", "d", ".dev-data", "path to persist data between runs") + command.PersistentFlags().StringToStringVar(&stringToStringResources, "resources", map[string]string{}, + "resources that the development worker will provide") return command } @@ -36,7 +45,14 @@ func runDev(cmd *cobra.Command, args []string) error { devDataDirPath = path.Join(pwd, devDataDirPath) } - devController, devWorker, err := CreateDevControllerAndWorker(devDataDirPath) + // Convert resources + resources, err := v1.NewResourcesFromStringToString(stringToStringResources) + if err != nil { + return fmt.Errorf("%w: %v", ErrFailed, err) + } + + devController, devWorker, err := CreateDevControllerAndWorker(devDataDirPath, + fmt.Sprintf(":%d", netconstants.DefaultControllerPort), resources) if err != nil { return err @@ -59,7 +75,11 @@ func runDev(cmd *cobra.Command, args []string) error { return <-errChan } -func CreateDevControllerAndWorker(devDataDirPath string) (*controller.Controller, *worker.Worker, error) { +func CreateDevControllerAndWorker( + devDataDirPath string, + controllerListenAddr string, + resources v1.Resources, +) (*controller.Controller, *worker.Worker, error) { // Initialize the logger logger, err := zap.NewDevelopment() if err != nil { @@ -78,8 +98,9 @@ func CreateDevControllerAndWorker(devDataDirPath string) (*controller.Controller devController, err := controller.New( controller.WithDataDir(dataDir), - controller.WithListenAddr(":0"), + controller.WithListenAddr(controllerListenAddr), controller.WithInsecureAuthDisabled(), + controller.WithSwaggerDocs(), controller.WithLogger(logger), ) if err != nil { @@ -90,7 +111,7 @@ func CreateDevControllerAndWorker(devDataDirPath string) (*controller.Controller if err != nil { return nil, nil, err } - devWorker, err := worker.New(defaultClient, worker.WithLogger(logger)) + devWorker, err := worker.New(defaultClient, worker.WithResources(resources), worker.WithLogger(logger)) if err != nil { return nil, nil, err } diff --git a/internal/command/worker/run.go b/internal/command/worker/run.go index c70e1bc..6cda779 100644 --- a/internal/command/worker/run.go +++ b/internal/command/worker/run.go @@ -2,20 +2,25 @@ package worker import ( "errors" + "fmt" "github.com/cirruslabs/orchard/internal/bootstraptoken" "github.com/cirruslabs/orchard/internal/netconstants" "github.com/cirruslabs/orchard/internal/worker" "github.com/cirruslabs/orchard/pkg/client" + v1 "github.com/cirruslabs/orchard/pkg/resource/v1" "github.com/spf13/cobra" "go.uber.org/zap" "go.uber.org/zap/zapcore" "gopkg.in/natefinch/lumberjack.v2" ) +var ErrRunFailed = errors.New("failed to run worker") + var ErrBootstrapTokenNotProvided = errors.New("no bootstrap token provided") var bootstrapTokenRaw string var logFilePath string +var stringToStringResources map[string]string func newRunCommand() *cobra.Command { cmd := &cobra.Command{ @@ -24,10 +29,14 @@ func newRunCommand() *cobra.Command { RunE: runWorker, Args: cobra.ExactArgs(1), } + cmd.PersistentFlags().StringVar(&bootstrapTokenRaw, "bootstrap-token", "", "a bootstrap token retrieved via `orchard get bootstrap-token `") cmd.PersistentFlags().StringVar(&logFilePath, "log-file", "", "optional path to a file where logs (up to 100 Mb) will be written.") + cmd.PersistentFlags().StringToStringVar(&stringToStringResources, "resources", map[string]string{}, + "resources that this worker provides") + return cmd } @@ -44,6 +53,12 @@ func runWorker(cmd *cobra.Command, args []string) (err error) { return err } + // Convert resources + resources, err := v1.NewResourcesFromStringToString(stringToStringResources) + if err != nil { + return fmt.Errorf("%w: %v", ErrRunFailed, err) + } + controllerClient, err := client.New( client.WithAddress(controllerURL.String()), client.WithTrustedCertificate(bootstrapToken.Certificate()), @@ -65,7 +80,11 @@ func runWorker(cmd *cobra.Command, args []string) (err error) { } }() - workerInstance, err := worker.New(controllerClient, worker.WithLogger(logger)) + workerInstance, err := worker.New( + controllerClient, + worker.WithResources(resources), + worker.WithLogger(logger), + ) if err != nil { return err } diff --git a/internal/controller/api.go b/internal/controller/api.go index ba5a964..395e460 100644 --- a/internal/controller/api.go +++ b/internal/controller/api.go @@ -4,12 +4,14 @@ import ( "context" "crypto/subtle" "errors" + "github.com/cirruslabs/orchard/api" storepkg "github.com/cirruslabs/orchard/internal/controller/store" "github.com/cirruslabs/orchard/internal/responder" v1pkg "github.com/cirruslabs/orchard/pkg/resource/v1" "github.com/cirruslabs/orchard/rpc" "github.com/deckarep/golang-set/v2" "github.com/gin-gonic/gin" + "github.com/go-openapi/runtime/middleware" "google.golang.org/grpc/metadata" "net/http" ) @@ -29,9 +31,22 @@ func (controller *Controller) initAPI() *gin.Engine { // A way to for the clients to check that the API is working v1.GET("/", func(c *gin.Context) { - c.Status(http.StatusOK) + if controller.enableSwaggerDocs { + middleware.SwaggerUI(middleware.SwaggerUIOpts{ + Path: "/v1", + SpecURL: "/v1/openapi.yaml", + }, nil).ServeHTTP(c.Writer, c.Request) + } else { + c.Status(http.StatusOK) + } }) + if controller.enableSwaggerDocs { + v1.GET("/openapi.yaml", func(c *gin.Context) { + c.Data(200, "text/yaml", api.Spec) + }) + } + // Service accounts v1.POST("/service-accounts", func(c *gin.Context) { controller.createServiceAccount(c).Respond(c) diff --git a/internal/controller/api_vms.go b/internal/controller/api_vms.go index d6d90b8..b589555 100644 --- a/internal/controller/api_vms.go +++ b/internal/controller/api_vms.go @@ -30,6 +30,14 @@ func (controller *Controller) createVM(ctx *gin.Context) responder.Responder { vm.CreatedAt = time.Now() vm.UID = uuid.New().String() + // Provide resource defaults + if vm.Resources == nil { + vm.Resources = make(v1.Resources) + } + if _, ok := vm.Resources[v1.ResourceTartVMs]; !ok { + vm.Resources[v1.ResourceTartVMs] = 1 + } + return controller.storeUpdate(func(txn storepkg.Transaction) responder.Responder { // Does the VM resource with this name already exists? _, err := txn.GetVM(vm.Name) diff --git a/internal/controller/controller.go b/internal/controller/controller.go index a837ef4..335e6fb 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/cirruslabs/orchard/internal/controller/notifier" "github.com/cirruslabs/orchard/internal/controller/proxy" + "github.com/cirruslabs/orchard/internal/controller/scheduler" storepkg "github.com/cirruslabs/orchard/internal/controller/store" "github.com/cirruslabs/orchard/internal/controller/store/badger" "github.com/cirruslabs/orchard/internal/netconstants" @@ -40,6 +41,7 @@ type Controller struct { grpcServer *grpc.Server workerNotifier *notifier.Notifier proxy *proxy.Proxy + enableSwaggerDocs bool rpc.UnimplementedControllerServer } @@ -137,7 +139,7 @@ func (controller *Controller) Run(ctx context.Context) error { // Run the scheduler so that each VM will eventually // be assigned to a specific Worker go func() { - err := runScheduler(controller.store) + err := scheduler.Run(controller.store) if err != nil { panic(err) } diff --git a/internal/controller/option.go b/internal/controller/option.go index a79f1b8..67f084d 100644 --- a/internal/controller/option.go +++ b/internal/controller/option.go @@ -31,6 +31,12 @@ func WithInsecureAuthDisabled() Option { } } +func WithSwaggerDocs() Option { + return func(controller *Controller) { + controller.enableSwaggerDocs = true + } +} + func WithLogger(logger *zap.Logger) Option { return func(controller *Controller) { controller.logger = logger.Sugar() diff --git a/internal/controller/scheduler.go b/internal/controller/scheduler.go deleted file mode 100644 index 4c47c69..0000000 --- a/internal/controller/scheduler.go +++ /dev/null @@ -1,76 +0,0 @@ -package controller - -import ( - storepkg "github.com/cirruslabs/orchard/internal/controller/store" - "github.com/cirruslabs/orchard/pkg/resource/v1" - "sort" - "time" -) - -const schedulerInterval = 5 * time.Second - -func runScheduler(store storepkg.Store) error { - ticker := time.NewTicker(schedulerInterval) - - for { - if err := runSchedulerInner(store); err != nil { - return err - } - - <-ticker.C - } -} - -func runSchedulerInner(store storepkg.Store) error { - var vms []v1.VM - var workers []v1.Worker - var err error - - err = store.View(func(txn storepkg.Transaction) error { - vms, err = txn.ListVMs() - if err != nil { - return err - } - - workers, err = txn.ListWorkers() - if err != nil { - return err - } - - return nil - }) - if err != nil { - return err - } - - // Sort VMs by date of creation - sort.Slice(vms, func(i, j int) bool { - return vms[i].CreatedAt.Before(vms[j].CreatedAt) - }) - - for _, vm := range vms { - vm := vm - - if vm.Worker != "" { - continue - } - - // Find an appropriate worker to run this VM on - for _, worker := range workers { - worker := worker - - vm.Worker = worker.Name - - err := store.Update(func(txn storepkg.Transaction) error { - return txn.SetVM(vm) - }) - if err != nil { - return err - } - - break - } - } - - return nil -} diff --git a/internal/controller/scheduler/scheduler.go b/internal/controller/scheduler/scheduler.go new file mode 100644 index 0000000..5ada296 --- /dev/null +++ b/internal/controller/scheduler/scheduler.go @@ -0,0 +1,77 @@ +package scheduler + +import ( + storepkg "github.com/cirruslabs/orchard/internal/controller/store" + "github.com/cirruslabs/orchard/pkg/resource/v1" + "sort" + "time" +) + +const schedulerInterval = 5 * time.Second + +func Run(store storepkg.Store) error { + ticker := time.NewTicker(schedulerInterval) + + for { + if err := runInner(store); err != nil { + return err + } + + <-ticker.C + } +} + +func runInner(store storepkg.Store) error { + return store.Update(func(txn storepkg.Transaction) error { + vms, err := txn.ListVMs() + if err != nil { + return err + } + unscheduledVMs, workerToResources := processVMs(vms) + + workers, err := txn.ListWorkers() + if err != nil { + return err + } + + for _, unscheduledVM := range unscheduledVMs { + // Find a worker that can run this VM + for _, worker := range workers { + resourcesUsed := workerToResources.Get(worker.Name) + resourcesRemaining := worker.Resources.Subtracted(resourcesUsed) + + if resourcesRemaining.CanFit(unscheduledVM.Resources) { + unscheduledVM.Worker = worker.Name + + if err := txn.SetVM(unscheduledVM); err != nil { + return err + } + + workerToResources.Add(worker.Name, unscheduledVM.Resources) + } + } + } + + return nil + }) +} + +func processVMs(vms []v1.VM) ([]v1.VM, WorkerToResources) { + var unscheduledVMs []v1.VM + workerToResources := make(WorkerToResources) + + for _, vm := range vms { + if vm.Worker == "" { + unscheduledVMs = append(unscheduledVMs, vm) + } else if !vm.TerminalState() { + workerToResources.Add(vm.Worker, vm.Resources) + } + } + + // Sort unscheduled VMs by the date of creation + sort.Slice(unscheduledVMs, func(i, j int) bool { + return unscheduledVMs[i].CreatedAt.Before(unscheduledVMs[j].CreatedAt) + }) + + return unscheduledVMs, workerToResources +} diff --git a/internal/controller/scheduler/workertoresources.go b/internal/controller/scheduler/workertoresources.go new file mode 100644 index 0000000..f794945 --- /dev/null +++ b/internal/controller/scheduler/workertoresources.go @@ -0,0 +1,26 @@ +package scheduler + +import v1 "github.com/cirruslabs/orchard/pkg/resource/v1" + +type WorkerToResources map[string]v1.Resources + +func (workerToResources WorkerToResources) Add(name string, other v1.Resources) { + workerResources, ok := workerToResources[name] + if !ok { + workerResources = make(v1.Resources) + } + + workerResources.Add(other) + + workerToResources[name] = workerResources +} + +func (workerToResources WorkerToResources) Get(name string) v1.Resources { + workerResources, ok := workerToResources[name] + if !ok { + workerResources = make(v1.Resources) + workerToResources[name] = workerResources + } + + return workerResources +} diff --git a/internal/controller/scheduler/workertoresources_test.go b/internal/controller/scheduler/workertoresources_test.go new file mode 100644 index 0000000..eb04715 --- /dev/null +++ b/internal/controller/scheduler/workertoresources_test.go @@ -0,0 +1,24 @@ +package scheduler_test + +import ( + "github.com/cirruslabs/orchard/internal/controller/scheduler" + v1 "github.com/cirruslabs/orchard/pkg/resource/v1" + "github.com/stretchr/testify/require" + "testing" +) + +func TestWorkerToResources(t *testing.T) { + workerToResources := make(scheduler.WorkerToResources) + require.Len(t, workerToResources, 0) + + workerToResources.Add("worker-name", v1.Resources{ + "tart-vms": 1, + }) + require.Len(t, workerToResources, 1) + + workerToResources.Add("worker-name", v1.Resources{ + "tart-vms": 1, + }) + require.Len(t, workerToResources, 1) + require.Equal(t, v1.Resources{"tart-vms": 2}, workerToResources.Get("worker-name")) +} diff --git a/internal/tests/integration_test.go b/internal/tests/integration_test.go index 9b701a0..3faeb02 100644 --- a/internal/tests/integration_test.go +++ b/internal/tests/integration_test.go @@ -151,7 +151,7 @@ func Wait(duration time.Duration, condition func() bool) bool { func StartIntegrationTestEnvironment(t *testing.T) *client.Client { t.Setenv("ORCHARD_HOME", t.TempDir()) - devController, devWorker, err := dev.CreateDevControllerAndWorker(t.TempDir()) + devController, devWorker, err := dev.CreateDevControllerAndWorker(t.TempDir(), ":0", nil) if err != nil { t.Fatal(err) } diff --git a/internal/worker/option.go b/internal/worker/option.go index dfff19b..4ea85d0 100644 --- a/internal/worker/option.go +++ b/internal/worker/option.go @@ -1,11 +1,18 @@ package worker import ( + v1 "github.com/cirruslabs/orchard/pkg/resource/v1" "go.uber.org/zap" ) type Option func(*Worker) +func WithResources(resources v1.Resources) Option { + return func(worker *Worker) { + worker.resources = resources + } +} + func WithLogger(logger *zap.Logger) Option { return func(worker *Worker) { worker.logger = logger.Sugar() diff --git a/internal/worker/worker.go b/internal/worker/worker.go index b1c966e..a85b2cc 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -23,10 +23,11 @@ var ErrPollFailed = errors.New("failed to poll controller") var ErrRegistrationFailed = errors.New("failed to register worker on the controller") type Worker struct { - name string - vmm *vmmanager.VMManager - client *client.Client - logger *zap.SugaredLogger + name string + vmm *vmmanager.VMManager + client *client.Client + resources v1.Resources + logger *zap.SugaredLogger } func New(client *client.Client, opts ...Option) (*Worker, error) { @@ -49,6 +50,12 @@ func New(client *client.Client, opts ...Option) (*Worker, error) { worker.name = hostname } + + defaultResources := v1.Resources{ + v1.ResourceTartVMs: 2, + } + worker.resources = defaultResources.Merged(worker.resources) + if worker.logger == nil { worker.logger = zap.NewNop().Sugar() } @@ -116,6 +123,7 @@ func (worker *Worker) registerWorker(ctx context.Context) error { Meta: v1.Meta{ Name: worker.name, }, + Resources: worker.resources, LastSeen: time.Now(), MachineID: platformUUID, }) diff --git a/pkg/resource/v1/resources.go b/pkg/resource/v1/resources.go new file mode 100644 index 0000000..28bcaa1 --- /dev/null +++ b/pkg/resource/v1/resources.go @@ -0,0 +1,100 @@ +package v1 + +import ( + "errors" + "fmt" + "strconv" +) + +var ErrInvalidResourceValue = errors.New("resource values should be 64-bit unsigned integers") + +const ( + ResourceTartVMs = "org.cirruslabs.tart-vms" +) + +type Resources map[string]uint64 + +func NewResourcesFromStringToString( + stringToStringResources map[string]string, +) (Resources, error) { + result := make(Resources) + + for key, value := range stringToStringResources { + typedValue, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidResourceValue, err) + } + + result[key] = typedValue + } + + return result, nil +} + +func (resources Resources) Copy() Resources { + result := make(Resources) + + for key, value := range resources { + result[key] = value + } + + return result +} + +func (resources Resources) Add(other Resources) { + for otherKey, otherValue := range other { + resources[otherKey] += otherValue + } +} + +func (resources Resources) Added(other Resources) Resources { + result := resources.Copy() + + for otherKey, otherValue := range other { + result[otherKey] += otherValue + } + + return result +} + +func (resources Resources) Subtract(other Resources) { + for otherKey, otherValue := range other { + resources[otherKey] -= otherValue + } +} + +func (resources Resources) Subtracted(other Resources) Resources { + result := resources.Copy() + + for otherKey, otherValue := range other { + result[otherKey] -= otherValue + } + + return result +} + +func (resources Resources) Merge(other Resources) { + for otherKey, otherValue := range other { + resources[otherKey] = otherValue + } +} + +func (resources Resources) Merged(other Resources) Resources { + result := resources.Copy() + + for otherKey, otherValue := range other { + result[otherKey] = otherValue + } + + return result +} + +func (resources Resources) CanFit(other Resources) bool { + for otherKey, otherValue := range other { + if otherValue > resources[otherKey] { + return false + } + } + + return true +} diff --git a/pkg/resource/v1/resources_test.go b/pkg/resource/v1/resources_test.go new file mode 100644 index 0000000..903ccfd --- /dev/null +++ b/pkg/resource/v1/resources_test.go @@ -0,0 +1,106 @@ +package v1_test + +import ( + v1 "github.com/cirruslabs/orchard/pkg/resource/v1" + "github.com/stretchr/testify/require" + "math" + "testing" +) + +func TestResourcesAdd(t *testing.T) { + resources := v1.Resources{ + v1.ResourceTartVMs: 1, + } + + resources.Add(v1.Resources{v1.ResourceTartVMs: 3}) + + require.Equal(t, v1.Resources{v1.ResourceTartVMs: 4}, resources) +} + +func TestResourcesAdded(t *testing.T) { + resources := v1.Resources{ + v1.ResourceTartVMs: 1, + } + + resources = resources.Added(v1.Resources{v1.ResourceTartVMs: 3}) + + require.Equal(t, v1.Resources{v1.ResourceTartVMs: 4}, resources) +} + +func TestResourcesSubtract(t *testing.T) { + resources := v1.Resources{ + v1.ResourceTartVMs: 2, + } + + resources.Subtract(v1.Resources{ + v1.ResourceTartVMs: 1, + }) + + require.False(t, resources.CanFit(v1.Resources{ + v1.ResourceTartVMs: 2, + })) +} + +func TestResourcesSubtracted(t *testing.T) { + resources := v1.Resources{ + v1.ResourceTartVMs: 2, + } + + resources = resources.Subtracted(v1.Resources{ + v1.ResourceTartVMs: 1, + }) + + require.False(t, resources.CanFit(v1.Resources{ + v1.ResourceTartVMs: 2, + })) +} + +func TestResourcesCanFit(t *testing.T) { + resources := v1.Resources{ + v1.ResourceTartVMs: 2, + } + + require.True(t, resources.CanFit(v1.Resources{ + v1.ResourceTartVMs: 0, + })) + require.True(t, resources.CanFit(v1.Resources{ + v1.ResourceTartVMs: 1, + })) + require.True(t, resources.CanFit(v1.Resources{ + v1.ResourceTartVMs: 2, + })) + + require.False(t, resources.CanFit(v1.Resources{ + v1.ResourceTartVMs: 3, + })) + require.False(t, resources.CanFit(v1.Resources{ + v1.ResourceTartVMs: math.MaxUint64, + })) +} + +func TestResourcesMerge(t *testing.T) { + resources := v1.Resources{ + v1.ResourceTartVMs: 2, + "some-other": 5, + } + resources.Merge(v1.Resources{ + v1.ResourceTartVMs: 4, + }) + require.Equal(t, v1.Resources{ + v1.ResourceTartVMs: 4, + "some-other": 5, + }, resources) +} + +func TestResourcesMerged(t *testing.T) { + resources := v1.Resources{ + v1.ResourceTartVMs: 2, + "some-other": 5, + } + require.Equal(t, v1.Resources{ + v1.ResourceTartVMs: 4, + "some-other": 5, + }, resources.Merged(v1.Resources{ + v1.ResourceTartVMs: 4, + })) +} diff --git a/pkg/resource/v1/v1.go b/pkg/resource/v1/v1.go index c75cba4..e2b3eb0 100644 --- a/pkg/resource/v1/v1.go +++ b/pkg/resource/v1/v1.go @@ -25,6 +25,9 @@ type Worker struct { MachineID string + // Resources available on this Worker. + Resources Resources `json:"resources"` + Meta } @@ -52,6 +55,9 @@ type VM struct { // It is populated by the Controller when receiving a POST request. UID string `json:"uid"` + // Resources required by this VM. + Resources Resources `json:"resources"` + Meta }