Bump cloud.google.com/go/storage from 1.18.2 to 1.19.0 (#1903)
Bumps [cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) from 1.18.2 to 1.19.0. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/storage/v1.18.2...spanner/v1.19.0) --- updated-dependencies: - dependency-name: cloud.google.com/go/storage dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
This commit is contained in:
dependabot[bot]
GitHub
parent
025f42977f
commit
9ec7ab2d21
2
go.mod
2
go.mod
|
|
@ -11,7 +11,7 @@ replace (
|
|||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/storage v1.18.2
|
||||
cloud.google.com/go/storage v1.19.0
|
||||
github.com/Azure/azure-storage-blob-go v0.14.0
|
||||
github.com/aws/aws-sdk-go v1.42.44
|
||||
github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20211215200129-69c85dc22db6
|
||||
|
|
|
|||
8
go.sum
8
go.sum
|
|
@ -65,8 +65,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
|
|||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
|
||||
cloud.google.com/go/storage v1.18.2 h1:5NQw6tOn3eMm0oE8vTkfjau18kjL79FlMjy/CHTpmoY=
|
||||
cloud.google.com/go/storage v1.18.2/go.mod h1:AiIj7BWXyhO5gGVmYJ+S8tbkCx3yb0IMjua8Aw4naVM=
|
||||
cloud.google.com/go/storage v1.19.0 h1:XOQSnPJD8hRtZJ3VdCyK0mBZsGGImrzPAMbSWcHSe6Q=
|
||||
cloud.google.com/go/storage v1.19.0/go.mod h1:6rgiTRjOqI/Zd9YKimub5TIB4d+p3LH33V3ZE1DMuUM=
|
||||
code.gitea.io/sdk/gitea v0.12.0/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY=
|
||||
contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA=
|
||||
contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0=
|
||||
|
|
@ -1746,7 +1746,6 @@ golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
|
@ -1920,7 +1919,6 @@ google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6
|
|||
google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
|
||||
google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
|
||||
google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
|
||||
google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E=
|
||||
google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU=
|
||||
google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
|
||||
google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw=
|
||||
|
|
@ -2006,10 +2004,8 @@ google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEc
|
|||
google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/genproto v0.0.0-20211016002631-37fc39342514/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
|
|
|
|||
|
|
@ -1,5 +1,37 @@
|
|||
# Changes
|
||||
|
||||
## [1.19.0](https://www.github.com/googleapis/google-cloud-go/compare/storage/v1.18.2...storage/v1.19.0) (2022-01-25)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **storage:** add fully configurable and idempotency-aware retry strategy ([#5384](https://www.github.com/googleapis/google-cloud-go/issues/5384), [#5185](https://www.github.com/googleapis/google-cloud-go/issues/5185), [#5170](https://www.github.com/googleapis/google-cloud-go/issues/5170), [#5223](https://www.github.com/googleapis/google-cloud-go/issues/5223), [#5221](https://www.github.com/googleapis/google-cloud-go/issues/5221), [#5193](https://www.github.com/googleapis/google-cloud-go/issues/5193), [#5159](https://www.github.com/googleapis/google-cloud-go/issues/5159), [#5165](https://www.github.com/googleapis/google-cloud-go/issues/5165), [#5166](https://www.github.com/googleapis/google-cloud-go/issues/5166), [#5210](https://www.github.com/googleapis/google-cloud-go/issues/5210), [#5172](https://www.github.com/googleapis/google-cloud-go/issues/5172), [#5314](https://www.github.com/googleapis/google-cloud-go/issues/5314))
|
||||
* This release contains changes to fully align this library's retry strategy
|
||||
with best practices as described in the
|
||||
Cloud Storage [docs](https://cloud.google.com/storage/docs/retry-strategy).
|
||||
* The library will now retry only idempotent operations by default. This means
|
||||
that for certain operations, including object upload, compose, rewrite,
|
||||
update, and delete, requests will not be retried by default unless
|
||||
[idempotency conditions](https://cloud.google.com/storage/docs/retry-strategy#idempotency)
|
||||
for the request have been met.
|
||||
* The library now has methods to configure aspects of retry policy for
|
||||
API calls, including which errors are retried, the timing of the
|
||||
exponential backoff, and how idempotency is taken into account.
|
||||
* If you wish to re-enable retries for a non-idempotent request, use the
|
||||
[RetryAlways](https://pkg.go.dev/cloud.google.com/go/storage@main#RetryAlways)
|
||||
policy.
|
||||
* For full details on how to configure retries, see the
|
||||
[package docs](https://pkg.go.dev/cloud.google.com/go/storage@main#hdr-Retrying_failed_requests)
|
||||
and the
|
||||
[Cloud Storage docs](https://cloud.google.com/storage/docs/retry-strategy)
|
||||
* **storage:** GenerateSignedPostPolicyV4 can use existing creds to authenticate ([#5105](https://www.github.com/googleapis/google-cloud-go/issues/5105)) ([46489f4](https://www.github.com/googleapis/google-cloud-go/commit/46489f4c8a634068a3e7cf2fd5e5ca11b555c0a8))
|
||||
* **storage:** post policy can be signed with a fn that takes raw bytes ([#5079](https://www.github.com/googleapis/google-cloud-go/issues/5079)) ([25d1278](https://www.github.com/googleapis/google-cloud-go/commit/25d1278cab539fbfdd8563ed6b297e30d3fe555c))
|
||||
* **storage:** add rpo (turbo replication) support ([#5003](https://www.github.com/googleapis/google-cloud-go/issues/5003)) ([3bd5995](https://www.github.com/googleapis/google-cloud-go/commit/3bd59958e0c06d2655b67fcb5410668db3c52af0))
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **storage:** fix nil check in gRPC Reader ([#5376](https://www.github.com/googleapis/google-cloud-go/issues/5376)) ([5e7d722](https://www.github.com/googleapis/google-cloud-go/commit/5e7d722d18a62b28ba98169b3bdbb49401377264))
|
||||
|
||||
### [1.18.2](https://www.github.com/googleapis/google-cloud-go/compare/storage/v1.18.1...storage/v1.18.2) (2021-10-18)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
- [About Cloud Storage](https://cloud.google.com/storage/)
|
||||
- [API documentation](https://cloud.google.com/storage/docs)
|
||||
- [Go client documentation](https://pkg.go.dev/cloud.google.com/go/storage)
|
||||
- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/storage)
|
||||
- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/main/storage)
|
||||
|
||||
### Example Usage
|
||||
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ type ACLHandle struct {
|
|||
object string
|
||||
isDefault bool
|
||||
userProject string // for requester-pays buckets
|
||||
retry *retryConfig
|
||||
}
|
||||
|
||||
// Delete permanently deletes the ACL entry for the given entity.
|
||||
|
|
@ -120,12 +121,12 @@ func (a *ACLHandle) List(ctx context.Context) (rules []ACLRule, err error) {
|
|||
func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) {
|
||||
var acls *raw.ObjectAccessControls
|
||||
var err error
|
||||
err = runWithRetry(ctx, func() error {
|
||||
err = run(ctx, func() error {
|
||||
req := a.c.raw.DefaultObjectAccessControls.List(a.bucket)
|
||||
a.configureCall(ctx, req)
|
||||
acls, err = req.Do()
|
||||
return err
|
||||
})
|
||||
}, a.retry, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -135,18 +136,21 @@ func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) {
|
|||
func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error {
|
||||
req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity))
|
||||
a.configureCall(ctx, req)
|
||||
return req.Do()
|
||||
|
||||
return run(ctx, func() error {
|
||||
return req.Do()
|
||||
}, a.retry, false)
|
||||
}
|
||||
|
||||
func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) {
|
||||
var acls *raw.BucketAccessControls
|
||||
var err error
|
||||
err = runWithRetry(ctx, func() error {
|
||||
err = run(ctx, func() error {
|
||||
req := a.c.raw.BucketAccessControls.List(a.bucket)
|
||||
a.configureCall(ctx, req)
|
||||
acls, err = req.Do()
|
||||
return err
|
||||
})
|
||||
}, a.retry, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -161,25 +165,29 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol
|
|||
}
|
||||
req := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl)
|
||||
a.configureCall(ctx, req)
|
||||
_, err := req.Do()
|
||||
return err
|
||||
return run(ctx, func() error {
|
||||
_, err := req.Do()
|
||||
return err
|
||||
}, a.retry, false)
|
||||
}
|
||||
|
||||
func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error {
|
||||
req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity))
|
||||
a.configureCall(ctx, req)
|
||||
return req.Do()
|
||||
return run(ctx, func() error {
|
||||
return req.Do()
|
||||
}, a.retry, false)
|
||||
}
|
||||
|
||||
func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) {
|
||||
var acls *raw.ObjectAccessControls
|
||||
var err error
|
||||
err = runWithRetry(ctx, func() error {
|
||||
err = run(ctx, func() error {
|
||||
req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object)
|
||||
a.configureCall(ctx, req)
|
||||
acls, err = req.Do()
|
||||
return err
|
||||
})
|
||||
}, a.retry, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -204,14 +212,18 @@ func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRol
|
|||
req = a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl)
|
||||
}
|
||||
a.configureCall(ctx, req)
|
||||
_, err := req.Do()
|
||||
return err
|
||||
return run(ctx, func() error {
|
||||
_, err := req.Do()
|
||||
return err
|
||||
}, a.retry, false)
|
||||
}
|
||||
|
||||
func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error {
|
||||
req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity))
|
||||
a.configureCall(ctx, req)
|
||||
return req.Do()
|
||||
return run(ctx, func() error {
|
||||
return req.Do()
|
||||
}, a.retry, false)
|
||||
}
|
||||
|
||||
func (a *ACLHandle) configureCall(ctx context.Context, call interface{ Header() http.Header }) {
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ type BucketHandle struct {
|
|||
defaultObjectACL ACLHandle
|
||||
conds *BucketConditions
|
||||
userProject string // project for Requester Pays buckets
|
||||
retry *retryConfig
|
||||
}
|
||||
|
||||
// Bucket returns a BucketHandle, which provides operations on the named bucket.
|
||||
|
|
@ -54,18 +55,22 @@ type BucketHandle struct {
|
|||
// found at:
|
||||
// https://cloud.google.com/storage/docs/bucket-naming
|
||||
func (c *Client) Bucket(name string) *BucketHandle {
|
||||
retry := c.retry.clone()
|
||||
return &BucketHandle{
|
||||
c: c,
|
||||
name: name,
|
||||
acl: ACLHandle{
|
||||
c: c,
|
||||
bucket: name,
|
||||
retry: retry,
|
||||
},
|
||||
defaultObjectACL: ACLHandle{
|
||||
c: c,
|
||||
bucket: name,
|
||||
isDefault: true,
|
||||
retry: retry,
|
||||
},
|
||||
retry: retry,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -95,7 +100,7 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck
|
|||
if attrs != nil && attrs.PredefinedDefaultObjectACL != "" {
|
||||
req.PredefinedDefaultObjectAcl(attrs.PredefinedDefaultObjectACL)
|
||||
}
|
||||
return runWithRetry(ctx, func() error { _, err := req.Context(ctx).Do(); return err })
|
||||
return run(ctx, func() error { _, err := req.Context(ctx).Do(); return err }, b.retry, true)
|
||||
}
|
||||
|
||||
// Delete deletes the Bucket.
|
||||
|
|
@ -107,7 +112,8 @@ func (b *BucketHandle) Delete(ctx context.Context) (err error) {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return runWithRetry(ctx, func() error { return req.Context(ctx).Do() })
|
||||
|
||||
return run(ctx, func() error { return req.Context(ctx).Do() }, b.retry, true)
|
||||
}
|
||||
|
||||
func (b *BucketHandle) newDeleteCall() (*raw.BucketsDeleteCall, error) {
|
||||
|
|
@ -144,6 +150,7 @@ func (b *BucketHandle) DefaultObjectACL() *ACLHandle {
|
|||
// for valid object names can be found at:
|
||||
// https://cloud.google.com/storage/docs/naming-objects
|
||||
func (b *BucketHandle) Object(name string) *ObjectHandle {
|
||||
retry := b.retry.clone()
|
||||
return &ObjectHandle{
|
||||
c: b.c,
|
||||
bucket: b.name,
|
||||
|
|
@ -153,9 +160,11 @@ func (b *BucketHandle) Object(name string) *ObjectHandle {
|
|||
bucket: b.name,
|
||||
object: name,
|
||||
userProject: b.userProject,
|
||||
retry: retry,
|
||||
},
|
||||
gen: -1,
|
||||
userProject: b.userProject,
|
||||
retry: retry,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -169,10 +178,10 @@ func (b *BucketHandle) Attrs(ctx context.Context) (attrs *BucketAttrs, err error
|
|||
return nil, err
|
||||
}
|
||||
var resp *raw.Bucket
|
||||
err = runWithRetry(ctx, func() error {
|
||||
err = run(ctx, func() error {
|
||||
resp, err = req.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
}, b.retry, true)
|
||||
var e *googleapi.Error
|
||||
if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound {
|
||||
return nil, ErrBucketNotExist
|
||||
|
|
@ -210,12 +219,20 @@ func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (
|
|||
if uattrs.PredefinedDefaultObjectACL != "" {
|
||||
req.PredefinedDefaultObjectAcl(uattrs.PredefinedDefaultObjectACL)
|
||||
}
|
||||
// TODO(jba): retry iff metagen is set?
|
||||
rb, err := req.Context(ctx).Do()
|
||||
if err != nil {
|
||||
|
||||
isIdempotent := b.conds != nil && b.conds.MetagenerationMatch != 0
|
||||
|
||||
var rawBucket *raw.Bucket
|
||||
call := func() error {
|
||||
rb, err := req.Context(ctx).Do()
|
||||
rawBucket = rb
|
||||
return err
|
||||
}
|
||||
|
||||
if err := run(ctx, call, b.retry, isIdempotent); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newBucket(rb)
|
||||
return newBucket(rawBucket)
|
||||
}
|
||||
|
||||
func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPatchCall, error) {
|
||||
|
|
@ -282,8 +299,54 @@ func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string,
|
|||
return SignedURL(b.name, object, newopts)
|
||||
}
|
||||
|
||||
// TODO: Add a similar wrapper for GenerateSignedPostPolicyV4 allowing users to
|
||||
// omit PrivateKey/SignBytes
|
||||
// GenerateSignedPostPolicyV4 generates a PostPolicyV4 value from bucket, object and opts.
|
||||
// The generated URL and fields will then allow an unauthenticated client to perform multipart uploads.
|
||||
//
|
||||
// This method only requires the Expires field in the specified PostPolicyV4Options
|
||||
// to be non-nil. If not provided, it attempts to fill the GoogleAccessID and PrivateKey
|
||||
// from the GOOGLE_APPLICATION_CREDENTIALS environment variable.
|
||||
// If you are authenticating with a custom HTTP client, Service Account based
|
||||
// auto-detection will be hindered.
|
||||
//
|
||||
// If no private key is found, it attempts to use the GoogleAccessID to sign the URL.
|
||||
// This requires the IAM Service Account Credentials API to be enabled
|
||||
// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview)
|
||||
// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account.
|
||||
// If you do not want these fields set for you, you may pass them in through opts or use
|
||||
// GenerateSignedPostPolicyV4(bucket, name string, opts *PostPolicyV4Options) instead.
|
||||
func (b *BucketHandle) GenerateSignedPostPolicyV4(object string, opts *PostPolicyV4Options) (*PostPolicyV4, error) {
|
||||
if opts.GoogleAccessID != "" && (opts.SignRawBytes != nil || opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
|
||||
return GenerateSignedPostPolicyV4(b.name, object, opts)
|
||||
}
|
||||
// Make a copy of opts so we don't modify the pointer parameter.
|
||||
newopts := opts.clone()
|
||||
|
||||
if newopts.GoogleAccessID == "" {
|
||||
id, err := b.detectDefaultGoogleAccessID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
newopts.GoogleAccessID = id
|
||||
}
|
||||
if newopts.SignBytes == nil && newopts.SignRawBytes == nil && len(newopts.PrivateKey) == 0 {
|
||||
if b.c.creds != nil && len(b.c.creds.JSON) > 0 {
|
||||
var sa struct {
|
||||
PrivateKey string `json:"private_key"`
|
||||
}
|
||||
err := json.Unmarshal(b.c.creds.JSON, &sa)
|
||||
if err == nil && sa.PrivateKey != "" {
|
||||
newopts.PrivateKey = []byte(sa.PrivateKey)
|
||||
}
|
||||
}
|
||||
|
||||
// Don't error out if we can't unmarshal the private key from the client,
|
||||
// fallback to the default sign function for the service account.
|
||||
if len(newopts.PrivateKey) == 0 {
|
||||
newopts.SignRawBytes = b.defaultSignBytesFunc(newopts.GoogleAccessID)
|
||||
}
|
||||
}
|
||||
return GenerateSignedPostPolicyV4(b.name, object, newopts)
|
||||
}
|
||||
|
||||
func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) {
|
||||
returnErr := errors.New("no credentials found on client and not on GCE (Google Compute Engine)")
|
||||
|
|
@ -461,6 +524,12 @@ type BucketAttrs struct {
|
|||
// The project number of the project the bucket belongs to.
|
||||
// This field is read-only.
|
||||
ProjectNumber uint64
|
||||
|
||||
// RPO configures the Recovery Point Objective (RPO) policy of the bucket.
|
||||
// Set to RPOAsyncTurbo to turn on Turbo Replication for a bucket.
|
||||
// See https://cloud.google.com/storage/docs/managing-turbo-replication for
|
||||
// more information.
|
||||
RPO RPO
|
||||
}
|
||||
|
||||
// BucketPolicyOnly is an alias for UniformBucketLevelAccess.
|
||||
|
|
@ -728,6 +797,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
|
|||
Etag: b.Etag,
|
||||
LocationType: b.LocationType,
|
||||
ProjectNumber: b.ProjectNumber,
|
||||
RPO: toRPO(b),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -780,6 +850,7 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket {
|
|||
Logging: b.Logging.toRawBucketLogging(),
|
||||
Website: b.Website.toRawBucketWebsite(),
|
||||
IamConfiguration: bktIAM,
|
||||
Rpo: b.RPO.String(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -889,6 +960,12 @@ type BucketAttrsToUpdate struct {
|
|||
// See https://cloud.google.com/storage/docs/json_api/v1/buckets/patch.
|
||||
PredefinedDefaultObjectACL string
|
||||
|
||||
// RPO configures the Recovery Point Objective (RPO) policy of the bucket.
|
||||
// Set to RPOAsyncTurbo to turn on Turbo Replication for a bucket.
|
||||
// See https://cloud.google.com/storage/docs/managing-turbo-replication for
|
||||
// more information.
|
||||
RPO RPO
|
||||
|
||||
setLabels map[string]string
|
||||
deleteLabels map[string]bool
|
||||
}
|
||||
|
|
@ -1001,7 +1078,10 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
|
|||
rb.DefaultObjectAcl = nil
|
||||
rb.ForceSendFields = append(rb.ForceSendFields, "DefaultObjectAcl")
|
||||
}
|
||||
|
||||
rb.StorageClass = ua.StorageClass
|
||||
rb.Rpo = ua.RPO.String()
|
||||
|
||||
if ua.setLabels != nil || ua.deleteLabels != nil {
|
||||
rb.Labels = map[string]string{}
|
||||
for k, v := range ua.setLabels {
|
||||
|
|
@ -1081,10 +1161,10 @@ func (b *BucketHandle) LockRetentionPolicy(ctx context.Context) error {
|
|||
metageneration = b.conds.MetagenerationMatch
|
||||
}
|
||||
req := b.c.raw.Buckets.LockRetentionPolicy(b.name, metageneration)
|
||||
return runWithRetry(ctx, func() error {
|
||||
return run(ctx, func() error {
|
||||
_, err := req.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
}, b.retry, true)
|
||||
}
|
||||
|
||||
// applyBucketConds modifies the provided call using the conditions in conds.
|
||||
|
|
@ -1347,6 +1427,20 @@ func toPublicAccessPrevention(b *raw.BucketIamConfiguration) PublicAccessPrevent
|
|||
}
|
||||
}
|
||||
|
||||
func toRPO(b *raw.Bucket) RPO {
|
||||
if b == nil {
|
||||
return RPOUnknown
|
||||
}
|
||||
switch b.Rpo {
|
||||
case rpoDefault:
|
||||
return RPODefault
|
||||
case rpoAsyncTurbo:
|
||||
return RPOAsyncTurbo
|
||||
default:
|
||||
return RPOUnknown
|
||||
}
|
||||
}
|
||||
|
||||
// Objects returns an iterator over the objects in the bucket that match the
|
||||
// Query q. If q is nil, no filtering is done. Objects will be iterated over
|
||||
// lexicographically by name.
|
||||
|
|
@ -1367,6 +1461,33 @@ func (b *BucketHandle) Objects(ctx context.Context, q *Query) *ObjectIterator {
|
|||
return it
|
||||
}
|
||||
|
||||
// Retryer returns a bucket handle that is configured with custom retry
|
||||
// behavior as specified by the options that are passed to it. All operations
|
||||
// on the new handle will use the customized retry configuration.
|
||||
// Retry options set on a object handle will take precedence over options set on
|
||||
// the bucket handle.
|
||||
// These retry options will merge with the client's retry configuration (if set)
|
||||
// for the returned handle. Options passed into this method will take precedence
|
||||
// over retry options on the client. Note that you must explicitly pass in each
|
||||
// option you want to override.
|
||||
func (b *BucketHandle) Retryer(opts ...RetryOption) *BucketHandle {
|
||||
b2 := *b
|
||||
var retry *retryConfig
|
||||
if b.retry != nil {
|
||||
// merge the options with the existing retry
|
||||
retry = b.retry
|
||||
} else {
|
||||
retry = &retryConfig{}
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt.apply(retry)
|
||||
}
|
||||
b2.retry = retry
|
||||
b2.acl.retry = retry
|
||||
b2.defaultObjectACL.retry = retry
|
||||
return &b2
|
||||
}
|
||||
|
||||
// An ObjectIterator is an iterator over ObjectAttrs.
|
||||
//
|
||||
// Note: This iterator is not safe for concurrent operations without explicit synchronization.
|
||||
|
|
@ -1434,10 +1555,10 @@ func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error)
|
|||
}
|
||||
var resp *raw.Objects
|
||||
var err error
|
||||
err = runWithRetry(it.ctx, func() error {
|
||||
err = run(it.ctx, func() error {
|
||||
resp, err = req.Context(it.ctx).Do()
|
||||
return err
|
||||
})
|
||||
}, it.bucket.retry, true)
|
||||
if err != nil {
|
||||
var e *googleapi.Error
|
||||
if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound {
|
||||
|
|
@ -1518,10 +1639,10 @@ func (it *BucketIterator) fetch(pageSize int, pageToken string) (token string, e
|
|||
req.MaxResults(int64(pageSize))
|
||||
}
|
||||
var resp *raw.Buckets
|
||||
err = runWithRetry(it.ctx, func() error {
|
||||
err = run(it.ctx, func() error {
|
||||
resp, err = req.Context(it.ctx).Do()
|
||||
return err
|
||||
})
|
||||
}, it.client.retry, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
@ -1534,3 +1655,39 @@ func (it *BucketIterator) fetch(pageSize int, pageToken string) (token string, e
|
|||
}
|
||||
return resp.NextPageToken, nil
|
||||
}
|
||||
|
||||
// RPO (Recovery Point Objective) configures the turbo replication feature. See
|
||||
// https://cloud.google.com/storage/docs/managing-turbo-replication for more information.
|
||||
type RPO int
|
||||
|
||||
const (
|
||||
// RPOUnknown is a zero value. It may be returned from bucket.Attrs() if RPO
|
||||
// is not present in the bucket metadata, that is, the bucket is not dual-region.
|
||||
// This value is also used if the RPO field is not set in a call to GCS.
|
||||
RPOUnknown RPO = iota
|
||||
|
||||
// RPODefault represents default replication. It is used to reset RPO on an
|
||||
// existing bucket that has this field set to RPOAsyncTurbo. Otherwise it
|
||||
// is equivalent to RPOUnknown, and is always ignored. This value is valid
|
||||
// for dual- or multi-region buckets.
|
||||
RPODefault
|
||||
|
||||
// RPOAsyncTurbo represents turbo replication and is used to enable Turbo
|
||||
// Replication on a bucket. This value is only valid for dual-region buckets.
|
||||
RPOAsyncTurbo
|
||||
|
||||
rpoUnknown string = ""
|
||||
rpoDefault = "DEFAULT"
|
||||
rpoAsyncTurbo = "ASYNC_TURBO"
|
||||
)
|
||||
|
||||
func (rpo RPO) String() string {
|
||||
switch rpo {
|
||||
case RPODefault:
|
||||
return rpoDefault
|
||||
case RPOAsyncTurbo:
|
||||
return rpoAsyncTurbo
|
||||
default:
|
||||
return rpoUnknown
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -138,8 +138,11 @@ func (c *Copier) callRewrite(ctx context.Context, rawObj *raw.Object) (*raw.Rewr
|
|||
var res *raw.RewriteResponse
|
||||
var err error
|
||||
setClientHeader(call.Header())
|
||||
err = runWithRetry(ctx, func() error { res, err = call.Do(); return err })
|
||||
if err != nil {
|
||||
|
||||
retryCall := func() error { res, err = call.Do(); return err }
|
||||
isIdempotent := c.dst.conds != nil && (c.dst.conds.GenerationMatch != 0 || c.dst.conds.DoesNotExist)
|
||||
|
||||
if err := run(ctx, retryCall, c.dst.retry, isIdempotent); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.RewriteToken = res.RewriteToken
|
||||
|
|
@ -230,8 +233,11 @@ func (c *Composer) Run(ctx context.Context) (attrs *ObjectAttrs, err error) {
|
|||
}
|
||||
var obj *raw.Object
|
||||
setClientHeader(call.Header())
|
||||
err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err })
|
||||
if err != nil {
|
||||
|
||||
retryCall := func() error { obj, err = call.Do(); return err }
|
||||
isIdempotent := c.dst.conds != nil && (c.dst.conds.GenerationMatch != 0 || c.dst.conds.DoesNotExist)
|
||||
|
||||
if err := run(ctx, retryCall, c.dst.retry, isIdempotent); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newObject(obj), nil
|
||||
|
|
|
|||
|
|
@ -19,15 +19,9 @@ Google Cloud Storage stores data in named objects, which are grouped into bucket
|
|||
More information about Google Cloud Storage is available at
|
||||
https://cloud.google.com/storage/docs.
|
||||
|
||||
See https://godoc.org/cloud.google.com/go for authentication, timeouts,
|
||||
See https://pkg.go.dev/cloud.google.com/go for authentication, timeouts,
|
||||
connection pooling and similar aspects of this package.
|
||||
|
||||
All of the methods of this package use exponential backoff to retry calls that fail
|
||||
with certain errors, as described in
|
||||
https://cloud.google.com/storage/docs/exponential-backoff. Retrying continues
|
||||
indefinitely unless the controlling context is canceled or the client is closed. See
|
||||
context.WithTimeout and context.WithCancel.
|
||||
|
||||
|
||||
Creating a Client
|
||||
|
||||
|
|
@ -246,12 +240,52 @@ as the documentation of GenerateSignedPostPolicyV4.
|
|||
|
||||
Errors
|
||||
|
||||
Errors returned by this client are often of the type [`googleapi.Error`](https://godoc.org/google.golang.org/api/googleapi#Error).
|
||||
These errors can be introspected for more information by using `errors.As` with the richer `googleapi.Error` type. For example:
|
||||
Errors returned by this client are often of the type googleapi.Error.
|
||||
These errors can be introspected for more information by using errors.As
|
||||
with the richer googleapi.Error type. For example:
|
||||
|
||||
var e *googleapi.Error
|
||||
if ok := errors.As(err, &e); ok {
|
||||
if e.Code == 409 { ... }
|
||||
}
|
||||
|
||||
See https://pkg.go.dev/google.golang.org/api/googleapi#Error for more information.
|
||||
|
||||
Retrying failed requests
|
||||
|
||||
Methods in this package may retry calls that fail with transient errors.
|
||||
Retrying continues indefinitely unless the controlling context is canceled, the
|
||||
client is closed, or a non-transient error is received. To stop retries from
|
||||
continuing, use context timeouts or cancellation.
|
||||
|
||||
The retry strategy in this library follows best practices for Cloud Storage. By
|
||||
default, operations are retried only if they are idempotent, and exponential
|
||||
backoff with jitter is employed. In addition, errors are only retried if they
|
||||
are defined as transient by the service. See
|
||||
https://cloud.google.com/storage/docs/retry-strategy for more information.
|
||||
|
||||
Users can configure non-default retry behavior for a single library call (using
|
||||
BucketHandle.Retryer and ObjectHandle.Retryer) or for all calls made by a
|
||||
client (using Client.SetRetry). For example:
|
||||
|
||||
o := client.Bucket(bucket).Object(object).Retryer(
|
||||
// Use WithBackoff to change the timing of the exponential backoff.
|
||||
storage.WithBackoff(gax.Backoff{
|
||||
Initial: 2 * time.Second,
|
||||
}),
|
||||
// Use WithPolicy to configure the idempotency policy. RetryAlways will
|
||||
// retry the operation even if it is non-idempotent.
|
||||
storage.WithPolicy(storage.RetryAlways),
|
||||
)
|
||||
|
||||
// Use a context timeout to set an overall deadline on the call, including all
|
||||
// potential retries.
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Delete an object using the specified strategy and timeout.
|
||||
if err := o.Delete(ctx); err != nil {
|
||||
// Handle err.
|
||||
}
|
||||
*/
|
||||
package storage // import "cloud.google.com/go/storage"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,71 @@
|
|||
#!/bin/bash
|
||||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License..
|
||||
|
||||
# Fail on any error
|
||||
set -eo pipefail
|
||||
|
||||
# Display commands being run
|
||||
set -x
|
||||
|
||||
# Only run on Go 1.17+
|
||||
min_minor_ver=17
|
||||
|
||||
v=`go version | { read _ _ v _; echo ${v#go}; }`
|
||||
comps=(${v//./ })
|
||||
minor_ver=${comps[1]}
|
||||
|
||||
if [ "$minor_ver" -lt "$min_minor_ver" ]; then
|
||||
echo minor version $minor_ver, skipping
|
||||
exit 0
|
||||
fi
|
||||
|
||||
export STORAGE_EMULATOR_HOST="http://localhost:9000"
|
||||
|
||||
DEFAULT_IMAGE_NAME='gcr.io/cloud-devrel-public-resources/storage-testbench'
|
||||
DEFAULT_IMAGE_TAG='latest'
|
||||
DOCKER_IMAGE=${DEFAULT_IMAGE_NAME}:${DEFAULT_IMAGE_TAG}
|
||||
CONTAINER_NAME=storage_testbench
|
||||
|
||||
# Get the docker image for the testbench
|
||||
docker pull $DOCKER_IMAGE
|
||||
|
||||
# Start the testbench
|
||||
# Note: --net=host makes the container bind directly to the Docker host’s network,
|
||||
# with no network isolation. If we were to use port-mapping instead, reset connection errors
|
||||
# would be captured differently and cause unexpected test behaviour.
|
||||
# The host networking driver works only on Linux hosts.
|
||||
# See more about using host networking: https://docs.docker.com/network/host/
|
||||
docker run --name $CONTAINER_NAME --rm --net=host $DOCKER_IMAGE &
|
||||
echo "Running the Cloud Storage testbench: $STORAGE_EMULATOR_HOST"
|
||||
|
||||
# Check that the server is running - retry several times to allow for start-up time
|
||||
response=$(curl -w "%{http_code}\n" $STORAGE_EMULATOR_HOST --retry-connrefused --retry 5 -o /dev/null)
|
||||
|
||||
if [[ $response != 200 ]]
|
||||
then
|
||||
echo "Testbench server did not start correctly"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Stop the testbench & cleanup environment variables
|
||||
function cleanup() {
|
||||
echo "Cleanup testbench"
|
||||
docker stop $CONTAINER_NAME
|
||||
unset STORAGE_EMULATOR_HOST;
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Run tests
|
||||
go test -v -timeout 10m ./ -run="TestRetryConformance" -short 2>&1 | tee -a sponge_log.log
|
||||
|
|
@ -89,8 +89,8 @@ type HMACKey struct {
|
|||
type HMACKeyHandle struct {
|
||||
projectID string
|
||||
accessID string
|
||||
|
||||
raw *raw.ProjectsHmacKeysService
|
||||
retry *retryConfig
|
||||
raw *raw.ProjectsHmacKeysService
|
||||
}
|
||||
|
||||
// HMACKeyHandle creates a handle that will be used for HMACKey operations.
|
||||
|
|
@ -100,6 +100,7 @@ func (c *Client) HMACKeyHandle(projectID, accessID string) *HMACKeyHandle {
|
|||
return &HMACKeyHandle{
|
||||
projectID: projectID,
|
||||
accessID: accessID,
|
||||
retry: c.retry,
|
||||
raw: raw.NewProjectsHmacKeysService(c.raw),
|
||||
}
|
||||
}
|
||||
|
|
@ -126,10 +127,10 @@ func (hkh *HMACKeyHandle) Get(ctx context.Context, opts ...HMACKeyOption) (*HMAC
|
|||
|
||||
var metadata *raw.HmacKeyMetadata
|
||||
var err error
|
||||
err = runWithRetry(ctx, func() error {
|
||||
err = run(ctx, func() error {
|
||||
metadata, err = call.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
}, hkh.retry, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -156,9 +157,9 @@ func (hkh *HMACKeyHandle) Delete(ctx context.Context, opts ...HMACKeyOption) err
|
|||
}
|
||||
setClientHeader(delCall.Header())
|
||||
|
||||
return runWithRetry(ctx, func() error {
|
||||
return run(ctx, func() error {
|
||||
return delCall.Context(ctx).Do()
|
||||
})
|
||||
}, hkh.retry, true)
|
||||
}
|
||||
|
||||
func pbHmacKeyToHMACKey(pb *raw.HmacKey, updatedTimeCanBeNil bool) (*HMACKey, error) {
|
||||
|
|
@ -214,8 +215,13 @@ func (c *Client) CreateHMACKey(ctx context.Context, projectID, serviceAccountEma
|
|||
|
||||
setClientHeader(call.Header())
|
||||
|
||||
hkPb, err := call.Context(ctx).Do()
|
||||
if err != nil {
|
||||
var hkPb *raw.HmacKey
|
||||
|
||||
if err := run(ctx, func() error {
|
||||
h, err := call.Context(ctx).Do()
|
||||
hkPb = h
|
||||
return err
|
||||
}, c.retry, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
|
@ -257,10 +263,11 @@ func (h *HMACKeyHandle) Update(ctx context.Context, au HMACKeyAttrsToUpdate, opt
|
|||
|
||||
var metadata *raw.HmacKeyMetadata
|
||||
var err error
|
||||
err = runWithRetry(ctx, func() error {
|
||||
isIdempotent := len(au.Etag) > 0
|
||||
err = run(ctx, func() error {
|
||||
metadata, err = call.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
}, h.retry, isIdempotent)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -285,6 +292,7 @@ type HMACKeysIterator struct {
|
|||
nextFunc func() error
|
||||
index int
|
||||
desc hmacKeyDesc
|
||||
retry *retryConfig
|
||||
}
|
||||
|
||||
// ListHMACKeys returns an iterator for listing HMACKeys.
|
||||
|
|
@ -297,6 +305,7 @@ func (c *Client) ListHMACKeys(ctx context.Context, projectID string, opts ...HMA
|
|||
ctx: ctx,
|
||||
raw: raw.NewProjectsHmacKeysService(c.raw),
|
||||
projectID: projectID,
|
||||
retry: c.retry,
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
|
|
@ -361,10 +370,10 @@ func (it *HMACKeysIterator) fetch(pageSize int, pageToken string) (token string,
|
|||
|
||||
ctx := it.ctx
|
||||
var resp *raw.HmacKeysMetadata
|
||||
err = runWithRetry(it.ctx, func() error {
|
||||
err = run(it.ctx, func() error {
|
||||
resp, err = call.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
}, it.retry, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ func (b *BucketHandle) IAM() *iam.Handle {
|
|||
return iam.InternalNewHandleClient(&iamClient{
|
||||
raw: b.c.raw,
|
||||
userProject: b.userProject,
|
||||
retry: b.retry,
|
||||
}, b.name)
|
||||
}
|
||||
|
||||
|
|
@ -36,6 +37,7 @@ func (b *BucketHandle) IAM() *iam.Handle {
|
|||
type iamClient struct {
|
||||
raw *raw.Service
|
||||
userProject string
|
||||
retry *retryConfig
|
||||
}
|
||||
|
||||
func (c *iamClient) Get(ctx context.Context, resource string) (p *iampb.Policy, err error) {
|
||||
|
|
@ -52,10 +54,10 @@ func (c *iamClient) GetWithVersion(ctx context.Context, resource string, request
|
|||
call.UserProject(c.userProject)
|
||||
}
|
||||
var rp *raw.Policy
|
||||
err = runWithRetry(ctx, func() error {
|
||||
err = run(ctx, func() error {
|
||||
rp, err = call.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
}, c.retry, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -72,10 +74,11 @@ func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) (
|
|||
if c.userProject != "" {
|
||||
call.UserProject(c.userProject)
|
||||
}
|
||||
return runWithRetry(ctx, func() error {
|
||||
isIdempotent := len(p.Etag) > 0
|
||||
return run(ctx, func() error {
|
||||
_, err := call.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
}, c.retry, isIdempotent)
|
||||
}
|
||||
|
||||
func (c *iamClient) Test(ctx context.Context, resource string, perms []string) (permissions []string, err error) {
|
||||
|
|
@ -88,10 +91,10 @@ func (c *iamClient) Test(ctx context.Context, resource string, perms []string) (
|
|||
call.UserProject(c.userProject)
|
||||
}
|
||||
var res *raw.TestIamPermissionsResponse
|
||||
err = runWithRetry(ctx, func() error {
|
||||
err = run(ctx, func() error {
|
||||
res, err = call.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
}, c.retry, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright 2021 Google LLC
|
||||
// Copyright 2022 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
|
@ -39,6 +39,22 @@
|
|||
//
|
||||
// The following is an example of making an API call with the newly created client.
|
||||
//
|
||||
// ctx := context.Background()
|
||||
// c, err := storage.NewClient(ctx)
|
||||
// if err != nil {
|
||||
// // TODO: Handle error.
|
||||
// }
|
||||
// defer c.Close()
|
||||
//
|
||||
// req := &storagepb.DeleteBucketRequest{
|
||||
// // TODO: Fill request struct fields.
|
||||
// // See https://pkg.go.dev/google.golang.org/genproto/googleapis/storage/v2#DeleteBucketRequest.
|
||||
// }
|
||||
// err = c.DeleteBucket(ctx, req)
|
||||
// if err != nil {
|
||||
// // TODO: Handle error.
|
||||
// }
|
||||
//
|
||||
// Use of Context
|
||||
//
|
||||
// The ctx passed to NewClient is used for authentication requests and
|
||||
|
|
@ -68,7 +84,7 @@ import (
|
|||
type clientHookParams struct{}
|
||||
type clientHook func(context.Context, clientHookParams) ([]option.ClientOption, error)
|
||||
|
||||
const versionClient = "20211015"
|
||||
const versionClient = "20220114"
|
||||
|
||||
func insertMetadata(ctx context.Context, mds ...metadata.MD) context.Context {
|
||||
out, _ := metadata.FromOutgoingContext(ctx)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,101 @@
|
|||
"grpc": {
|
||||
"libraryClient": "Client",
|
||||
"rpcs": {
|
||||
"ComposeObject": {
|
||||
"methods": [
|
||||
"ComposeObject"
|
||||
]
|
||||
},
|
||||
"CreateBucket": {
|
||||
"methods": [
|
||||
"CreateBucket"
|
||||
]
|
||||
},
|
||||
"CreateHmacKey": {
|
||||
"methods": [
|
||||
"CreateHmacKey"
|
||||
]
|
||||
},
|
||||
"CreateNotification": {
|
||||
"methods": [
|
||||
"CreateNotification"
|
||||
]
|
||||
},
|
||||
"DeleteBucket": {
|
||||
"methods": [
|
||||
"DeleteBucket"
|
||||
]
|
||||
},
|
||||
"DeleteHmacKey": {
|
||||
"methods": [
|
||||
"DeleteHmacKey"
|
||||
]
|
||||
},
|
||||
"DeleteNotification": {
|
||||
"methods": [
|
||||
"DeleteNotification"
|
||||
]
|
||||
},
|
||||
"DeleteObject": {
|
||||
"methods": [
|
||||
"DeleteObject"
|
||||
]
|
||||
},
|
||||
"GetBucket": {
|
||||
"methods": [
|
||||
"GetBucket"
|
||||
]
|
||||
},
|
||||
"GetHmacKey": {
|
||||
"methods": [
|
||||
"GetHmacKey"
|
||||
]
|
||||
},
|
||||
"GetIamPolicy": {
|
||||
"methods": [
|
||||
"GetIamPolicy"
|
||||
]
|
||||
},
|
||||
"GetNotification": {
|
||||
"methods": [
|
||||
"GetNotification"
|
||||
]
|
||||
},
|
||||
"GetObject": {
|
||||
"methods": [
|
||||
"GetObject"
|
||||
]
|
||||
},
|
||||
"GetServiceAccount": {
|
||||
"methods": [
|
||||
"GetServiceAccount"
|
||||
]
|
||||
},
|
||||
"ListBuckets": {
|
||||
"methods": [
|
||||
"ListBuckets"
|
||||
]
|
||||
},
|
||||
"ListHmacKeys": {
|
||||
"methods": [
|
||||
"ListHmacKeys"
|
||||
]
|
||||
},
|
||||
"ListNotifications": {
|
||||
"methods": [
|
||||
"ListNotifications"
|
||||
]
|
||||
},
|
||||
"ListObjects": {
|
||||
"methods": [
|
||||
"ListObjects"
|
||||
]
|
||||
},
|
||||
"LockBucketRetentionPolicy": {
|
||||
"methods": [
|
||||
"LockBucketRetentionPolicy"
|
||||
]
|
||||
},
|
||||
"QueryWriteStatus": {
|
||||
"methods": [
|
||||
"QueryWriteStatus"
|
||||
|
|
@ -20,11 +115,41 @@
|
|||
"ReadObject"
|
||||
]
|
||||
},
|
||||
"RewriteObject": {
|
||||
"methods": [
|
||||
"RewriteObject"
|
||||
]
|
||||
},
|
||||
"SetIamPolicy": {
|
||||
"methods": [
|
||||
"SetIamPolicy"
|
||||
]
|
||||
},
|
||||
"StartResumableWrite": {
|
||||
"methods": [
|
||||
"StartResumableWrite"
|
||||
]
|
||||
},
|
||||
"TestIamPermissions": {
|
||||
"methods": [
|
||||
"TestIamPermissions"
|
||||
]
|
||||
},
|
||||
"UpdateBucket": {
|
||||
"methods": [
|
||||
"UpdateBucket"
|
||||
]
|
||||
},
|
||||
"UpdateHmacKey": {
|
||||
"methods": [
|
||||
"UpdateHmacKey"
|
||||
]
|
||||
},
|
||||
"UpdateObject": {
|
||||
"methods": [
|
||||
"UpdateObject"
|
||||
]
|
||||
},
|
||||
"WriteObject": {
|
||||
"methods": [
|
||||
"WriteObject"
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -17,42 +17,66 @@ package storage
|
|||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"cloud.google.com/go/internal"
|
||||
gax "github.com/googleapis/gax-go/v2"
|
||||
"golang.org/x/xerrors"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// runWithRetry calls the function until it returns nil or a non-retryable error, or
|
||||
// the context is done.
|
||||
func runWithRetry(ctx context.Context, call func() error) error {
|
||||
return internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
|
||||
var defaultRetry *retryConfig = &retryConfig{}
|
||||
|
||||
// run determines whether a retry is necessary based on the config and
|
||||
// idempotency information. It then calls the function with or without retries
|
||||
// as appropriate, using the configured settings.
|
||||
func run(ctx context.Context, call func() error, retry *retryConfig, isIdempotent bool) error {
|
||||
if retry == nil {
|
||||
retry = defaultRetry
|
||||
}
|
||||
if (retry.policy == RetryIdempotent && !isIdempotent) || retry.policy == RetryNever {
|
||||
return call()
|
||||
}
|
||||
bo := gax.Backoff{}
|
||||
if retry.backoff != nil {
|
||||
bo.Multiplier = retry.backoff.Multiplier
|
||||
bo.Initial = retry.backoff.Initial
|
||||
bo.Max = retry.backoff.Max
|
||||
}
|
||||
var errorFunc func(err error) bool = shouldRetry
|
||||
if retry.shouldRetry != nil {
|
||||
errorFunc = retry.shouldRetry
|
||||
}
|
||||
return internal.Retry(ctx, bo, func() (stop bool, err error) {
|
||||
err = call()
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if shouldRetry(err) {
|
||||
return false, err
|
||||
}
|
||||
return true, err
|
||||
return !errorFunc(err), err
|
||||
})
|
||||
}
|
||||
|
||||
func shouldRetry(err error) bool {
|
||||
if err == io.ErrUnexpectedEOF {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if xerrors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return true
|
||||
}
|
||||
|
||||
switch e := err.(type) {
|
||||
case *net.OpError:
|
||||
if strings.Contains(e.Error(), "use of closed network connection") {
|
||||
// TODO: check against net.ErrClosed (go 1.16+) instead of string
|
||||
return true
|
||||
}
|
||||
case *googleapi.Error:
|
||||
// Retry on 429 and 5xx, according to
|
||||
// Retry on 408, 429, and 5xx, according to
|
||||
// https://cloud.google.com/storage/docs/exponential-backoff.
|
||||
return e.Code == 429 || (e.Code >= 500 && e.Code < 600)
|
||||
return e.Code == 408 || e.Code == 429 || (e.Code >= 500 && e.Code < 600)
|
||||
case *url.Error:
|
||||
// Retry socket-level errors ECONNREFUSED and ENETUNREACH (from syscall).
|
||||
// Retry socket-level errors ECONNREFUSED and ECONNRESET (from syscall).
|
||||
// Unfortunately the error type is unexported, so we resort to string
|
||||
// matching.
|
||||
retriable := []string{"connection refused", "connection reset"}
|
||||
|
|
|
|||
|
|
@ -137,7 +137,12 @@ func (b *BucketHandle) AddNotification(ctx context.Context, n *Notification) (re
|
|||
if b.userProject != "" {
|
||||
call.UserProject(b.userProject)
|
||||
}
|
||||
rn, err := call.Context(ctx).Do()
|
||||
|
||||
var rn *raw.Notification
|
||||
err = run(ctx, func() error {
|
||||
rn, err = call.Context(ctx).Do()
|
||||
return err
|
||||
}, b.retry, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -156,10 +161,10 @@ func (b *BucketHandle) Notifications(ctx context.Context) (n map[string]*Notific
|
|||
call.UserProject(b.userProject)
|
||||
}
|
||||
var res *raw.Notifications
|
||||
err = runWithRetry(ctx, func() error {
|
||||
err = run(ctx, func() error {
|
||||
res, err = call.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
}, b.retry, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -184,7 +189,7 @@ func (b *BucketHandle) DeleteNotification(ctx context.Context, id string) (err e
|
|||
if b.userProject != "" {
|
||||
call.UserProject(b.userProject)
|
||||
}
|
||||
return runWithRetry(ctx, func() error {
|
||||
return run(ctx, func() error {
|
||||
return call.Context(ctx).Do()
|
||||
})
|
||||
}, b.retry, true)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,22 +52,38 @@ type PostPolicyV4Options struct {
|
|||
// Exactly one of PrivateKey or SignBytes must be non-nil.
|
||||
PrivateKey []byte
|
||||
|
||||
// SignBytes is a function for implementing custom signing. For example, if
|
||||
// SignBytes is a function for implementing custom signing.
|
||||
//
|
||||
// Deprecated: Use SignRawBytes. If both SignBytes and SignRawBytes are defined,
|
||||
// SignBytes will be ignored.
|
||||
// This SignBytes function expects the bytes it receives to be hashed, while
|
||||
// SignRawBytes accepts the raw bytes without hashing, allowing more flexibility.
|
||||
// Add the following to the top of your signing function to hash the bytes
|
||||
// to use SignRawBytes instead:
|
||||
// shaSum := sha256.Sum256(bytes)
|
||||
// bytes = shaSum[:]
|
||||
//
|
||||
SignBytes func(hashBytes []byte) (signature []byte, err error)
|
||||
|
||||
// SignRawBytes is a function for implementing custom signing. For example, if
|
||||
// your application is running on Google App Engine, you can use
|
||||
// appengine's internal signing function:
|
||||
// ctx := appengine.NewContext(request)
|
||||
// acc, _ := appengine.ServiceAccount(ctx)
|
||||
// url, err := SignedURL("bucket", "object", &SignedURLOptions{
|
||||
// GoogleAccessID: acc,
|
||||
// SignBytes: func(b []byte) ([]byte, error) {
|
||||
// _, signedBytes, err := appengine.SignBytes(ctx, b)
|
||||
// return signedBytes, err
|
||||
// },
|
||||
// // etc.
|
||||
// })
|
||||
// ctx := appengine.NewContext(request)
|
||||
// acc, _ := appengine.ServiceAccount(ctx)
|
||||
// &PostPolicyV4Options{
|
||||
// GoogleAccessID: acc,
|
||||
// SignRawBytes: func(b []byte) ([]byte, error) {
|
||||
// _, signedBytes, err := appengine.SignBytes(ctx, b)
|
||||
// return signedBytes, err
|
||||
// },
|
||||
// // etc.
|
||||
// })
|
||||
//
|
||||
// Exactly one of PrivateKey or SignBytes must be non-nil.
|
||||
SignBytes func(hashBytes []byte) (signature []byte, err error)
|
||||
// SignRawBytes is equivalent to the SignBytes field on SignedURLOptions;
|
||||
// that is, you may use the same signing function for the two.
|
||||
//
|
||||
// Exactly one of PrivateKey or SignRawBytes must be non-nil.
|
||||
SignRawBytes func(bytes []byte) (signature []byte, err error)
|
||||
|
||||
// Expires is the expiration time on the signed URL.
|
||||
// It must be a time in the future.
|
||||
|
|
@ -96,6 +112,23 @@ type PostPolicyV4Options struct {
|
|||
// a 4XX status code, back with the message describing the problem.
|
||||
// Optional.
|
||||
Conditions []PostPolicyV4Condition
|
||||
|
||||
shouldHashSignBytes bool
|
||||
}
|
||||
|
||||
func (opts *PostPolicyV4Options) clone() *PostPolicyV4Options {
|
||||
return &PostPolicyV4Options{
|
||||
GoogleAccessID: opts.GoogleAccessID,
|
||||
PrivateKey: opts.PrivateKey,
|
||||
SignBytes: opts.SignBytes,
|
||||
SignRawBytes: opts.SignRawBytes,
|
||||
Expires: opts.Expires,
|
||||
Style: opts.Style,
|
||||
Insecure: opts.Insecure,
|
||||
Fields: opts.Fields,
|
||||
Conditions: opts.Conditions,
|
||||
shouldHashSignBytes: opts.shouldHashSignBytes,
|
||||
}
|
||||
}
|
||||
|
||||
// PolicyV4Fields describes the attributes for a PostPolicyV4 request.
|
||||
|
|
@ -220,20 +253,22 @@ func GenerateSignedPostPolicyV4(bucket, object string, opts *PostPolicyV4Options
|
|||
|
||||
var signingFn func(hashedBytes []byte) ([]byte, error)
|
||||
switch {
|
||||
case opts.SignBytes != nil:
|
||||
case opts.SignRawBytes != nil:
|
||||
signingFn = opts.SignRawBytes
|
||||
case opts.shouldHashSignBytes:
|
||||
signingFn = opts.SignBytes
|
||||
|
||||
case len(opts.PrivateKey) != 0:
|
||||
parsedRSAPrivKey, err := parseKey(opts.PrivateKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
signingFn = func(hashedBytes []byte) ([]byte, error) {
|
||||
return rsa.SignPKCS1v15(rand.Reader, parsedRSAPrivKey, crypto.SHA256, hashedBytes)
|
||||
signingFn = func(b []byte) ([]byte, error) {
|
||||
sum := sha256.Sum256(b)
|
||||
return rsa.SignPKCS1v15(rand.Reader, parsedRSAPrivKey, crypto.SHA256, sum[:])
|
||||
}
|
||||
|
||||
default:
|
||||
return nil, errors.New("storage: exactly one of PrivateKey or SignedBytes must be set")
|
||||
return nil, errors.New("storage: exactly one of PrivateKey or SignRawBytes must be set")
|
||||
}
|
||||
|
||||
var descFields PolicyV4Fields
|
||||
|
|
@ -307,10 +342,18 @@ func GenerateSignedPostPolicyV4(bucket, object string, opts *PostPolicyV4Options
|
|||
}
|
||||
|
||||
b64Policy := base64.StdEncoding.EncodeToString(condsAsJSON)
|
||||
shaSum := sha256.Sum256([]byte(b64Policy))
|
||||
signature, err := signingFn(shaSum[:])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var signature []byte
|
||||
var signErr error
|
||||
|
||||
if opts.shouldHashSignBytes {
|
||||
// SignBytes expects hashed bytes as input instead of raw bytes, so we hash them
|
||||
shaSum := sha256.Sum256([]byte(b64Policy))
|
||||
signature, signErr = signingFn(shaSum[:])
|
||||
} else {
|
||||
signature, signErr = signingFn([]byte(b64Policy))
|
||||
}
|
||||
if signErr != nil {
|
||||
return nil, signErr
|
||||
}
|
||||
|
||||
policyFields["policy"] = b64Policy
|
||||
|
|
@ -348,15 +391,16 @@ func GenerateSignedPostPolicyV4(bucket, object string, opts *PostPolicyV4Options
|
|||
|
||||
// validatePostPolicyV4Options checks that:
|
||||
// * GoogleAccessID is set
|
||||
// * either but not both PrivateKey and SignBytes are set or nil, but not both
|
||||
// * Expires, the deadline is not in the past
|
||||
// * either PrivateKey or SignRawBytes/SignBytes is set, but not both
|
||||
// * the deadline set in Expires is not in the past
|
||||
// * if Style is not set, it'll use PathStyle
|
||||
// * sets shouldHashSignBytes to true if opts.SignBytes should be used
|
||||
func validatePostPolicyV4Options(opts *PostPolicyV4Options, now time.Time) error {
|
||||
if opts == nil || opts.GoogleAccessID == "" {
|
||||
return errors.New("storage: missing required GoogleAccessID")
|
||||
}
|
||||
if privBlank, signBlank := len(opts.PrivateKey) == 0, opts.SignBytes == nil; privBlank == signBlank {
|
||||
return errors.New("storage: exactly one of PrivateKey or SignedBytes must be set")
|
||||
if privBlank, signBlank := len(opts.PrivateKey) == 0, opts.SignBytes == nil && opts.SignRawBytes == nil; privBlank == signBlank {
|
||||
return errors.New("storage: exactly one of PrivateKey or SignRawBytes must be set")
|
||||
}
|
||||
if opts.Expires.Before(now) {
|
||||
return errors.New("storage: expecting Expires to be in the future")
|
||||
|
|
@ -364,6 +408,9 @@ func validatePostPolicyV4Options(opts *PostPolicyV4Options, now time.Time) error
|
|||
if opts.Style == nil {
|
||||
opts.Style = PathStyle()
|
||||
}
|
||||
if opts.SignRawBytes == nil && opts.SignBytes != nil {
|
||||
opts.shouldHashSignBytes = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -163,7 +163,7 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
|
|||
}
|
||||
|
||||
var res *http.Response
|
||||
err = runWithRetry(ctx, func() error {
|
||||
err = run(ctx, func() error {
|
||||
res, err = o.c.hc.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -210,7 +210,7 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
|
|||
gen = gen64
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}, o.retry, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -483,7 +483,7 @@ func (o *ObjectHandle) newRangeReaderWithGRPC(ctx context.Context, offset, lengt
|
|||
var msg *storagepb.ReadObjectResponse
|
||||
var err error
|
||||
|
||||
err = runWithRetry(cc, func() error {
|
||||
err = run(cc, func() error {
|
||||
stream, err = o.c.gc.ReadObject(cc, req)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -492,7 +492,7 @@ func (o *ObjectHandle) newRangeReaderWithGRPC(ctx context.Context, offset, lengt
|
|||
msg, err = stream.Recv()
|
||||
|
||||
return err
|
||||
})
|
||||
}, o.retry, true)
|
||||
if err != nil {
|
||||
// Close the stream context we just created to ensure we don't leak
|
||||
// resources.
|
||||
|
|
@ -541,8 +541,8 @@ func (o *ObjectHandle) newRangeReaderWithGRPC(ctx context.Context, offset, lengt
|
|||
}
|
||||
|
||||
// Only support checksums when reading an entire object, not a range.
|
||||
if msg.GetObjectChecksums().Crc32C != nil && offset == 0 && length == 0 {
|
||||
r.wantCRC = msg.GetObjectChecksums().GetCrc32C()
|
||||
if checksums := msg.GetObjectChecksums(); checksums != nil && checksums.Crc32C != nil && offset == 0 && length == 0 {
|
||||
r.wantCRC = checksums.GetCrc32C()
|
||||
r.checkCRC = true
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ import (
|
|||
"cloud.google.com/go/internal/trace"
|
||||
"cloud.google.com/go/internal/version"
|
||||
gapic "cloud.google.com/go/storage/internal/apiv2"
|
||||
"github.com/googleapis/gax-go/v2"
|
||||
"golang.org/x/oauth2/google"
|
||||
"golang.org/x/xerrors"
|
||||
"google.golang.org/api/googleapi"
|
||||
|
|
@ -50,6 +51,7 @@ import (
|
|||
"google.golang.org/api/transport"
|
||||
htransport "google.golang.org/api/transport/http"
|
||||
storagepb "google.golang.org/genproto/googleapis/storage/v2"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/reflect/protoreflect"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
|
@ -81,6 +83,12 @@ const (
|
|||
// ScopeReadWrite grants permissions to manage your
|
||||
// data in Google Cloud Storage.
|
||||
ScopeReadWrite = raw.DevstorageReadWriteScope
|
||||
|
||||
// defaultConnPoolSize is the default number of connections
|
||||
// to initialize in the GAPIC gRPC connection pool. A larger
|
||||
// connection pool may be necessary for jobs that require
|
||||
// high throughput and/or leverage many concurrent streams.
|
||||
defaultConnPoolSize = 4
|
||||
)
|
||||
|
||||
var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo)
|
||||
|
|
@ -102,6 +110,7 @@ type Client struct {
|
|||
readHost string
|
||||
// May be nil.
|
||||
creds *google.Credentials
|
||||
retry *retryConfig
|
||||
|
||||
// gc is an optional gRPC-based, GAPIC client.
|
||||
//
|
||||
|
|
@ -203,11 +212,34 @@ func newHybridClient(ctx context.Context, opts *hybridClientOptions) (*Client, e
|
|||
if opts == nil {
|
||||
opts = &hybridClientOptions{}
|
||||
}
|
||||
opts.GRPCOpts = append(defaultGRPCOptions(), opts.GRPCOpts...)
|
||||
|
||||
c, err := NewClient(ctx, opts.HTTPOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Set emulator options for gRPC if an emulator was specified. Note that in a
|
||||
// hybrid client, STORAGE_EMULATOR_HOST will set the host to use for HTTP and
|
||||
// STORAGE_EMULATOR_HOST_GRPC will set the host to use for gRPC (when using a
|
||||
// local emulator, HTTP and gRPC must use different ports, so this is
|
||||
// necessary).
|
||||
// TODO: when full gRPC client is available, remove STORAGE_EMULATOR_HOST_GRPC
|
||||
// and use STORAGE_EMULATOR_HOST for both the HTTP and gRPC based clients.
|
||||
if host := os.Getenv("STORAGE_EMULATOR_HOST_GRPC"); host != "" {
|
||||
// Strip the scheme from the emulator host. WithEndpoint does not take a
|
||||
// scheme for gRPC.
|
||||
if strings.Contains(host, "://") {
|
||||
host = strings.SplitN(host, "://", 2)[1]
|
||||
}
|
||||
|
||||
opts.GRPCOpts = append(opts.GRPCOpts,
|
||||
option.WithEndpoint(host),
|
||||
option.WithGRPCDialOption(grpc.WithInsecure()),
|
||||
option.WithoutAuthentication(),
|
||||
)
|
||||
}
|
||||
|
||||
g, err := gapic.NewClient(ctx, opts.GRPCOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -217,6 +249,14 @@ func newHybridClient(ctx context.Context, opts *hybridClientOptions) (*Client, e
|
|||
return c, nil
|
||||
}
|
||||
|
||||
// defaultGRPCOptions returns a set of the default client options
|
||||
// for gRPC client initialization.
|
||||
func defaultGRPCOptions() []option.ClientOption {
|
||||
return []option.ClientOption{
|
||||
option.WithGRPCConnectionPool(defaultConnPoolSize),
|
||||
}
|
||||
}
|
||||
|
||||
// Close closes the Client.
|
||||
//
|
||||
// Close need not be called at program exit.
|
||||
|
|
@ -836,6 +876,7 @@ type ObjectHandle struct {
|
|||
encryptionKey []byte // AES-256 key
|
||||
userProject string // for requester-pays buckets
|
||||
readCompressed bool // Accept-Encoding: gzip
|
||||
retry *retryConfig
|
||||
}
|
||||
|
||||
// ACL provides access to the object's access control list.
|
||||
|
|
@ -899,7 +940,7 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (attrs *ObjectAttrs, err error
|
|||
}
|
||||
var obj *raw.Object
|
||||
setClientHeader(call.Header())
|
||||
err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err })
|
||||
err = run(ctx, func() error { obj, err = call.Do(); return err }, o.retry, true)
|
||||
var e *googleapi.Error
|
||||
if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound {
|
||||
return nil, ErrObjectNotExist
|
||||
|
|
@ -1000,7 +1041,11 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (
|
|||
}
|
||||
var obj *raw.Object
|
||||
setClientHeader(call.Header())
|
||||
err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err })
|
||||
var isIdempotent bool
|
||||
if o.conds != nil && o.conds.MetagenerationMatch != 0 {
|
||||
isIdempotent = true
|
||||
}
|
||||
err = run(ctx, func() error { obj, err = call.Do(); return err }, o.retry, isIdempotent)
|
||||
var e *googleapi.Error
|
||||
if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound {
|
||||
return nil, ErrObjectNotExist
|
||||
|
|
@ -1064,7 +1109,13 @@ func (o *ObjectHandle) Delete(ctx context.Context) error {
|
|||
}
|
||||
// Encryption doesn't apply to Delete.
|
||||
setClientHeader(call.Header())
|
||||
err := runWithRetry(ctx, func() error { return call.Do() })
|
||||
var isIdempotent bool
|
||||
// Delete is idempotent if GenerationMatch or Generation have been passed in.
|
||||
// The default generation is negative to get the latest version of the object.
|
||||
if (o.conds != nil && o.conds.GenerationMatch != 0) || o.gen >= 0 {
|
||||
isIdempotent = true
|
||||
}
|
||||
err := run(ctx, func() error { return call.Do() }, o.retry, isIdempotent)
|
||||
var e *googleapi.Error
|
||||
if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound {
|
||||
return ErrObjectNotExist
|
||||
|
|
@ -1759,6 +1810,169 @@ func setConditionField(call reflect.Value, name string, value interface{}) bool
|
|||
return true
|
||||
}
|
||||
|
||||
// Retryer returns an object handle that is configured with custom retry
|
||||
// behavior as specified by the options that are passed to it. All operations
|
||||
// on the new handle will use the customized retry configuration.
|
||||
// These retry options will merge with the bucket's retryer (if set) for the
|
||||
// returned handle. Options passed into this method will take precedence over
|
||||
// retry options on the bucket and client. Note that you must explicitly pass in
|
||||
// each option you want to override.
|
||||
func (o *ObjectHandle) Retryer(opts ...RetryOption) *ObjectHandle {
|
||||
o2 := *o
|
||||
var retry *retryConfig
|
||||
if o.retry != nil {
|
||||
// merge the options with the existing retry
|
||||
retry = o.retry
|
||||
} else {
|
||||
retry = &retryConfig{}
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt.apply(retry)
|
||||
}
|
||||
o2.retry = retry
|
||||
o2.acl.retry = retry
|
||||
return &o2
|
||||
}
|
||||
|
||||
// SetRetry configures the client with custom retry behavior as specified by the
|
||||
// options that are passed to it. All operations using this client will use the
|
||||
// customized retry configuration.
|
||||
// This should be called once before using the client for network operations, as
|
||||
// there could be indeterminate behaviour with operations in progress.
|
||||
// Retry options set on a bucket or object handle will take precedence over
|
||||
// these options.
|
||||
func (c *Client) SetRetry(opts ...RetryOption) {
|
||||
var retry *retryConfig
|
||||
if c.retry != nil {
|
||||
// merge the options with the existing retry
|
||||
retry = c.retry
|
||||
} else {
|
||||
retry = &retryConfig{}
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt.apply(retry)
|
||||
}
|
||||
c.retry = retry
|
||||
}
|
||||
|
||||
// RetryOption allows users to configure non-default retry behavior for API
|
||||
// calls made to GCS.
|
||||
type RetryOption interface {
|
||||
apply(config *retryConfig)
|
||||
}
|
||||
|
||||
// WithBackoff allows configuration of the backoff timing used for retries.
|
||||
// Available configuration options (Initial, Max and Multiplier) are described
|
||||
// at https://pkg.go.dev/github.com/googleapis/gax-go/v2#Backoff. If any fields
|
||||
// are not supplied by the user, gax default values will be used.
|
||||
func WithBackoff(backoff gax.Backoff) RetryOption {
|
||||
return &withBackoff{
|
||||
backoff: backoff,
|
||||
}
|
||||
}
|
||||
|
||||
type withBackoff struct {
|
||||
backoff gax.Backoff
|
||||
}
|
||||
|
||||
func (wb *withBackoff) apply(config *retryConfig) {
|
||||
config.backoff = &wb.backoff
|
||||
}
|
||||
|
||||
// RetryPolicy describes the available policies for which operations should be
|
||||
// retried. The default is `RetryIdempotent`.
|
||||
type RetryPolicy int
|
||||
|
||||
const (
|
||||
// RetryIdempotent causes only idempotent operations to be retried when the
|
||||
// service returns a transient error. Using this policy, fully idempotent
|
||||
// operations (such as `ObjectHandle.Attrs()`) will always be retried.
|
||||
// Conditionally idempotent operations (for example `ObjectHandle.Update()`)
|
||||
// will be retried only if the necessary conditions have been supplied (in
|
||||
// the case of `ObjectHandle.Update()` this would mean supplying a
|
||||
// `Conditions.MetagenerationMatch` condition is required).
|
||||
RetryIdempotent RetryPolicy = iota
|
||||
|
||||
// RetryAlways causes all operations to be retried when the service returns a
|
||||
// transient error, regardless of idempotency considerations.
|
||||
RetryAlways
|
||||
|
||||
// RetryNever causes the client to not perform retries on failed operations.
|
||||
RetryNever
|
||||
)
|
||||
|
||||
// WithPolicy allows the configuration of which operations should be performed
|
||||
// with retries for transient errors.
|
||||
func WithPolicy(policy RetryPolicy) RetryOption {
|
||||
return &withPolicy{
|
||||
policy: policy,
|
||||
}
|
||||
}
|
||||
|
||||
type withPolicy struct {
|
||||
policy RetryPolicy
|
||||
}
|
||||
|
||||
func (ws *withPolicy) apply(config *retryConfig) {
|
||||
config.policy = ws.policy
|
||||
}
|
||||
|
||||
// WithErrorFunc allows users to pass a custom function to the retryer. Errors
|
||||
// will be retried if and only if `shouldRetry(err)` returns true.
|
||||
// By default, the following errors are retried (see invoke.go for the default
|
||||
// shouldRetry function):
|
||||
//
|
||||
// - HTTP responses with codes 408, 429, 502, 503, and 504.
|
||||
//
|
||||
// - Transient network errors such as connection reset and io.ErrUnexpectedEOF.
|
||||
//
|
||||
// - Errors which are considered transient using the Temporary() interface.
|
||||
//
|
||||
// - Wrapped versions of these errors.
|
||||
//
|
||||
// This option can be used to retry on a different set of errors than the
|
||||
// default.
|
||||
func WithErrorFunc(shouldRetry func(err error) bool) RetryOption {
|
||||
return &withErrorFunc{
|
||||
shouldRetry: shouldRetry,
|
||||
}
|
||||
}
|
||||
|
||||
type withErrorFunc struct {
|
||||
shouldRetry func(err error) bool
|
||||
}
|
||||
|
||||
func (wef *withErrorFunc) apply(config *retryConfig) {
|
||||
config.shouldRetry = wef.shouldRetry
|
||||
}
|
||||
|
||||
type retryConfig struct {
|
||||
backoff *gax.Backoff
|
||||
policy RetryPolicy
|
||||
shouldRetry func(err error) bool
|
||||
}
|
||||
|
||||
func (r *retryConfig) clone() *retryConfig {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var bo *gax.Backoff
|
||||
if r.backoff != nil {
|
||||
bo = &gax.Backoff{
|
||||
Initial: r.backoff.Initial,
|
||||
Max: r.backoff.Max,
|
||||
Multiplier: r.backoff.Multiplier,
|
||||
}
|
||||
}
|
||||
|
||||
return &retryConfig{
|
||||
backoff: bo,
|
||||
policy: r.policy,
|
||||
shouldRetry: r.shouldRetry,
|
||||
}
|
||||
}
|
||||
|
||||
// composeSourceObj wraps a *raw.ComposeRequestSourceObjects, but adds the methods
|
||||
// that modifyCall searches for by name.
|
||||
type composeSourceObj struct {
|
||||
|
|
@ -1802,10 +2016,10 @@ func (c *Client) ServiceAccount(ctx context.Context, projectID string) (string,
|
|||
r := c.raw.Projects.ServiceAccount.Get(projectID)
|
||||
var res *raw.ServiceAccount
|
||||
var err error
|
||||
err = runWithRetry(ctx, func() error {
|
||||
err = run(ctx, func() error {
|
||||
res, err = r.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
}, c.retry, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -172,6 +172,22 @@ func (w *Writer) open() error {
|
|||
// call to set up the upload as well as calls to upload individual chunks
|
||||
// for a resumable upload (as long as the chunk size is non-zero). Hence
|
||||
// there is no need to add retries here.
|
||||
|
||||
// Retry only when the operation is idempotent or the retry policy is RetryAlways.
|
||||
isIdempotent := w.o.conds != nil && (w.o.conds.GenerationMatch >= 0 || w.o.conds.DoesNotExist == true)
|
||||
var useRetry bool
|
||||
if (w.o.retry == nil || w.o.retry.policy == RetryIdempotent) && isIdempotent {
|
||||
useRetry = true
|
||||
} else if w.o.retry != nil && w.o.retry.policy == RetryAlways {
|
||||
useRetry = true
|
||||
}
|
||||
if useRetry {
|
||||
if w.o.retry != nil {
|
||||
call.WithRetry(w.o.retry.backoff, w.o.retry.shouldRetry)
|
||||
} else {
|
||||
call.WithRetry(nil, nil)
|
||||
}
|
||||
}
|
||||
resp, err = call.Do()
|
||||
}
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ cloud.google.com/go/compute/metadata
|
|||
# cloud.google.com/go/iam v0.1.1
|
||||
## explicit; go 1.11
|
||||
cloud.google.com/go/iam
|
||||
# cloud.google.com/go/storage v1.18.2
|
||||
# cloud.google.com/go/storage v1.19.0
|
||||
## explicit; go 1.11
|
||||
cloud.google.com/go/storage
|
||||
cloud.google.com/go/storage/internal/apiv2
|
||||
|
|
|
|||
Loading…
Reference in New Issue