public_git/kaniko
1
0
Fork 0
mirror of https://github.com/GoogleContainerTools/kaniko synced 2026-10-09 16:46:18 +02:00
Code Issues Releases Wiki Activity

fix(executor): make pax tar builds reproducible again (#2384)

Browse Source 
In v1.8.0 (commit 7410007) kaniko switched to using the pax tar header
format for compressing image layers, since this format allows for greater
precision in recording timestamps, however this inadvertendly broke the
"--reproducible" functionality, due to an bug in the underlying
go-containerregistry dependency which did not set the additional
timestamps in the pax header when canonicalizing image layers. This
oversight has since been fixed in the dependency.

This commit bumps the google/go-containerregistry dependency to the
first commit which has fixed the bug
(v0.13.1-0.20230201183932-824efc7772b0). It also bumps the version of
cloud.google.com/go/storage to v1.29.0 to be compatible with the higher
transitive dependency.
This commit is contained in:
Joël Pepper
2023-03-21 18:08:29 -04:00
committed by GitHub
parent 14ea7c4071
commit 0cbb7664de
873 changed files with 78917 additions and 29139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+38 -32
View File
@@ -11,19 +11,19 @@ replace (
)
require (
cloud.google.com/go/storage v1.22.0
cloud.google.com/go/storage v1.29.0
github.com/Azure/azure-storage-blob-go v0.14.0
github.com/aws/aws-sdk-go v1.44.24
github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20220228164355-396b2034c795
github.com/chrismellard/docker-credential-acr-env v0.0.0-20220119192733-fe33c00cee21
github.com/containerd/cgroups v1.0.3 // indirect
github.com/docker/docker v20.10.14+incompatible
github.com/docker/docker v20.10.20+incompatible
github.com/go-git/go-billy/v5 v5.3.1
github.com/go-git/go-git/v5 v5.4.2
github.com/godbus/dbus/v5 v5.0.6 // indirect
github.com/golang/mock v1.6.0
github.com/google/go-cmp v0.5.7
github.com/google/go-containerregistry v0.8.1-0.20220507185902-82405e5dfa82
github.com/google/go-cmp v0.5.9
github.com/google/go-containerregistry v0.13.1-0.20230201183932-824efc7772b0
github.com/google/go-github v17.0.0+incompatible
github.com/google/slowjam v1.0.0
github.com/karrick/godirwalk v1.16.1
@@ -31,21 +31,28 @@ require (
github.com/moby/buildkit v0.9.3
github.com/otiai10/copy v1.7.0
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.8.1
github.com/sirupsen/logrus v1.9.0
github.com/spf13/afero v1.8.2
github.com/spf13/cobra v1.4.0
github.com/spf13/cobra v1.6.0
github.com/spf13/pflag v1.0.5
golang.org/x/net v0.0.0-20220325170049-de3da57026de
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/net v0.1.0
golang.org/x/oauth2 v0.1.0
golang.org/x/sync v0.1.0
)
require github.com/containerd/containerd v1.6.2
require (
cloud.google.com/go v0.100.2 // indirect
cloud.google.com/go/compute v1.5.0 // indirect
cloud.google.com/go/iam v0.3.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
golang.org/x/mod v0.6.0 // indirect
golang.org/x/tools v0.1.12 // indirect
)
require (
cloud.google.com/go v0.107.0 // indirect
cloud.google.com/go/compute v1.14.0 // indirect
cloud.google.com/go/iam v0.8.0 // indirect
github.com/Azure/azure-pipeline-go v0.2.3 // indirect
github.com/Azure/azure-sdk-for-go v61.3.0+incompatible // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
@@ -57,7 +64,7 @@ require (
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/Microsoft/hcsshim v0.9.2 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
@@ -79,19 +86,19 @@ require (
github.com/cilium/ebpf v0.8.0 // indirect
github.com/containerd/continuity v0.2.2 // indirect
github.com/containerd/fifo v1.0.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.11.1 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.12.1 // indirect
github.com/containerd/typeurl v1.0.2 // indirect
github.com/coreos/etcd v3.3.27+incompatible // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/docker/cli v20.10.12+incompatible // indirect
github.com/docker/distribution v2.8.0+incompatible // indirect
github.com/docker/docker-credential-helpers v0.6.4 // indirect
github.com/docker/cli v20.10.20+incompatible // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/docker/swarmkit v1.12.1-0.20180726190244-7567d47988d8 // indirect
github.com/ePirat/docker-credential-gitlabci v1.0.0
github.com/emirpasic/gods v1.12.0 // indirect
@@ -103,17 +110,16 @@ require (
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/gax-go/v2 v2.2.0 // indirect
github.com/googleapis/go-type-adapters v1.0.0 // indirect
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-memdb v1.3.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kevinburke/ssh_config v1.1.0 // indirect
github.com/klauspost/compress v1.14.4 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/mattn/go-ieproxy v0.0.2 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -124,7 +130,7 @@ require (
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20220114050600-8b9d41f48198 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
github.com/opencontainers/runc v1.1.0 // indirect
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
github.com/opencontainers/selinux v1.10.0 // indirect
@@ -137,16 +143,16 @@ require (
github.com/tonistiigi/fsutil v0.0.0-20220115021204-b19f7f9cb274 // indirect
github.com/vbatts/tar-split v0.11.2 // indirect
github.com/xanzy/ssh-agent v0.3.1 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce // indirect
golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886
golang.org/x/text v0.3.7 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/sys v0.1.0
golang.org/x/text v0.5.0 // indirect
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/api v0.74.0
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.106.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220405205423-9d709892a2bf // indirect
google.golang.org/grpc v1.45.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
google.golang.org/grpc v1.51.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
)
go.sum
+84 -678
View File
File diff suppressed because it is too large Load Diff
vendor/cloud.google.com/go/.gitignore
Generated
-12
View File
@@ -1,12 +0,0 @@
# Editors
.idea
.vscode
*.swp
.history
# Test files
*.test
coverage.txt
# Other
.DS_Store
vendor/cloud.google.com/go/CHANGES.md
Generated Vendored
-2372
View File
File diff suppressed because it is too large Load Diff
vendor/cloud.google.com/go/CODE_OF_CONDUCT.md
Generated Vendored
-44
View File
@@ -1,44 +0,0 @@
# Contributor Code of Conduct
As contributors and maintainers of this project,
and in the interest of fostering an open and welcoming community,
we pledge to respect all people who contribute through reporting issues,
posting feature requests, updating documentation,
submitting pull requests or patches, and other activities.
We are committed to making participation in this project
a harassment-free experience for everyone,
regardless of level of experience, gender, gender identity and expression,
sexual orientation, disability, personal appearance,
body size, race, ethnicity, age, religion, or nationality.
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery
* Personal attacks
* Trolling or insulting/derogatory comments
* Public or private harassment
* Publishing other's private information,
such as physical or electronic
addresses, without explicit permission
* Other unethical or unprofessional conduct.
Project maintainers have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct.
By adopting this Code of Conduct,
project maintainers commit themselves to fairly and consistently
applying these principles to every aspect of managing this project.
Project maintainers who do not follow or enforce the Code of Conduct
may be permanently removed from the project team.
This code of conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community.
Instances of abusive, harassing, or otherwise unacceptable behavior
may be reported by opening an issue
or contacting one or more of the project maintainers.
This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0,
available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/)
vendor/cloud.google.com/go/CONTRIBUTING.md
Generated Vendored
-327
View File
@@ -1,327 +0,0 @@
# Contributing
1. [File an issue](https://github.com/googleapis/google-cloud-go/issues/new/choose).
The issue will be used to discuss the bug or feature and should be created
before sending a CL.
1. [Install Go](https://golang.org/dl/).
1. Ensure that your `GOBIN` directory (by default `$(go env GOPATH)/bin`)
is in your `PATH`.
1. Check it's working by running `go version`.
* If it doesn't work, check the install location, usually
`/usr/local/go`, is on your `PATH`.
1. Sign one of the
[contributor license agreements](#contributor-license-agreements) below.
1. Clone the repo:
`git clone https://github.com/googleapis/google-cloud-go`
1. Change into the checked out source:
`cd google-cloud-go`
1. Fork the repo.
1. Set your fork as a remote:
`git remote add fork git@github.com:GITHUB_USERNAME/google-cloud-go.git`
1. Make changes, commit to your fork.
Commit messages should follow the
[Conventional Commits Style](https://www.conventionalcommits.org). The scope
portion should always be filled with the name of the package affected by the
changes being made. For example:
```
feat(functions): add gophers codelab
```
1. Send a pull request with your changes.
To minimize friction, consider setting `Allow edits from maintainers` on the
PR, which will enable project committers and automation to update your PR.
1. A maintainer will review the pull request and make comments.
Prefer adding additional commits over amending and force-pushing since it can
be difficult to follow code reviews when the commit history changes.
Commits will be squashed when they're merged.
## Testing
We test code against two versions of Go, the minimum and maximum versions
supported by our clients. To see which versions these are checkout our
[README](README.md#supported-versions).
### Integration Tests
In addition to the unit tests, you may run the integration test suite. These
directions describe setting up your environment to run integration tests for
_all_ packages: note that many of these instructions may be redundant if you
intend only to run integration tests on a single package.
#### GCP Setup
To run the integrations tests, creation and configuration of two projects in
the Google Developers Console is required: one specifically for Firestore
integration tests, and another for all other integration tests. We'll refer to
these projects as "general project" and "Firestore project".
After creating each project, you must [create a service account](https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount)
for each project. Ensure the project-level **Owner**
[IAM role](https://console.cloud.google.com/iam-admin/iam/project) role is added to
each service account. During the creation of the service account, you should
download the JSON credential file for use later.
Next, ensure the following APIs are enabled in the general project:
- BigQuery API
- BigQuery Data Transfer API
- Cloud Dataproc API
- Cloud Dataproc Control API Private
- Cloud Datastore API
- Cloud Firestore API
- Cloud Key Management Service (KMS) API
- Cloud Natural Language API
- Cloud OS Login API
- Cloud Pub/Sub API
- Cloud Resource Manager API
- Cloud Spanner API
- Cloud Speech API
- Cloud Translation API
- Cloud Video Intelligence API
- Cloud Vision API
- Compute Engine API
- Compute Engine Instance Group Manager API
- Container Registry API
- Firebase Rules API
- Google Cloud APIs
- Google Cloud Deployment Manager V2 API
- Google Cloud SQL
- Google Cloud Storage
- Google Cloud Storage JSON API
- Google Compute Engine Instance Group Updater API
- Google Compute Engine Instance Groups API
- Kubernetes Engine API
- Cloud Error Reporting API
- Pub/Sub Lite API
Next, create a Datastore database in the general project, and a Firestore
database in the Firestore project.
Finally, in the general project, create an API key for the translate API:
- Go to GCP Developer Console.
- Navigate to APIs & Services > Credentials.
- Click Create Credentials > API Key.
- Save this key for use in `GCLOUD_TESTS_API_KEY` as described below.
#### Local Setup
Once the two projects are created and configured, set the following environment
variables:
- `GCLOUD_TESTS_GOLANG_PROJECT_ID`: Developers Console project's ID (e.g.
bamboo-shift-455) for the general project.
- `GCLOUD_TESTS_GOLANG_KEY`: The path to the JSON key file of the general
project's service account.
- `GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID`: Developers Console project's ID
(e.g. doorway-cliff-677) for the Firestore project.
- `GCLOUD_TESTS_GOLANG_FIRESTORE_KEY`: The path to the JSON key file of the
Firestore project's service account.
- `GCLOUD_TESTS_API_KEY`: API key for using the Translate API created above.
As part of the setup that follows, the following variables will be configured:
- `GCLOUD_TESTS_GOLANG_KEYRING`: The full name of the keyring for the tests,
in the form
"projects/P/locations/L/keyRings/R". The creation of this is described below.
- `GCLOUD_TESTS_BIGTABLE_KEYRING`: The full name of the keyring for the bigtable tests,
in the form
"projects/P/locations/L/keyRings/R". The creation of this is described below. Expected to be single region.
- `GCLOUD_TESTS_GOLANG_ZONE`: Compute Engine zone.
Install the [gcloud command-line tool][gcloudcli] to your machine and use it to
create some resources used in integration tests.
From the project's root directory:
``` sh
# Sets the default project in your env.
$ gcloud config set project $GCLOUD_TESTS_GOLANG_PROJECT_ID
# Authenticates the gcloud tool with your account.
$ gcloud auth login
# Create the indexes used in the datastore integration tests.
$ gcloud datastore indexes create datastore/testdata/index.yaml
# Creates a Google Cloud storage bucket with the same name as your test project,
# and with the Cloud Logging service account as owner, for the sink
# integration tests in logging.
$ gsutil mb gs://$GCLOUD_TESTS_GOLANG_PROJECT_ID
$ gsutil acl ch -g cloud-logs@google.com:O gs://$GCLOUD_TESTS_GOLANG_PROJECT_ID
# Creates a PubSub topic for integration tests of storage notifications.
$ gcloud beta pubsub topics create go-storage-notification-test
# Next, go to the Pub/Sub dashboard in GCP console. Authorize the user
# "service-<numeric project id>@gs-project-accounts.iam.gserviceaccount.com"
# as a publisher to that topic.
# Creates a Spanner instance for the spanner integration tests.
$ gcloud beta spanner instances create go-integration-test --config regional-us-central1 --nodes 10 --description 'Instance for go client test'
# NOTE: Spanner instances are priced by the node-hour, so you may want to
# delete the instance after testing with 'gcloud beta spanner instances delete'.
$ export MY_KEYRING=some-keyring-name
$ export MY_LOCATION=global
$ export MY_SINGLE_LOCATION=us-central1
# Creates a KMS keyring, in the same location as the default location for your
# project's buckets.
$ gcloud kms keyrings create $MY_KEYRING --location $MY_LOCATION
# Creates two keys in the keyring, named key1 and key2.
$ gcloud kms keys create key1 --keyring $MY_KEYRING --location $MY_LOCATION --purpose encryption
$ gcloud kms keys create key2 --keyring $MY_KEYRING --location $MY_LOCATION --purpose encryption
# Sets the GCLOUD_TESTS_GOLANG_KEYRING environment variable.
$ export GCLOUD_TESTS_GOLANG_KEYRING=projects/$GCLOUD_TESTS_GOLANG_PROJECT_ID/locations/$MY_LOCATION/keyRings/$MY_KEYRING
# Authorizes Google Cloud Storage to encrypt and decrypt using key1.
$ gsutil kms authorize -p $GCLOUD_TESTS_GOLANG_PROJECT_ID -k $GCLOUD_TESTS_GOLANG_KEYRING/cryptoKeys/key1
# Create KMS Key in one region for Bigtable
$ gcloud kms keyrings create $MY_KEYRING --location $MY_SINGLE_LOCATION
$ gcloud kms keys create key1 --keyring $MY_KEYRING --location $MY_SINGLE_LOCATION --purpose encryption
# Sets the GCLOUD_TESTS_BIGTABLE_KEYRING environment variable.
$ export GCLOUD_TESTS_BIGTABLE_KEYRING=projects/$GCLOUD_TESTS_GOLANG_PROJECT_ID/locations/$MY_SINGLE_LOCATION/keyRings/$MY_KEYRING
# Create a service agent, https://cloud.google.com/bigtable/docs/use-cmek#gcloud:
$ gcloud beta services identity create \
--service=bigtableadmin.googleapis.com \
--project $GCLOUD_TESTS_GOLANG_PROJECT_ID
# Note the service agent email for the agent created.
$ export SERVICE_AGENT_EMAIL=<service agent email, from last step>
# Authorizes Google Cloud Bigtable to encrypt and decrypt using key1
$ gcloud kms keys add-iam-policy-binding key1 \
--keyring $MY_KEYRING \
--location $MY_SINGLE_LOCATION \
--role roles/cloudkms.cryptoKeyEncrypterDecrypter \
--member "serviceAccount:$SERVICE_AGENT_EMAIL" \
--project $GCLOUD_TESTS_GOLANG_PROJECT_ID
```
It may be useful to add exports to your shell initialization for future use.
For instance, in `.zshrc`:
```sh
#### START GO SDK Test Variables
# Developers Console project's ID (e.g. bamboo-shift-455) for the general project.
export GCLOUD_TESTS_GOLANG_PROJECT_ID=your-project
# The path to the JSON key file of the general project's service account.
export GCLOUD_TESTS_GOLANG_KEY=~/directory/your-project-abcd1234.json
# Developers Console project's ID (e.g. doorway-cliff-677) for the Firestore project.
export GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID=your-firestore-project
# The path to the JSON key file of the Firestore project's service account.
export GCLOUD_TESTS_GOLANG_FIRESTORE_KEY=~/directory/your-firestore-project-abcd1234.json
# The full name of the keyring for the tests, in the form "projects/P/locations/L/keyRings/R".
# The creation of this is described below.
export MY_KEYRING=my-golang-sdk-test
export MY_LOCATION=global
export GCLOUD_TESTS_GOLANG_KEYRING=projects/$GCLOUD_TESTS_GOLANG_PROJECT_ID/locations/$MY_LOCATION/keyRings/$MY_KEYRING
# API key for using the Translate API.
export GCLOUD_TESTS_API_KEY=abcdefghijk123456789
# Compute Engine zone. (https://cloud.google.com/compute/docs/regions-zones)
export GCLOUD_TESTS_GOLANG_ZONE=your-chosen-region
#### END GO SDK Test Variables
```
#### Running
Once you've done the necessary setup, you can run the integration tests by
running:
``` sh
$ go test -v ./...
```
Note that the above command will not run the tests in other modules. To run
tests on other modules, first navigate to the appropriate
subdirectory. For instance, to run only the tests for datastore:
``` sh
$ cd datastore
$ go test -v ./...
```
#### Replay
Some packages can record the RPCs during integration tests to a file for
subsequent replay. To record, pass the `-record` flag to `go test`. The
recording will be saved to the _package_`.replay` file. To replay integration
tests from a saved recording, the replay file must be present, the `-short`
flag must be passed to `go test`, and the `GCLOUD_TESTS_GOLANG_ENABLE_REPLAY`
environment variable must have a non-empty value.
## Contributor License Agreements
Before we can accept your pull requests you'll need to sign a Contributor
License Agreement (CLA):
- **If you are an individual writing original source code** and **you own the
intellectual property**, then you'll need to sign an [individual CLA][indvcla].
- **If you work for a company that wants to allow you to contribute your
work**, then you'll need to sign a [corporate CLA][corpcla].
You can sign these electronically (just scroll to the bottom). After that,
we'll be able to accept your pull requests.
## Contributor Code of Conduct
As contributors and maintainers of this project,
and in the interest of fostering an open and welcoming community,
we pledge to respect all people who contribute through reporting issues,
posting feature requests, updating documentation,
submitting pull requests or patches, and other activities.
We are committed to making participation in this project
a harassment-free experience for everyone,
regardless of level of experience, gender, gender identity and expression,
sexual orientation, disability, personal appearance,
body size, race, ethnicity, age, religion, or nationality.
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery
* Personal attacks
* Trolling or insulting/derogatory comments
* Public or private harassment
* Publishing other's private information,
such as physical or electronic
addresses, without explicit permission
* Other unethical or unprofessional conduct.
Project maintainers have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct.
By adopting this Code of Conduct,
project maintainers commit themselves to fairly and consistently
applying these principles to every aspect of managing this project.
Project maintainers who do not follow or enforce the Code of Conduct
may be permanently removed from the project team.
This code of conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community.
Instances of abusive, harassing, or otherwise unacceptable behavior
may be reported by opening an issue
or contacting one or more of the project maintainers.
This Code of Conduct is adapted from the [Contributor Covenant](https://contributor-covenant.org), version 1.2.0,
available at [https://contributor-covenant.org/version/1/2/0/](https://contributor-covenant.org/version/1/2/0/)
[gcloudcli]: https://developers.google.com/cloud/sdk/gcloud/
[indvcla]: https://developers.google.com/open-source/cla/individual
[corpcla]: https://developers.google.com/open-source/cla/corporate
vendor/cloud.google.com/go/README.md
Generated Vendored
-132
View File
@@ -1,132 +0,0 @@
# Google Cloud Client Libraries for Go
[![Go Reference](https://pkg.go.dev/badge/cloud.google.com/go.svg)](https://pkg.go.dev/cloud.google.com/go)
Go packages for [Google Cloud Platform](https://cloud.google.com) services.
``` go
import "cloud.google.com/go"
```
To install the packages on your system, *do not clone the repo*. Instead:
1. Change to your project directory:
```bash
cd /my/cloud/project
```
1. Get the package you want to use. Some products have their own module, so it's
best to `go get` the package(s) you want to use:
```
$ go get cloud.google.com/go/firestore # Replace with the package you want to use.
```
**NOTE:** Some of these packages are under development, and may occasionally
make backwards-incompatible changes.
## Supported APIs
For an updated list of all of our released APIs please see our
[reference docs](https://cloud.google.com/go/docs/reference).
## [Go Versions Supported](#supported-versions)
We currently support Go versions 1.11 and newer.
## Authorization
By default, each API will use [Google Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials)
for authorization credentials used in calling the API endpoints. This will allow your
application to run in many environments without requiring explicit configuration.
[snip]:# (auth)
```go
client, err := storage.NewClient(ctx)
```
To authorize using a
[JSON key file](https://cloud.google.com/iam/docs/managing-service-account-keys),
pass
[`option.WithCredentialsFile`](https://pkg.go.dev/google.golang.org/api/option#WithCredentialsFile)
to the `NewClient` function of the desired package. For example:
[snip]:# (auth-JSON)
```go
client, err := storage.NewClient(ctx, option.WithCredentialsFile("path/to/keyfile.json"))
```
You can exert more control over authorization by using the
[`golang.org/x/oauth2`](https://pkg.go.dev/golang.org/x/oauth2) package to
create an `oauth2.TokenSource`. Then pass
[`option.WithTokenSource`](https://pkg.go.dev/google.golang.org/api/option#WithTokenSource)
to the `NewClient` function:
[snip]:# (auth-ts)
```go
tokenSource := ...
client, err := storage.NewClient(ctx, option.WithTokenSource(tokenSource))
```
## Contributing
Contributions are welcome. Please, see the
[CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md)
document for details.
Please note that this project is released with a Contributor Code of Conduct.
By participating in this project you agree to abide by its terms.
See [Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct)
for more information.
[cloud-asset]: https://cloud.google.com/security-command-center/docs/how-to-asset-inventory
[cloud-automl]: https://cloud.google.com/automl
[cloud-build]: https://cloud.google.com/cloud-build/
[cloud-bigquery]: https://cloud.google.com/bigquery/
[cloud-bigtable]: https://cloud.google.com/bigtable/
[cloud-compute]: https://cloud.google.com/compute
[cloud-container]: https://cloud.google.com/containers/
[cloud-containeranalysis]: https://cloud.google.com/container-registry/docs/container-analysis
[cloud-dataproc]: https://cloud.google.com/dataproc/
[cloud-datastore]: https://cloud.google.com/datastore/
[cloud-dialogflow]: https://cloud.google.com/dialogflow-enterprise/
[cloud-debugger]: https://cloud.google.com/debugger/
[cloud-dlp]: https://cloud.google.com/dlp/
[cloud-errors]: https://cloud.google.com/error-reporting/
[cloud-firestore]: https://cloud.google.com/firestore/
[cloud-iam]: https://cloud.google.com/iam/
[cloud-iot]: https://cloud.google.com/iot-core/
[cloud-irm]: https://cloud.google.com/incident-response/docs/concepts
[cloud-kms]: https://cloud.google.com/kms/
[cloud-pubsub]: https://cloud.google.com/pubsub/
[cloud-pubsublite]: https://cloud.google.com/pubsub/lite
[cloud-storage]: https://cloud.google.com/storage/
[cloud-language]: https://cloud.google.com/natural-language
[cloud-logging]: https://cloud.google.com/logging/
[cloud-natural-language]: https://cloud.google.com/natural-language/
[cloud-memorystore]: https://cloud.google.com/memorystore/
[cloud-monitoring]: https://cloud.google.com/monitoring/
[cloud-oslogin]: https://cloud.google.com/compute/docs/oslogin/rest
[cloud-phishingprotection]: https://cloud.google.com/phishing-protection/
[cloud-securitycenter]: https://cloud.google.com/security-command-center/
[cloud-scheduler]: https://cloud.google.com/scheduler
[cloud-spanner]: https://cloud.google.com/spanner/
[cloud-speech]: https://cloud.google.com/speech
[cloud-talent]: https://cloud.google.com/solutions/talent-solution/
[cloud-tasks]: https://cloud.google.com/tasks/
[cloud-texttospeech]: https://cloud.google.com/texttospeech/
[cloud-talent]: https://cloud.google.com/solutions/talent-solution/
[cloud-trace]: https://cloud.google.com/trace/
[cloud-translate]: https://cloud.google.com/translate
[cloud-recaptcha]: https://cloud.google.com/recaptcha-enterprise/
[cloud-recommender]: https://cloud.google.com/recommendations/
[cloud-video]: https://cloud.google.com/video-intelligence/
[cloud-vision]: https://cloud.google.com/vision
[cloud-webrisk]: https://cloud.google.com/web-risk/
## Links
- [Go on Google Cloud](https://cloud.google.com/go/home)
- [Getting started with Go on Google Cloud](https://cloud.google.com/go/getting-started)
- [App Engine Quickstart](https://cloud.google.com/appengine/docs/standard/go/quickstart)
- [Cloud Functions Quickstart](https://cloud.google.com/functions/docs/quickstart-go)
- [Cloud Run Quickstart](https://cloud.google.com/run/docs/quickstarts/build-and-deploy#go)
vendor/cloud.google.com/go/RELEASING.md
Generated Vendored
-141
View File
@@ -1,141 +0,0 @@
# Releasing
## Determine which module to release
The Go client libraries have several modules. Each module does not strictly
correspond to a single library - they correspond to trees of directories. If a
file needs to be released, you must release the closest ancestor module.
To see all modules:
```bash
$ cat `find . -name go.mod` | grep module
module cloud.google.com/go/pubsub
module cloud.google.com/go/spanner
module cloud.google.com/go
module cloud.google.com/go/bigtable
module cloud.google.com/go/bigquery
module cloud.google.com/go/storage
module cloud.google.com/go/pubsublite
module cloud.google.com/go/firestore
module cloud.google.com/go/logging
module cloud.google.com/go/internal/gapicgen
module cloud.google.com/go/internal/godocfx
module cloud.google.com/go/internal/examples/fake
module cloud.google.com/go/internal/examples/mock
module cloud.google.com/go/datastore
```
The `cloud.google.com/go` is the repository root module. Each other module is
a submodule.
So, if you need to release a change in `bigtable/bttest/inmem.go`, the closest
ancestor module is `cloud.google.com/go/bigtable` - so you should release a new
version of the `cloud.google.com/go/bigtable` submodule.
If you need to release a change in `asset/apiv1/asset_client.go`, the closest
ancestor module is `cloud.google.com/go` - so you should release a new version
of the `cloud.google.com/go` repository root module. Note: releasing
`cloud.google.com/go` has no impact on any of the submodules, and vice-versa.
They are released entirely independently.
## Test failures
If there are any test failures in the Kokoro build, releases are blocked until
the failures have been resolved.
## How to release
### Automated Releases (`cloud.google.com/go` and submodules)
We now use [release-please](https://github.com/googleapis/release-please) to
perform automated releases for `cloud.google.com/go` and all submodules.
1. If there are changes that have not yet been released, a
[pull request](https://github.com/googleapis/google-cloud-go/pull/2971) will
be automatically opened by release-please
with a title like "chore: release X.Y.Z" (for the root module) or
"chore: release datastore X.Y.Z" (for the datastore submodule), where X.Y.Z
is the next version to be released. Find the desired pull request
[here](https://github.com/googleapis/google-cloud-go/pulls)
1. Check for failures in the
[continuous Kokoro build](http://go/google-cloud-go-continuous). If there are
any failures in the most recent build, address them before proceeding with
the release. (This applies even if the failures are in a different submodule
from the one being released.)
1. Review the release notes. These are automatically generated from the titles
of any merged commits since the previous release. If you would like to edit
them, this can be done by updating the changes in the release PR.
1. To cut a release, approve and merge the pull request. Doing so will
update the `CHANGES.md`, tag the merged commit with the appropriate version,
and draft a GitHub release which will copy the notes from `CHANGES.md`.
### Manual Release (`cloud.google.com/go`)
If for whatever reason the automated release process is not working as expected,
here is how to manually cut a release of `cloud.google.com/go`.
1. Check for failures in the
[continuous Kokoro build](http://go/google-cloud-go-continuous). If there are
any failures in the most recent build, address them before proceeding with
the release.
1. Navigate to `google-cloud-go/` and switch to main.
1. `git pull`
1. Run `git tag -l | grep -v beta | grep -v alpha` to see all existing releases.
The current latest tag `$CV` is the largest tag. It should look something
like `vX.Y.Z` (note: ignore all `LIB/vX.Y.Z` tags - these are tags for a
specific library, not the module root). We'll call the current version `$CV`
and the new version `$NV`.
1. On main, run `git log $CV...` to list all the changes since the last
release. NOTE: You must manually visually parse out changes to submodules [1]
(the `git log` is going to show you things in submodules, which are not going
to be part of your release).
1. Edit `CHANGES.md` to include a summary of the changes.
1. In `internal/version/version.go`, update `const Repo` to today's date with
the format `YYYYMMDD`.
1. In `internal/version` run `go generate`.
1. Commit the changes, ignoring the generated `.go-r` file. Push to your fork,
and create a PR titled `chore: release $NV`.
1. Wait for the PR to be reviewed and merged. Once it's merged, and without
merging any other PRs in the meantime:
a. Switch to main.
b. `git pull`
c. Tag the repo with the next version: `git tag $NV`.
d. Push the tag to origin:
`git push origin $NV`
1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases)
with the new release, copying the contents of `CHANGES.md`.
### Manual Releases (submodules)
If for whatever reason the automated release process is not working as expected,
here is how to manually cut a release of a submodule.
(these instructions assume we're releasing `cloud.google.com/go/datastore` - adjust accordingly)
1. Check for failures in the
[continuous Kokoro build](http://go/google-cloud-go-continuous). If there are
any failures in the most recent build, address them before proceeding with
the release. (This applies even if the failures are in a different submodule
from the one being released.)
1. Navigate to `google-cloud-go/` and switch to main.
1. `git pull`
1. Run `git tag -l | grep datastore | grep -v beta | grep -v alpha` to see all
existing releases. The current latest tag `$CV` is the largest tag. It
should look something like `datastore/vX.Y.Z`. We'll call the current version
`$CV` and the new version `$NV`.
1. On main, run `git log $CV.. -- datastore/` to list all the changes to the
submodule directory since the last release.
1. Edit `datastore/CHANGES.md` to include a summary of the changes.
1. In `internal/version` run `go generate`.
1. Commit the changes, ignoring the generated `.go-r` file. Push to your fork,
and create a PR titled `chore(datastore): release $NV`.
1. Wait for the PR to be reviewed and merged. Once it's merged, and without
merging any other PRs in the meantime:
a. Switch to main.
b. `git pull`
c. Tag the repo with the next version: `git tag $NV`.
d. Push the tag to origin:
`git push origin $NV`
1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases)
with the new release, copying the contents of `datastore/CHANGES.md`.
vendor/cloud.google.com/go/SECURITY.md
Generated Vendored
-7
View File
@@ -1,7 +0,0 @@
# Security Policy
To report a security issue, please use [g.co/vulnz](https://g.co/vulnz).
The Google Security Team will respond within 5 working days of your report on g.co/vulnz.
We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue.
vendor/github.com/googleapis/go-type-adapters/adapters/doc.go → vendor/cloud.google.com/go/compute/internal/version.go
Generated Vendored
+6 -5
View File
@@ -1,10 +1,10 @@
// Copyright 2021 Google LLC
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// package adapters provides helper functions for the google.type protobuf
// messages (Decimal, Fraction, etc.).
package adapters
package internal
// Version is the current tagged release of the library.
const Version = "1.14.0"
vendor/cloud.google.com/go/compute/metadata/CHANGES.md
Generated Vendored
+19
View File
@@ -0,0 +1,19 @@
# Changes
## [0.2.3](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.2.2...compute/metadata/v0.2.3) (2022-12-15)
### Bug Fixes
* **compute/metadata:** Switch DNS lookup to an absolute lookup ([119b410](https://github.com/googleapis/google-cloud-go/commit/119b41060c7895e45e48aee5621ad35607c4d021)), refs [#7165](https://github.com/googleapis/google-cloud-go/issues/7165)
## [0.2.2](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.2.1...compute/metadata/v0.2.2) (2022-12-01)
### Bug Fixes
* **compute/metadata:** Set IdleConnTimeout for http.Client ([#7084](https://github.com/googleapis/google-cloud-go/issues/7084)) ([766516a](https://github.com/googleapis/google-cloud-go/commit/766516aaf3816bfb3159efeea65aa3d1d205a3e2)), refs [#5430](https://github.com/googleapis/google-cloud-go/issues/5430)
## [0.1.0] (2022-10-26)
Initial release of metadata being it's own module.
vendor/github.com/googleapis/go-type-adapters/LICENSE → vendor/cloud.google.com/go/compute/metadata/LICENSE
Generated Vendored
+1 -1
View File
@@ -193,7 +193,7 @@
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
vendor/cloud.google.com/go/compute/metadata/README.md
Generated Vendored
+27
View File
@@ -0,0 +1,27 @@
# Compute API
[![Go Reference](https://pkg.go.dev/badge/cloud.google.com/go/compute.svg)](https://pkg.go.dev/cloud.google.com/go/compute/metadata)
This is a utility library for communicating with Google Cloud metadata service
on Google Cloud.
## Install
```bash
go get cloud.google.com/go/compute/metadata
```
## Go Version Support
See the [Go Versions Supported](https://github.com/googleapis/google-cloud-go#go-versions-supported)
section in the root directory's README.
## Contributing
Contributions are welcome. Please, see the [CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md)
document for details.
Please note that this project is released with a Contributor Code of Conduct.
By participating in this project you agree to abide by its terms. See
[Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct)
for more information.
vendor/cloud.google.com/go/compute/metadata/metadata.go
Generated Vendored
+4 -2
View File
@@ -16,7 +16,7 @@
// metadata and API service accounts.
//
// This package is a wrapper around the GCE metadata service,
// as documented at https://developers.google.com/compute/docs/metadata.
// as documented at https://cloud.google.com/compute/docs/metadata/overview.
package metadata // import "cloud.google.com/go/compute/metadata"
import (
@@ -70,7 +70,9 @@ func newDefaultHTTPClient() *http.Client {
Timeout: 2 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
IdleConnTimeout: 60 * time.Second,
},
Timeout: 5 * time.Second,
}
}
@@ -145,7 +147,7 @@ func testOnGCE() bool {
go func() {
resolver := &net.Resolver{}
addrs, err := resolver.LookupHost(ctx, "metadata.google.internal")
addrs, err := resolver.LookupHost(ctx, "metadata.google.internal.")
if err != nil || len(addrs) == 0 {
resc <- false
return
vendor/cloud.google.com/go/iam/go_mod_tidy_hack.go → vendor/cloud.google.com/go/compute/metadata/tidyfix.go
Generated Vendored
+3 -3
View File
@@ -12,12 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// This file, and the cloud.google.com/go import, won't actually become part of
// This file, and the {{.RootMod}} import, won't actually become part of
// the resultant binary.
//go:build modhack
// +build modhack
package iam
package metadata
// Necessary for safely adding multi-module repo. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository
import _ "cloud.google.com/go"
import _ "cloud.google.com/go/compute/internal"
vendor/cloud.google.com/go/doc.go
Generated Vendored
-201
View File
@@ -1,201 +0,0 @@
// Copyright 2014 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Package cloud is the root of the packages used to access Google Cloud
Services. See https://godoc.org/cloud.google.com/go for a full list
of sub-packages.
Client Options
All clients in sub-packages are configurable via client options. These options are
described here: https://godoc.org/google.golang.org/api/option.
Authentication and Authorization
All the clients in sub-packages support authentication via Google Application Default
Credentials (see https://cloud.google.com/docs/authentication/production), or
by providing a JSON key file for a Service Account. See examples below.
Google Application Default Credentials (ADC) is the recommended way to authorize
and authenticate clients. For information on how to create and obtain
Application Default Credentials, see
https://cloud.google.com/docs/authentication/production. Here is an example
of a client using ADC to authenticate:
client, err := secretmanager.NewClient(context.Background())
if err != nil {
// TODO: handle error.
}
_ = client // Use the client.
You can use a file with credentials to authenticate and authorize, such as a JSON
key file associated with a Google service account. Service Account keys can be
created and downloaded from
https://console.cloud.google.com/iam-admin/serviceaccounts. This example uses
the Secret Manger client, but the same steps apply to the other client libraries
underneath this package. Example:
client, err := secretmanager.NewClient(context.Background(),
option.WithCredentialsFile("/path/to/service-account-key.json"))
if err != nil {
// TODO: handle error.
}
_ = client // Use the client.
In some cases (for instance, you don't want to store secrets on disk), you can
create credentials from in-memory JSON and use the WithCredentials option.
The google package in this example is at golang.org/x/oauth2/google.
This example uses the Secret Manager client, but the same steps apply to
the other client libraries underneath this package. Note that scopes can be
found at https://developers.google.com/identity/protocols/oauth2/scopes, and
are also provided in all auto-generated libraries: for example,
cloud.google.com/go/secretmanager/apiv1 provides DefaultAuthScopes. Example:
ctx := context.Background()
creds, err := google.CredentialsFromJSON(ctx, []byte("JSON creds"), secretmanager.DefaultAuthScopes()...)
if err != nil {
// TODO: handle error.
}
client, err := secretmanager.NewClient(ctx, option.WithCredentials(creds))
if err != nil {
// TODO: handle error.
}
_ = client // Use the client.
Timeouts and Cancellation
By default, non-streaming methods, like Create or Get, will have a default deadline applied to the
context provided at call time, unless a context deadline is already set. Streaming
methods have no default deadline and will run indefinitely. To set timeouts or
arrange for cancellation, use contexts. Transient
errors will be retried when correctness allows.
Here is an example of how to set a timeout for an RPC, use context.WithTimeout:
ctx := context.Background()
// Do not set a timeout on the context passed to NewClient: dialing happens
// asynchronously, and the context is used to refresh credentials in the
// background.
client, err := secretmanager.NewClient(ctx)
if err != nil {
// TODO: handle error.
}
// Time out if it takes more than 10 seconds to create a dataset.
tctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel() // Always call cancel.
req := &secretmanagerpb.DeleteSecretRequest{Name: "projects/project-id/secrets/name"}
if err := client.DeleteSecret(tctx, req); err != nil {
// TODO: handle error.
}
Here is an example of how to arrange for an RPC to be canceled, use context.WithCancel:
ctx := context.Background()
// Do not cancel the context passed to NewClient: dialing happens asynchronously,
// and the context is used to refresh credentials in the background.
client, err := secretmanager.NewClient(ctx)
if err != nil {
// TODO: handle error.
}
cctx, cancel := context.WithCancel(ctx)
defer cancel() // Always call cancel.
// TODO: Make the cancel function available to whatever might want to cancel the
// call--perhaps a GUI button.
req := &secretmanagerpb.DeleteSecretRequest{Name: "projects/proj/secrets/name"}
if err := client.DeleteSecret(cctx, req); err != nil {
// TODO: handle error.
}
To opt out of default deadlines, set the temporary environment variable
GOOGLE_API_GO_EXPERIMENTAL_DISABLE_DEFAULT_DEADLINE to "true" prior to client
creation. This affects all Google Cloud Go client libraries. This opt-out
mechanism will be removed in a future release. File an issue at
https://github.com/googleapis/google-cloud-go if the default deadlines
cannot work for you.
Do not attempt to control the initial connection (dialing) of a service by setting a
timeout on the context passed to NewClient. Dialing is non-blocking, so timeouts
would be ineffective and would only interfere with credential refreshing, which uses
the same context.
Connection Pooling
Connection pooling differs in clients based on their transport. Cloud
clients either rely on HTTP or gRPC transports to communicate
with Google Cloud.
Cloud clients that use HTTP (bigquery, compute, storage, and translate) rely on the
underlying HTTP transport to cache connections for later re-use. These are cached to
the default http.MaxIdleConns and http.MaxIdleConnsPerHost settings in
http.DefaultTransport.
For gRPC clients (all others in this repo), connection pooling is configurable. Users
of cloud client libraries may specify option.WithGRPCConnectionPool(n) as a client
option to NewClient calls. This configures the underlying gRPC connections to be
pooled and addressed in a round robin fashion.
Using the Libraries with Docker
Minimal docker images like Alpine lack CA certificates. This causes RPCs to appear to
hang, because gRPC retries indefinitely. See https://github.com/googleapis/google-cloud-go/issues/928
for more information.
Debugging
To see gRPC logs, set the environment variable GRPC_GO_LOG_SEVERITY_LEVEL. See
https://godoc.org/google.golang.org/grpc/grpclog for more information.
For HTTP logging, set the GODEBUG environment variable to "http2debug=1" or "http2debug=2".
Inspecting errors
Most of the errors returned by the generated clients can be converted into a
`grpc.Status`. Converting your errors to this type can be a useful to get
more information about what went wrong while debugging.
if err != {
if s, ok := status.FromError(err); ok {
log.Println(s.Message())
for _, d := range s.Proto().Details {
log.Println(d)
}
}
}
Client Stability
Clients in this repository are considered alpha or beta unless otherwise
marked as stable in the README.md. Semver is not used to communicate stability
of clients.
Alpha and beta clients may change or go away without notice.
Clients marked stable will maintain compatibility with future versions for as
long as we can reasonably sustain. Incompatible changes might be made in some
situations, including:
- Security bugs may prompt backwards-incompatible changes.
- Situations in which components are no longer feasible to maintain without
making breaking changes, including removal.
- Parts of the client surface may be outright unstable and subject to change.
These parts of the surface will be labeled with the note, "It is EXPERIMENTAL
and subject to change or removal without notice."
*/
package cloud // import "cloud.google.com/go"
vendor/cloud.google.com/go/iam/CHANGES.md
Generated Vendored
+35
View File
@@ -1,5 +1,40 @@
# Changes
## [0.8.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.7.0...iam/v0.8.0) (2022-12-05)
### Features
* **iam:** Start generating and refresh some libraries ([#7089](https://github.com/googleapis/google-cloud-go/issues/7089)) ([a9045ff](https://github.com/googleapis/google-cloud-go/commit/a9045ff191a711089c37f1d94a63522d9939ce38))
## [0.7.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.6.0...iam/v0.7.0) (2022-11-03)
### Features
* **iam:** rewrite signatures in terms of new location ([3c4b2b3](https://github.com/googleapis/google-cloud-go/commit/3c4b2b34565795537aac1661e6af2442437e34ad))
## [0.6.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.5.0...iam/v0.6.0) (2022-10-25)
### Features
* **iam:** start generating stubs dir ([de2d180](https://github.com/googleapis/google-cloud-go/commit/de2d18066dc613b72f6f8db93ca60146dabcfdcc))
## [0.5.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.4.0...iam/v0.5.0) (2022-09-28)
### Features
* **iam:** remove ListApplicablePolicies ([52dddd1](https://github.com/googleapis/google-cloud-go/commit/52dddd1ed89fbe77e1859311c3b993a77a82bfc7))
## [0.4.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.3.0...iam/v0.4.0) (2022-09-06)
### Features
* **iam:** start generating apiv2 ([#6605](https://github.com/googleapis/google-cloud-go/issues/6605)) ([a6004e7](https://github.com/googleapis/google-cloud-go/commit/a6004e762f782869cd85688937475744f7b17e50))
## [0.3.0](https://github.com/googleapis/google-cloud-go/compare/iam/v0.2.0...iam/v0.3.0) (2022-02-23)
vendor/google.golang.org/genproto/googleapis/iam/v1/iam_policy.pb.go → vendor/cloud.google.com/go/iam/apiv1/iampb/iam_policy.pb.go
Generated Vendored
+2 -2
View File
@@ -15,10 +15,10 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.12.2
// protoc v3.21.5
// source: google/iam/v1/iam_policy.proto
package iam
package iampb
import (
context "context"
vendor/google.golang.org/genproto/googleapis/iam/v1/options.pb.go → vendor/cloud.google.com/go/iam/apiv1/iampb/options.pb.go
Generated Vendored
+2 -2
View File
@@ -15,10 +15,10 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.12.2
// protoc v3.21.5
// source: google/iam/v1/options.proto
package iam
package iampb
import (
reflect "reflect"
vendor/google.golang.org/genproto/googleapis/iam/v1/policy.pb.go → vendor/cloud.google.com/go/iam/apiv1/iampb/policy.pb.go
Generated Vendored
+126 -131
View File
@@ -15,10 +15,10 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.12.2
// protoc v3.21.5
// source: google/iam/v1/policy.proto
package iam
package iampb
import (
reflect "reflect"
@@ -203,7 +203,6 @@ func (AuditConfigDelta_Action) EnumDescriptor() ([]byte, []int) {
// An Identity and Access Management (IAM) policy, which specifies access
// controls for Google Cloud resources.
//
//
// A `Policy` is a collection of `bindings`. A `binding` binds one or more
// `members`, or principals, to a single `role`. Principals can be user
// accounts, service accounts, Google groups, and domains (such as G Suite). A
@@ -219,51 +218,51 @@ func (AuditConfigDelta_Action) EnumDescriptor() ([]byte, []int) {
//
// **JSON example:**
//
// {
// "bindings": [
// {
// "role": "roles/resourcemanager.organizationAdmin",
// "members": [
// "user:mike@example.com",
// "group:admins@example.com",
// "domain:google.com",
// "serviceAccount:my-project-id@appspot.gserviceaccount.com"
// ]
// },
// {
// "role": "roles/resourcemanager.organizationViewer",
// "members": [
// "user:eve@example.com"
// ],
// "condition": {
// "title": "expirable access",
// "description": "Does not grant access after Sep 2020",
// "expression": "request.time < timestamp('2020-10-01T00:00:00.000Z')",
// }
// }
// ],
// "etag": "BwWWja0YfJA=",
// "version": 3
// }
// {
// "bindings": [
// {
// "role": "roles/resourcemanager.organizationAdmin",
// "members": [
// "user:mike@example.com",
// "group:admins@example.com",
// "domain:google.com",
// "serviceAccount:my-project-id@appspot.gserviceaccount.com"
// ]
// },
// {
// "role": "roles/resourcemanager.organizationViewer",
// "members": [
// "user:eve@example.com"
// ],
// "condition": {
// "title": "expirable access",
// "description": "Does not grant access after Sep 2020",
// "expression": "request.time < timestamp('2020-10-01T00:00:00.000Z')",
// }
// }
// ],
// "etag": "BwWWja0YfJA=",
// "version": 3
// }
//
// **YAML example:**
//
// bindings:
// - members:
// - user:mike@example.com
// - group:admins@example.com
// - domain:google.com
// - serviceAccount:my-project-id@appspot.gserviceaccount.com
// role: roles/resourcemanager.organizationAdmin
// - members:
// - user:eve@example.com
// role: roles/resourcemanager.organizationViewer
// condition:
// title: expirable access
// description: Does not grant access after Sep 2020
// expression: request.time < timestamp('2020-10-01T00:00:00.000Z')
// etag: BwWWja0YfJA=
// version: 3
// bindings:
// - members:
// - user:mike@example.com
// - group:admins@example.com
// - domain:google.com
// - serviceAccount:my-project-id@appspot.gserviceaccount.com
// role: roles/resourcemanager.organizationAdmin
// - members:
// - user:eve@example.com
// role: roles/resourcemanager.organizationViewer
// condition:
// title: expirable access
// description: Does not grant access after Sep 2020
// expression: request.time < timestamp('2020-10-01T00:00:00.000Z')
// etag: BwWWja0YfJA=
// version: 3
//
// For a description of IAM and its features, see the
// [IAM documentation](https://cloud.google.com/iam/docs/).
@@ -280,11 +279,11 @@ type Policy struct {
// Any operation that affects conditional role bindings must specify version
// `3`. This requirement applies to the following operations:
//
// * Getting a policy that includes a conditional role binding
// * Adding a conditional role binding to a policy
// * Changing a conditional role binding in a policy
// * Removing any role binding, with or without a condition, from a policy
// that includes conditions
// - Getting a policy that includes a conditional role binding
// - Adding a conditional role binding to a policy
// - Changing a conditional role binding in a policy
// - Removing any role binding, with or without a condition, from a policy
// that includes conditions
//
// **Important:** If you use IAM Conditions, you must include the `etag` field
// whenever you call `setIamPolicy`. If you omit this field, then IAM allows
@@ -397,47 +396,43 @@ type Binding struct {
// Specifies the principals requesting access for a Cloud Platform resource.
// `members` can have the following values:
//
// * `allUsers`: A special identifier that represents anyone who is
// on the internet; with or without a Google account.
// - `allUsers`: A special identifier that represents anyone who is
// on the internet; with or without a Google account.
//
// * `allAuthenticatedUsers`: A special identifier that represents anyone
// who is authenticated with a Google account or a service account.
// - `allAuthenticatedUsers`: A special identifier that represents anyone
// who is authenticated with a Google account or a service account.
//
// * `user:{emailid}`: An email address that represents a specific Google
// account. For example, `alice@example.com` .
// - `user:{emailid}`: An email address that represents a specific Google
// account. For example, `alice@example.com` .
//
// - `serviceAccount:{emailid}`: An email address that represents a service
// account. For example, `my-other-app@appspot.gserviceaccount.com`.
//
// * `serviceAccount:{emailid}`: An email address that represents a service
// account. For example, `my-other-app@appspot.gserviceaccount.com`.
// - `group:{emailid}`: An email address that represents a Google group.
// For example, `admins@example.com`.
//
// * `group:{emailid}`: An email address that represents a Google group.
// For example, `admins@example.com`.
// - `deleted:user:{emailid}?uid={uniqueid}`: An email address (plus unique
// identifier) representing a user that has been recently deleted. For
// example, `alice@example.com?uid=123456789012345678901`. If the user is
// recovered, this value reverts to `user:{emailid}` and the recovered user
// retains the role in the binding.
//
// * `deleted:user:{emailid}?uid={uniqueid}`: An email address (plus unique
// identifier) representing a user that has been recently deleted. For
// example, `alice@example.com?uid=123456789012345678901`. If the user is
// recovered, this value reverts to `user:{emailid}` and the recovered user
// retains the role in the binding.
//
// * `deleted:serviceAccount:{emailid}?uid={uniqueid}`: An email address (plus
// unique identifier) representing a service account that has been recently
// deleted. For example,
// `my-other-app@appspot.gserviceaccount.com?uid=123456789012345678901`.
// If the service account is undeleted, this value reverts to
// `serviceAccount:{emailid}` and the undeleted service account retains the
// role in the binding.
//
// * `deleted:group:{emailid}?uid={uniqueid}`: An email address (plus unique
// identifier) representing a Google group that has been recently
// deleted. For example, `admins@example.com?uid=123456789012345678901`. If
// the group is recovered, this value reverts to `group:{emailid}` and the
// recovered group retains the role in the binding.
//
//
// * `domain:{domain}`: The G Suite domain (primary) that represents all the
// users of that domain. For example, `google.com` or `example.com`.
// - `deleted:serviceAccount:{emailid}?uid={uniqueid}`: An email address (plus
// unique identifier) representing a service account that has been recently
// deleted. For example,
// `my-other-app@appspot.gserviceaccount.com?uid=123456789012345678901`.
// If the service account is undeleted, this value reverts to
// `serviceAccount:{emailid}` and the undeleted service account retains the
// role in the binding.
//
// - `deleted:group:{emailid}?uid={uniqueid}`: An email address (plus unique
// identifier) representing a Google group that has been recently
// deleted. For example, `admins@example.com?uid=123456789012345678901`. If
// the group is recovered, this value reverts to `group:{emailid}` and the
// recovered group retains the role in the binding.
//
// - `domain:{domain}`: The G Suite domain (primary) that represents all the
// users of that domain. For example, `google.com` or `example.com`.
Members []string `protobuf:"bytes,2,rep,name=members,proto3" json:"members,omitempty"`
// The condition that is associated with this binding.
//
@@ -519,41 +514,41 @@ func (x *Binding) GetCondition() *expr.Expr {
//
// Example Policy with multiple AuditConfigs:
//
// {
// "audit_configs": [
// {
// "service": "allServices",
// "audit_log_configs": [
// {
// "log_type": "DATA_READ",
// "exempted_members": [
// "user:jose@example.com"
// ]
// },
// {
// "log_type": "DATA_WRITE"
// },
// {
// "log_type": "ADMIN_READ"
// }
// ]
// },
// {
// "service": "sampleservice.googleapis.com",
// "audit_log_configs": [
// {
// "log_type": "DATA_READ"
// },
// {
// "log_type": "DATA_WRITE",
// "exempted_members": [
// "user:aliya@example.com"
// ]
// }
// ]
// }
// ]
// }
// {
// "audit_configs": [
// {
// "service": "allServices",
// "audit_log_configs": [
// {
// "log_type": "DATA_READ",
// "exempted_members": [
// "user:jose@example.com"
// ]
// },
// {
// "log_type": "DATA_WRITE"
// },
// {
// "log_type": "ADMIN_READ"
// }
// ]
// },
// {
// "service": "sampleservice.googleapis.com",
// "audit_log_configs": [
// {
// "log_type": "DATA_READ"
// },
// {
// "log_type": "DATA_WRITE",
// "exempted_members": [
// "user:aliya@example.com"
// ]
// }
// ]
// }
// ]
// }
//
// For sampleservice, this policy enables DATA_READ, DATA_WRITE and ADMIN_READ
// logging. It also exempts jose@example.com from DATA_READ logging, and
@@ -620,19 +615,19 @@ func (x *AuditConfig) GetAuditLogConfigs() []*AuditLogConfig {
// Provides the configuration for logging a type of permissions.
// Example:
//
// {
// "audit_log_configs": [
// {
// "log_type": "DATA_READ",
// "exempted_members": [
// "user:jose@example.com"
// ]
// },
// {
// "log_type": "DATA_WRITE"
// }
// ]
// }
// {
// "audit_log_configs": [
// {
// "log_type": "DATA_READ",
// "exempted_members": [
// "user:jose@example.com"
// ]
// },
// {
// "log_type": "DATA_WRITE"
// }
// ]
// }
//
// This enables 'DATA_READ' and 'DATA_WRITE' logging, while exempting
// jose@example.com from DATA_READ logging.
vendor/cloud.google.com/go/internal/.repo-metadata-full.json
Generated Vendored
+587 -227
View File
File diff suppressed because it is too large Load Diff
vendor/cloud.google.com/go/internal/annotate.go
Generated Vendored
+2 -1
View File
@@ -31,7 +31,8 @@ import (
// - "google.golang.org/api/googleapi".Error
// If the error is not one of these types, Annotate behaves
// like
// fmt.Errorf("%s: %v", msg, err)
//
// fmt.Errorf("%s: %v", msg, err)
func Annotate(err error, msg string) error {
if err == nil {
panic("Annotate called with nil")
vendor/cloud.google.com/go/storage/.release-please-manifest.json
Generated Vendored
-3
View File
@@ -1,3 +0,0 @@
{
"storage": "1.22.0"
}
vendor/cloud.google.com/go/storage/CHANGES.md
Generated Vendored
+94
View File
@@ -1,6 +1,100 @@
# Changes
## [1.29.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.28.1...storage/v1.29.0) (2023-01-19)
### Features
* **storage:** Add ComponentCount as part of ObjectAttrs ([#7230](https://github.com/googleapis/google-cloud-go/issues/7230)) ([a19bca6](https://github.com/googleapis/google-cloud-go/commit/a19bca60704b4fbb674cf51d828580aa653c8210))
* **storage:** Add REST client ([06a54a1](https://github.com/googleapis/google-cloud-go/commit/06a54a16a5866cce966547c51e203b9e09a25bc0))
### Documentation
* **storage/internal:** Corrected typos and spellings ([7357077](https://github.com/googleapis/google-cloud-go/commit/735707796d81d7f6f32fc3415800c512fe62297e))
## [1.28.1](https://github.com/googleapis/google-cloud-go/compare/storage/v1.28.0...storage/v1.28.1) (2022-12-02)
### Bug Fixes
* **storage:** downgrade some dependencies ([7540152](https://github.com/googleapis/google-cloud-go/commit/754015236d5af7c82a75da218b71a87b9ead6eb5))
## [1.28.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.27.0...storage/v1.28.0) (2022-11-03)
### Features
* **storage/internal:** Add routing annotations ([ce3f945](https://github.com/googleapis/google-cloud-go/commit/ce3f9458e511eca0910992763232abbcd64698f1))
* **storage:** Add Autoclass support ([#6828](https://github.com/googleapis/google-cloud-go/issues/6828)) ([f7c7f41](https://github.com/googleapis/google-cloud-go/commit/f7c7f41e4d7fcffe05860e1114cb20f40c869da8))
### Bug Fixes
* **storage:** Fix read-write race in Writer.Write ([#6817](https://github.com/googleapis/google-cloud-go/issues/6817)) ([4766d3e](https://github.com/googleapis/google-cloud-go/commit/4766d3e1004119b93c6bd352024b5bf3404252eb))
* **storage:** Fix request token passing for Copier.Run ([#6863](https://github.com/googleapis/google-cloud-go/issues/6863)) ([faaab06](https://github.com/googleapis/google-cloud-go/commit/faaab066d8e509dc440bcbc87391557ecee7dbf2)), refs [#6857](https://github.com/googleapis/google-cloud-go/issues/6857)
### Documentation
* **storage:** Update broken links for SignURL and PostPolicy ([#6779](https://github.com/googleapis/google-cloud-go/issues/6779)) ([776138b](https://github.com/googleapis/google-cloud-go/commit/776138bc06a1e5fd45acbf8f9d36e9dc6ce31dd3))
## [1.27.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.26.0...storage/v1.27.0) (2022-09-22)
### Features
* **storage:** Find GoogleAccessID when using impersonated creds ([#6591](https://github.com/googleapis/google-cloud-go/issues/6591)) ([a2d16a7](https://github.com/googleapis/google-cloud-go/commit/a2d16a7a778c85d13217fc67955ec5dac1da34e8))
## [1.26.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.25.0...storage/v1.26.0) (2022-08-29)
### Features
* **storage:** export ShouldRetry ([#6370](https://github.com/googleapis/google-cloud-go/issues/6370)) ([0da9ab0](https://github.com/googleapis/google-cloud-go/commit/0da9ab0831540569dc04c0a23437b084b1564e15)), refs [#6362](https://github.com/googleapis/google-cloud-go/issues/6362)
### Bug Fixes
* **storage:** allow to use age=0 in OLM conditions ([#6204](https://github.com/googleapis/google-cloud-go/issues/6204)) ([c85704f](https://github.com/googleapis/google-cloud-go/commit/c85704f4284626ce728cb48f3b130f2ce2a0165e))
## [1.25.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.24.0...storage/v1.25.0) (2022-08-11)
### Features
* **storage/internal:** Add routing annotations ([8a8ba85](https://github.com/googleapis/google-cloud-go/commit/8a8ba85311f85701c97fd7c10f1d88b738ce423f))
* **storage:** refactor to use transport-agnostic interface ([#6465](https://github.com/googleapis/google-cloud-go/issues/6465)) ([d03c3e1](https://github.com/googleapis/google-cloud-go/commit/d03c3e15a79fe9afa1232d9c8bd4c484a9bb927e))
## [1.24.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.23.0...storage/v1.24.0) (2022-07-20)
### Features
* **storage:** add Custom Placement Config Dual Region Support ([#6294](https://github.com/googleapis/google-cloud-go/issues/6294)) ([5a8c607](https://github.com/googleapis/google-cloud-go/commit/5a8c607e3a9a3265887e27cb13f8943f3e3fa23d))
## [1.23.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.22.1...storage/v1.23.0) (2022-06-23)
### Features
* **storage:** add support for OLM Prefix/Suffix ([#5929](https://github.com/googleapis/google-cloud-go/issues/5929)) ([ec21d10](https://github.com/googleapis/google-cloud-go/commit/ec21d10d6d1b01aa97a52560319775041707690d))
* **storage:** support AbortIncompleteMultipartUpload LifecycleAction ([#5812](https://github.com/googleapis/google-cloud-go/issues/5812)) ([fdec929](https://github.com/googleapis/google-cloud-go/commit/fdec929b9da6e01dda0ab3c72544d44d6bd82bd4)), refs [#5795](https://github.com/googleapis/google-cloud-go/issues/5795)
### Bug Fixes
* **storage:** allow for Age *int64 type and int64 type ([#6230](https://github.com/googleapis/google-cloud-go/issues/6230)) ([cc7acb8](https://github.com/googleapis/google-cloud-go/commit/cc7acb8bffb31828e9e96d4834a65f9728494473))
### [1.22.1](https://github.com/googleapis/google-cloud-go/compare/storage/v1.22.0...storage/v1.22.1) (2022-05-19)
### Bug Fixes
* **storage:** bump genproto, remove deadcode ([#6059](https://github.com/googleapis/google-cloud-go/issues/6059)) ([bb10f9f](https://github.com/googleapis/google-cloud-go/commit/bb10f9faca57dc3b987e0fb601090887b3507f07))
* **storage:** remove field that no longer exists ([#6061](https://github.com/googleapis/google-cloud-go/issues/6061)) ([ee150cf](https://github.com/googleapis/google-cloud-go/commit/ee150cfd194463ddfcb59898cfb0237e47777973))
## [1.22.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.21.0...storage/v1.22.0) (2022-03-31)
vendor/cloud.google.com/go/storage/README.md
Generated Vendored
+2 -2
View File
@@ -2,7 +2,7 @@
- [About Cloud Storage](https://cloud.google.com/storage/)
- [API documentation](https://cloud.google.com/storage/docs)
- [Go client documentation](https://pkg.go.dev/cloud.google.com/go/storage)
- [Go client documentation](https://cloud.google.com/go/docs/reference/cloud.google.com/go/storage/latest)
- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/main/storage)
### Example Usage
@@ -25,7 +25,7 @@ if err != nil {
log.Fatal(err)
}
defer rc.Close()
body, err := ioutil.ReadAll(rc)
body, err := io.ReadAll(rc)
if err != nil {
log.Fatal(err)
}
vendor/cloud.google.com/go/storage/acl.go
Generated Vendored
+20 -84
View File
@@ -20,9 +20,8 @@ import (
"reflect"
"cloud.google.com/go/internal/trace"
"google.golang.org/api/googleapi"
storagepb "cloud.google.com/go/storage/internal/apiv2/stubs"
raw "google.golang.org/api/storage/v1"
storagepb "google.golang.org/genproto/googleapis/storage/v2"
)
// ACLRole is the level of access to grant.
@@ -67,6 +66,8 @@ type ProjectTeam struct {
}
// ACLHandle provides operations on an access control list for a Google Cloud Storage bucket or object.
// ACLHandle on an object operates on the latest generation of that object by default.
// Selecting a specific generation of an object is not currently supported by the client.
type ACLHandle struct {
c *Client
bucket string
@@ -119,111 +120,46 @@ func (a *ACLHandle) List(ctx context.Context) (rules []ACLRule, err error) {
}
func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) {
var acls *raw.ObjectAccessControls
var err error
err = run(ctx, func() error {
req := a.c.raw.DefaultObjectAccessControls.List(a.bucket)
a.configureCall(ctx, req)
acls, err = req.Do()
return err
}, a.retry, true)
if err != nil {
return nil, err
}
return toObjectACLRules(acls.Items), nil
opts := makeStorageOpts(true, a.retry, a.userProject)
return a.c.tc.ListDefaultObjectACLs(ctx, a.bucket, opts...)
}
func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error {
req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity))
a.configureCall(ctx, req)
return run(ctx, func() error {
return req.Do()
}, a.retry, false)
opts := makeStorageOpts(false, a.retry, a.userProject)
return a.c.tc.DeleteDefaultObjectACL(ctx, a.bucket, entity, opts...)
}
func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) {
var acls *raw.BucketAccessControls
var err error
err = run(ctx, func() error {
req := a.c.raw.BucketAccessControls.List(a.bucket)
a.configureCall(ctx, req)
acls, err = req.Do()
return err
}, a.retry, true)
if err != nil {
return nil, err
}
return toBucketACLRules(acls.Items), nil
opts := makeStorageOpts(true, a.retry, a.userProject)
return a.c.tc.ListBucketACLs(ctx, a.bucket, opts...)
}
func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRole) error {
acl := &raw.BucketAccessControl{
Bucket: a.bucket,
Entity: string(entity),
Role: string(role),
}
req := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl)
a.configureCall(ctx, req)
return run(ctx, func() error {
_, err := req.Do()
return err
}, a.retry, false)
opts := makeStorageOpts(false, a.retry, a.userProject)
return a.c.tc.UpdateBucketACL(ctx, a.bucket, entity, role, opts...)
}
func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error {
req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity))
a.configureCall(ctx, req)
return run(ctx, func() error {
return req.Do()
}, a.retry, false)
opts := makeStorageOpts(false, a.retry, a.userProject)
return a.c.tc.DeleteBucketACL(ctx, a.bucket, entity, opts...)
}
func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) {
var acls *raw.ObjectAccessControls
var err error
err = run(ctx, func() error {
req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object)
a.configureCall(ctx, req)
acls, err = req.Do()
return err
}, a.retry, true)
if err != nil {
return nil, err
}
return toObjectACLRules(acls.Items), nil
opts := makeStorageOpts(true, a.retry, a.userProject)
return a.c.tc.ListObjectACLs(ctx, a.bucket, a.object, opts...)
}
func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRole, isBucketDefault bool) error {
type setRequest interface {
Do(opts ...googleapi.CallOption) (*raw.ObjectAccessControl, error)
Header() http.Header
}
acl := &raw.ObjectAccessControl{
Bucket: a.bucket,
Entity: string(entity),
Role: string(role),
}
var req setRequest
opts := makeStorageOpts(false, a.retry, a.userProject)
if isBucketDefault {
req = a.c.raw.DefaultObjectAccessControls.Update(a.bucket, string(entity), acl)
} else {
req = a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl)
return a.c.tc.UpdateDefaultObjectACL(ctx, a.bucket, entity, role, opts...)
}
a.configureCall(ctx, req)
return run(ctx, func() error {
_, err := req.Do()
return err
}, a.retry, false)
return a.c.tc.UpdateObjectACL(ctx, a.bucket, a.object, entity, role, opts...)
}
func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error {
req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity))
a.configureCall(ctx, req)
return run(ctx, func() error {
return req.Do()
}, a.retry, false)
opts := makeStorageOpts(false, a.retry, a.userProject)
return a.c.tc.DeleteObjectACL(ctx, a.bucket, a.object, entity, opts...)
}
func (a *ACLHandle) configureCall(ctx context.Context, call interface{ Header() http.Header }) {
vendor/cloud.google.com/go/storage/bucket.go
Generated Vendored
+425 -311
View File
File diff suppressed because it is too large Load Diff
vendor/cloud.google.com/go/storage/client.go
Generated Vendored
+122 -30
View File
@@ -16,6 +16,8 @@ package storage
import (
"context"
"io"
"time"
gax "github.com/googleapis/gax-go/v2"
"google.golang.org/api/option"
@@ -42,49 +44,49 @@ type storageClient interface {
// Top-level methods.
GetServiceAccount(ctx context.Context, project string, opts ...storageOption) (string, error)
CreateBucket(ctx context.Context, project string, attrs *BucketAttrs, opts ...storageOption) (*BucketAttrs, error)
ListBuckets(ctx context.Context, project string, opts ...storageOption) (*BucketIterator, error)
CreateBucket(ctx context.Context, project, bucket string, attrs *BucketAttrs, opts ...storageOption) (*BucketAttrs, error)
ListBuckets(ctx context.Context, project string, opts ...storageOption) *BucketIterator
Close() error
// Bucket methods.
DeleteBucket(ctx context.Context, bucket string, conds *BucketConditions, opts ...storageOption) error
GetBucket(ctx context.Context, bucket string, conds *BucketConditions, opts ...storageOption) (*BucketAttrs, error)
UpdateBucket(ctx context.Context, uattrs *BucketAttrsToUpdate, conds *BucketConditions, opts ...storageOption) (*BucketAttrs, error)
UpdateBucket(ctx context.Context, bucket string, uattrs *BucketAttrsToUpdate, conds *BucketConditions, opts ...storageOption) (*BucketAttrs, error)
LockBucketRetentionPolicy(ctx context.Context, bucket string, conds *BucketConditions, opts ...storageOption) error
ListObjects(ctx context.Context, bucket string, q *Query, opts ...storageOption) (*ObjectIterator, error)
ListObjects(ctx context.Context, bucket string, q *Query, opts ...storageOption) *ObjectIterator
// Object metadata methods.
DeleteObject(ctx context.Context, bucket, object string, conds *Conditions, opts ...storageOption) error
GetObject(ctx context.Context, bucket, object string, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error)
UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error)
DeleteObject(ctx context.Context, bucket, object string, gen int64, conds *Conditions, opts ...storageOption) error
GetObject(ctx context.Context, bucket, object string, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error)
UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error)
// Default Object ACL methods.
DeleteDefaultObjectACL(ctx context.Context, bucket string, entity ACLEntity, opts ...storageOption) error
ListDefaultObjectACLs(ctx context.Context, bucket string, opts ...storageOption) ([]ACLRule, error)
UpdateDefaultObjectACL(ctx context.Context, opts ...storageOption) (*ACLRule, error)
UpdateDefaultObjectACL(ctx context.Context, bucket string, entity ACLEntity, role ACLRole, opts ...storageOption) error
// Bucket ACL methods.
DeleteBucketACL(ctx context.Context, bucket string, entity ACLEntity, opts ...storageOption) error
ListBucketACLs(ctx context.Context, bucket string, opts ...storageOption) ([]ACLRule, error)
UpdateBucketACL(ctx context.Context, bucket string, entity ACLEntity, role ACLRole, opts ...storageOption) (*ACLRule, error)
UpdateBucketACL(ctx context.Context, bucket string, entity ACLEntity, role ACLRole, opts ...storageOption) error
// Object ACL methods.
DeleteObjectACL(ctx context.Context, bucket, object string, entity ACLEntity, opts ...storageOption) error
ListObjectACLs(ctx context.Context, bucket, object string, opts ...storageOption) ([]ACLRule, error)
UpdateObjectACL(ctx context.Context, bucket, object string, entity ACLEntity, role ACLRole, opts ...storageOption) (*ACLRule, error)
UpdateObjectACL(ctx context.Context, bucket, object string, entity ACLEntity, role ACLRole, opts ...storageOption) error
// Media operations.
ComposeObject(ctx context.Context, req *composeObjectRequest, opts ...storageOption) (*ObjectAttrs, error)
RewriteObject(ctx context.Context, req *rewriteObjectRequest, opts ...storageOption) (*rewriteObjectResponse, error)
OpenReader(ctx context.Context, r *Reader, opts ...storageOption) error
OpenWriter(ctx context.Context, w *Writer, opts ...storageOption) error
NewRangeReader(ctx context.Context, params *newRangeReaderParams, opts ...storageOption) (*Reader, error)
OpenWriter(params *openWriterParams, opts ...storageOption) (*io.PipeWriter, error)
// IAM methods.
@@ -94,11 +96,16 @@ type storageClient interface {
// HMAC Key methods.
GetHMACKey(ctx context.Context, desc *hmacKeyDesc, opts ...storageOption) (*HMACKey, error)
ListHMACKey(ctx context.Context, desc *hmacKeyDesc, opts ...storageOption) *HMACKeysIterator
UpdateHMACKey(ctx context.Context, desc *hmacKeyDesc, attrs *HMACKeyAttrsToUpdate, opts ...storageOption) (*HMACKey, error)
CreateHMACKey(ctx context.Context, desc *hmacKeyDesc, opts ...storageOption) (*HMACKey, error)
DeleteHMACKey(ctx context.Context, desc *hmacKeyDesc, opts ...storageOption) error
GetHMACKey(ctx context.Context, project, accessID string, opts ...storageOption) (*HMACKey, error)
ListHMACKeys(ctx context.Context, project, serviceAccountEmail string, showDeletedKeys bool, opts ...storageOption) *HMACKeysIterator
UpdateHMACKey(ctx context.Context, project, serviceAccountEmail, accessID string, attrs *HMACKeyAttrsToUpdate, opts ...storageOption) (*HMACKey, error)
CreateHMACKey(ctx context.Context, project, serviceAccountEmail string, opts ...storageOption) (*HMACKey, error)
DeleteHMACKey(ctx context.Context, project, accessID string, opts ...storageOption) error
// Notification methods.
ListNotifications(ctx context.Context, bucket string, opts ...storageOption) (map[string]*Notification, error)
CreateNotification(ctx context.Context, bucket string, n *Notification, opts ...storageOption) (*Notification, error)
DeleteNotification(ctx context.Context, bucket string, id string, opts ...storageOption) error
}
// settings contains transport-agnostic configuration for API calls made via
@@ -155,6 +162,20 @@ func callSettings(defaults *settings, opts ...storageOption) *settings {
return &cs
}
// makeStorageOpts is a helper for generating a set of storageOption based on
// idempotency, retryConfig, and userProject. All top-level client operations
// will generally have to pass these options through the interface.
func makeStorageOpts(isIdempotent bool, retry *retryConfig, userProject string) []storageOption {
opts := []storageOption{idempotent(isIdempotent)}
if retry != nil {
opts = append(opts, withRetryConfig(retry))
}
if userProject != "" {
opts = append(opts, withUserProject(userProject))
}
return opts
}
// storageOption is the transport-agnostic call option for the storageClient
// interface.
type storageOption interface {
@@ -211,31 +232,102 @@ type userProjectOption struct {
func (o *userProjectOption) Apply(s *settings) { s.userProject = o.project }
type openWriterParams struct {
// Writer configuration
// ctx is the context used by the writer routine to make all network calls
// and to manage the writer routine - see `Writer.ctx`.
// Required.
ctx context.Context
// chunkSize - see `Writer.ChunkSize`.
// Optional.
chunkSize int
// chunkRetryDeadline - see `Writer.ChunkRetryDeadline`.
// Optional.
chunkRetryDeadline time.Duration
// Object/request properties
// bucket - see `Writer.o.bucket`.
// Required.
bucket string
// attrs - see `Writer.ObjectAttrs`.
// Required.
attrs *ObjectAttrs
// conds - see `Writer.o.conds`.
// Optional.
conds *Conditions
// encryptionKey - see `Writer.o.encryptionKey`
// Optional.
encryptionKey []byte
// sendCRC32C - see `Writer.SendCRC32C`.
// Optional.
sendCRC32C bool
// Writer callbacks
// donec - see `Writer.donec`.
// Required.
donec chan struct{}
// setError callback for reporting errors - see `Writer.error`.
// Required.
setError func(error)
// progress callback for reporting upload progress - see `Writer.progress`.
// Required.
progress func(int64)
// setObj callback for reporting the resulting object - see `Writer.obj`.
// Required.
setObj func(*ObjectAttrs)
}
type newRangeReaderParams struct {
bucket string
conds *Conditions
encryptionKey []byte
gen int64
length int64
object string
offset int64
readCompressed bool // Use accept-encoding: gzip. Only works for HTTP currently.
}
type composeObjectRequest struct {
dstBucket string
dstObject string
srcs []string
dstObject destinationObject
srcs []sourceObject
predefinedACL string
sendCRC32C bool
}
type sourceObject struct {
name string
bucket string
gen int64
conds *Conditions
predefinedACL string
encryptionKey []byte
}
type destinationObject struct {
name string
bucket string
conds *Conditions
attrs *ObjectAttrs // attrs to set on the destination object.
encryptionKey []byte
keyName string
}
type rewriteObjectRequest struct {
srcBucket string
srcObject string
dstBucket string
dstObject string
dstKeyName string
attrs *ObjectAttrs
gen int64
conds *Conditions
predefinedACL string
token string
srcObject sourceObject
dstObject destinationObject
predefinedACL string
token string
maxBytesRewrittenPerCall int64
}
type rewriteObjectResponse struct {
resource *ObjectAttrs
done bool
written int64
size int64
token string
}
vendor/cloud.google.com/go/storage/copy.go
Generated Vendored
+71 -82
View File
@@ -20,7 +20,6 @@ import (
"fmt"
"cloud.google.com/go/internal/trace"
raw "google.golang.org/api/storage/v1"
)
// CopierFrom creates a Copier that can copy src to dst.
@@ -70,6 +69,15 @@ type Copier struct {
DestinationKMSKeyName string
dst, src *ObjectHandle
// The maximum number of bytes that will be rewritten per rewrite request.
// Most callers shouldn't need to specify this parameter - it is primarily
// in place to support testing. If specified the value must be an integral
// multiple of 1 MiB (1048576). Also, this only applies to requests where
// the source and destination span locations and/or storage classes. Finally,
// this value must not change across rewrite calls else you'll get an error
// that the `rewriteToken` is invalid.
maxBytesRewrittenPerCall int64
}
// Run performs the copy.
@@ -86,69 +94,59 @@ func (c *Copier) Run(ctx context.Context) (attrs *ObjectAttrs, err error) {
if c.DestinationKMSKeyName != "" && c.dst.encryptionKey != nil {
return nil, errors.New("storage: cannot use DestinationKMSKeyName with a customer-supplied encryption key")
}
if c.dst.gen != defaultGen {
return nil, fmt.Errorf("storage: generation cannot be specified on copy destination, got %v", c.dst.gen)
}
// Convert destination attributes to raw form, omitting the bucket.
// If the bucket is included but name or content-type aren't, the service
// returns a 400 with "Required" as the only message. Omitting the bucket
// does not cause any problems.
rawObject := c.ObjectAttrs.toRawObject("")
req := &rewriteObjectRequest{
srcObject: sourceObject{
name: c.src.object,
bucket: c.src.bucket,
gen: c.src.gen,
conds: c.src.conds,
encryptionKey: c.src.encryptionKey,
},
dstObject: destinationObject{
name: c.dst.object,
bucket: c.dst.bucket,
conds: c.dst.conds,
attrs: &c.ObjectAttrs,
encryptionKey: c.dst.encryptionKey,
keyName: c.DestinationKMSKeyName,
},
predefinedACL: c.PredefinedACL,
token: c.RewriteToken,
maxBytesRewrittenPerCall: c.maxBytesRewrittenPerCall,
}
isIdempotent := c.dst.conds != nil && (c.dst.conds.GenerationMatch != 0 || c.dst.conds.DoesNotExist)
var userProject string
if c.dst.userProject != "" {
userProject = c.dst.userProject
} else if c.src.userProject != "" {
userProject = c.src.userProject
}
opts := makeStorageOpts(isIdempotent, c.dst.retry, userProject)
for {
res, err := c.callRewrite(ctx, rawObject)
res, err := c.dst.c.tc.RewriteObject(ctx, req, opts...)
if err != nil {
return nil, err
}
c.RewriteToken = res.token
req.token = res.token
if c.ProgressFunc != nil {
c.ProgressFunc(uint64(res.TotalBytesRewritten), uint64(res.ObjectSize))
c.ProgressFunc(uint64(res.written), uint64(res.size))
}
if res.Done { // Finished successfully.
return newObject(res.Resource), nil
if res.done { // Finished successfully.
return res.resource, nil
}
}
}
func (c *Copier) callRewrite(ctx context.Context, rawObj *raw.Object) (*raw.RewriteResponse, error) {
call := c.dst.c.raw.Objects.Rewrite(c.src.bucket, c.src.object, c.dst.bucket, c.dst.object, rawObj)
call.Context(ctx).Projection("full")
if c.RewriteToken != "" {
call.RewriteToken(c.RewriteToken)
}
if c.DestinationKMSKeyName != "" {
call.DestinationKmsKeyName(c.DestinationKMSKeyName)
}
if c.PredefinedACL != "" {
call.DestinationPredefinedAcl(c.PredefinedACL)
}
if err := applyConds("Copy destination", c.dst.gen, c.dst.conds, call); err != nil {
return nil, err
}
if c.dst.userProject != "" {
call.UserProject(c.dst.userProject)
} else if c.src.userProject != "" {
call.UserProject(c.src.userProject)
}
if err := applySourceConds(c.src.gen, c.src.conds, call); err != nil {
return nil, err
}
if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
return nil, err
}
if err := setEncryptionHeaders(call.Header(), c.src.encryptionKey, true); err != nil {
return nil, err
}
var res *raw.RewriteResponse
var err error
setClientHeader(call.Header())
retryCall := func() error { res, err = call.Do(); return err }
isIdempotent := c.dst.conds != nil && (c.dst.conds.GenerationMatch != 0 || c.dst.conds.DoesNotExist)
if err := run(ctx, retryCall, c.dst.retry, isIdempotent); err != nil {
return nil, err
}
c.RewriteToken = res.RewriteToken
return res, nil
}
// ComposerFrom creates a Composer that can compose srcs into dst.
// You can immediately call Run on the returned Composer, or you can
// configure it first.
@@ -188,17 +186,13 @@ func (c *Composer) Run(ctx context.Context) (attrs *ObjectAttrs, err error) {
if err := c.dst.validate(); err != nil {
return nil, err
}
if c.dst.gen != defaultGen {
return nil, fmt.Errorf("storage: generation cannot be specified on compose destination, got %v", c.dst.gen)
}
if len(c.srcs) == 0 {
return nil, errors.New("storage: at least one source object must be specified")
}
req := &raw.ComposeRequest{}
// Compose requires a non-empty Destination, so we always set it,
// even if the caller-provided ObjectAttrs is the zero value.
req.Destination = c.ObjectAttrs.toRawObject(c.dst.bucket)
if c.SendCRC32C {
req.Destination.Crc32c = encodeUint32(c.ObjectAttrs.CRC32C)
}
for _, src := range c.srcs {
if err := src.validate(); err != nil {
return nil, err
@@ -209,36 +203,31 @@ func (c *Composer) Run(ctx context.Context) (attrs *ObjectAttrs, err error) {
if src.encryptionKey != nil {
return nil, fmt.Errorf("storage: compose source %s.%s must not have encryption key", src.bucket, src.object)
}
srcObj := &raw.ComposeRequestSourceObjects{
Name: src.object,
}
if err := applyConds("ComposeFrom source", src.gen, src.conds, composeSourceObj{srcObj}); err != nil {
return nil, err
}
req.SourceObjects = append(req.SourceObjects, srcObj)
}
call := c.dst.c.raw.Objects.Compose(c.dst.bucket, c.dst.object, req).Context(ctx)
if err := applyConds("ComposeFrom destination", c.dst.gen, c.dst.conds, call); err != nil {
return nil, err
req := &composeObjectRequest{
dstBucket: c.dst.bucket,
predefinedACL: c.PredefinedACL,
sendCRC32C: c.SendCRC32C,
}
if c.dst.userProject != "" {
call.UserProject(c.dst.userProject)
req.dstObject = destinationObject{
name: c.dst.object,
bucket: c.dst.bucket,
conds: c.dst.conds,
attrs: &c.ObjectAttrs,
encryptionKey: c.dst.encryptionKey,
}
if c.PredefinedACL != "" {
call.DestinationPredefinedAcl(c.PredefinedACL)
for _, src := range c.srcs {
s := sourceObject{
name: src.object,
bucket: src.bucket,
gen: src.gen,
conds: src.conds,
}
req.srcs = append(req.srcs, s)
}
if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
return nil, err
}
var obj *raw.Object
setClientHeader(call.Header())
retryCall := func() error { obj, err = call.Do(); return err }
isIdempotent := c.dst.conds != nil && (c.dst.conds.GenerationMatch != 0 || c.dst.conds.DoesNotExist)
if err := run(ctx, retryCall, c.dst.retry, isIdempotent); err != nil {
return nil, err
}
return newObject(obj), nil
opts := makeStorageOpts(isIdempotent, c.dst.retry, c.dst.userProject)
return c.dst.c.tc.ComposeObject(ctx, req, opts...)
}
vendor/cloud.google.com/go/storage/doc.go
Generated Vendored
+174 -147
View File
@@ -22,185 +22,183 @@ https://cloud.google.com/storage/docs.
See https://pkg.go.dev/cloud.google.com/go for authentication, timeouts,
connection pooling and similar aspects of this package.
# Creating a Client
Creating a Client
To start working with this package, create a [Client]:
To start working with this package, create a client:
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
// TODO: Handle error.
}
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
// TODO: Handle error.
}
The client will use your default application credentials. Clients should be
reused instead of created as needed. The methods of Client are safe for
reused instead of created as needed. The methods of [Client] are safe for
concurrent use by multiple goroutines.
If you only wish to access public data, you can create
an unauthenticated client with
client, err := storage.NewClient(ctx, option.WithoutAuthentication())
client, err := storage.NewClient(ctx, option.WithoutAuthentication())
To use an emulator with this library, you can set the STORAGE_EMULATOR_HOST
environment variable to the address at which your emulator is running. This will
send requests to that address instead of to Cloud Storage. You can then create
and use a client as usual:
// Set STORAGE_EMULATOR_HOST environment variable.
err := os.Setenv("STORAGE_EMULATOR_HOST", "localhost:9000")
if err != nil {
// TODO: Handle error.
}
// Set STORAGE_EMULATOR_HOST environment variable.
err := os.Setenv("STORAGE_EMULATOR_HOST", "localhost:9000")
if err != nil {
// TODO: Handle error.
}
// Create client as usual.
client, err := storage.NewClient(ctx)
if err != nil {
// TODO: Handle error.
}
// Create client as usual.
client, err := storage.NewClient(ctx)
if err != nil {
// TODO: Handle error.
}
// This request is now directed to http://localhost:9000/storage/v1/b
// instead of https://storage.googleapis.com/storage/v1/b
if err := client.Bucket("my-bucket").Create(ctx, projectID, nil); err != nil {
// TODO: Handle error.
}
// This request is now directed to http://localhost:9000/storage/v1/b
// instead of https://storage.googleapis.com/storage/v1/b
if err := client.Bucket("my-bucket").Create(ctx, projectID, nil); err != nil {
// TODO: Handle error.
}
Please note that there is no official emulator for Cloud Storage.
Buckets
# Buckets
A Google Cloud Storage bucket is a collection of objects. To work with a
bucket, make a bucket handle:
bkt := client.Bucket(bucketName)
bkt := client.Bucket(bucketName)
A handle is a reference to a bucket. You can have a handle even if the
bucket doesn't exist yet. To create a bucket in Google Cloud Storage,
call Create on the handle:
call [BucketHandle.Create]:
if err := bkt.Create(ctx, projectID, nil); err != nil {
// TODO: Handle error.
}
if err := bkt.Create(ctx, projectID, nil); err != nil {
// TODO: Handle error.
}
Note that although buckets are associated with projects, bucket names are
global across all projects.
Each bucket has associated metadata, represented in this package by
BucketAttrs. The third argument to BucketHandle.Create allows you to set
the initial BucketAttrs of a bucket. To retrieve a bucket's attributes, use
Attrs:
[BucketAttrs]. The third argument to [BucketHandle.Create] allows you to set
the initial [BucketAttrs] of a bucket. To retrieve a bucket's attributes, use
[BucketHandle.Attrs]:
attrs, err := bkt.Attrs(ctx)
if err != nil {
// TODO: Handle error.
}
fmt.Printf("bucket %s, created at %s, is located in %s with storage class %s\n",
attrs.Name, attrs.Created, attrs.Location, attrs.StorageClass)
attrs, err := bkt.Attrs(ctx)
if err != nil {
// TODO: Handle error.
}
fmt.Printf("bucket %s, created at %s, is located in %s with storage class %s\n",
attrs.Name, attrs.Created, attrs.Location, attrs.StorageClass)
Objects
# Objects
An object holds arbitrary data as a sequence of bytes, like a file. You
refer to objects using a handle, just as with buckets, but unlike buckets
you don't explicitly create an object. Instead, the first time you write
to an object it will be created. You can use the standard Go io.Reader
and io.Writer interfaces to read and write object data:
to an object it will be created. You can use the standard Go [io.Reader]
and [io.Writer] interfaces to read and write object data:
obj := bkt.Object("data")
// Write something to obj.
// w implements io.Writer.
w := obj.NewWriter(ctx)
// Write some text to obj. This will either create the object or overwrite whatever is there already.
if _, err := fmt.Fprintf(w, "This object contains text.\n"); err != nil {
// TODO: Handle error.
}
// Close, just like writing a file.
if err := w.Close(); err != nil {
// TODO: Handle error.
}
obj := bkt.Object("data")
// Write something to obj.
// w implements io.Writer.
w := obj.NewWriter(ctx)
// Write some text to obj. This will either create the object or overwrite whatever is there already.
if _, err := fmt.Fprintf(w, "This object contains text.\n"); err != nil {
// TODO: Handle error.
}
// Close, just like writing a file.
if err := w.Close(); err != nil {
// TODO: Handle error.
}
// Read it back.
r, err := obj.NewReader(ctx)
if err != nil {
// TODO: Handle error.
}
defer r.Close()
if _, err := io.Copy(os.Stdout, r); err != nil {
// TODO: Handle error.
}
// Prints "This object contains text."
// Read it back.
r, err := obj.NewReader(ctx)
if err != nil {
// TODO: Handle error.
}
defer r.Close()
if _, err := io.Copy(os.Stdout, r); err != nil {
// TODO: Handle error.
}
// Prints "This object contains text."
Objects also have attributes, which you can fetch with Attrs:
Objects also have attributes, which you can fetch with [ObjectHandle.Attrs]:
objAttrs, err := obj.Attrs(ctx)
if err != nil {
// TODO: Handle error.
}
fmt.Printf("object %s has size %d and can be read using %s\n",
objAttrs.Name, objAttrs.Size, objAttrs.MediaLink)
objAttrs, err := obj.Attrs(ctx)
if err != nil {
// TODO: Handle error.
}
fmt.Printf("object %s has size %d and can be read using %s\n",
objAttrs.Name, objAttrs.Size, objAttrs.MediaLink)
Listing objects
# Listing objects
Listing objects in a bucket is done with the Bucket.Objects method:
Listing objects in a bucket is done with the [BucketHandle.Objects] method:
query := &storage.Query{Prefix: ""}
query := &storage.Query{Prefix: ""}
var names []string
it := bkt.Objects(ctx, query)
for {
attrs, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
log.Fatal(err)
}
names = append(names, attrs.Name)
}
var names []string
it := bkt.Objects(ctx, query)
for {
attrs, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
log.Fatal(err)
}
names = append(names, attrs.Name)
}
Objects are listed lexicographically by name. To filter objects
lexicographically, Query.StartOffset and/or Query.EndOffset can be used:
lexicographically, [Query.StartOffset] and/or [Query.EndOffset] can be used:
query := &storage.Query{
Prefix: "",
StartOffset: "bar/", // Only list objects lexicographically >= "bar/"
EndOffset: "foo/", // Only list objects lexicographically < "foo/"
}
query := &storage.Query{
Prefix: "",
StartOffset: "bar/", // Only list objects lexicographically >= "bar/"
EndOffset: "foo/", // Only list objects lexicographically < "foo/"
}
// ... as before
// ... as before
If only a subset of object attributes is needed when listing, specifying this
subset using Query.SetAttrSelection may speed up the listing process:
subset using [Query.SetAttrSelection] may speed up the listing process:
query := &storage.Query{Prefix: ""}
query.SetAttrSelection([]string{"Name"})
query := &storage.Query{Prefix: ""}
query.SetAttrSelection([]string{"Name"})
// ... as before
// ... as before
ACLs
# ACLs
Both objects and buckets have ACLs (Access Control Lists). An ACL is a list of
ACLRules, each of which specifies the role of a user, group or project. ACLs
are suitable for fine-grained control, but you may prefer using IAM to control
access at the project level (see
https://cloud.google.com/storage/docs/access-control/iam).
access at the project level (see [Cloud Storage IAM docs].
To list the ACLs of a bucket or object, obtain an ACLHandle and call its List method:
To list the ACLs of a bucket or object, obtain an [ACLHandle] and call [ACLHandle.List]:
acls, err := obj.ACL().List(ctx)
if err != nil {
// TODO: Handle error.
}
for _, rule := range acls {
fmt.Printf("%s has role %s\n", rule.Entity, rule.Role)
}
acls, err := obj.ACL().List(ctx)
if err != nil {
// TODO: Handle error.
}
for _, rule := range acls {
fmt.Printf("%s has role %s\n", rule.Entity, rule.Role)
}
You can also set and delete ACLs.
Conditions
# Conditions
Every object has a generation and a metageneration. The generation changes
whenever the content changes, and the metageneration changes whenever the
metadata changes. Conditions let you check these values before an operation;
metadata changes. [Conditions] let you check these values before an operation;
the operation only executes if the conditions match. You can use conditions to
prevent race conditions in read-modify-write operations.
@@ -208,60 +206,81 @@ For example, say you've read an object's metadata into objAttrs. Now
you want to write to that object, but only if its contents haven't changed
since you read it. Here is how to express that:
w = obj.If(storage.Conditions{GenerationMatch: objAttrs.Generation}).NewWriter(ctx)
// Proceed with writing as above.
w = obj.If(storage.Conditions{GenerationMatch: objAttrs.Generation}).NewWriter(ctx)
// Proceed with writing as above.
Signed URLs
# Signed URLs
You can obtain a URL that lets anyone read or write an object for a limited time.
Signing a URL requires credentials authorized to sign a URL. To use the same
authentication that was used when instantiating the Storage client, use the
BucketHandle.SignedURL method.
authentication that was used when instantiating the Storage client, use
[BucketHandle.SignedURL].
url, err := client.Bucket(bucketName).SignedURL(objectName, opts)
if err != nil {
// TODO: Handle error.
}
fmt.Println(url)
url, err := client.Bucket(bucketName).SignedURL(objectName, opts)
if err != nil {
// TODO: Handle error.
}
fmt.Println(url)
You can also sign a URL wihout creating a client. See the documentation of
SignedURL for details.
You can also sign a URL without creating a client. See the documentation of
[SignedURL] for details.
url, err := storage.SignedURL(bucketName, "shared-object", opts)
if err != nil {
// TODO: Handle error.
}
fmt.Println(url)
url, err := storage.SignedURL(bucketName, "shared-object", opts)
if err != nil {
// TODO: Handle error.
}
fmt.Println(url)
Post Policy V4 Signed Request
# Post Policy V4 Signed Request
A type of signed request that allows uploads through HTML forms directly to Cloud Storage with
temporary permission. Conditions can be applied to restrict how the HTML form is used and exercised
by a user.
For more information, please see https://cloud.google.com/storage/docs/xml-api/post-object as well
as the documentation of BucketHandle.GenerateSignedPostPolicyV4.
For more information, please see the [XML POST Object docs] as well
as the documentation of [BucketHandle.GenerateSignedPostPolicyV4].
pv4, err := client.Bucket(bucketName).GenerateSignedPostPolicyV4(objectName, opts)
if err != nil {
// TODO: Handle error.
}
fmt.Printf("URL: %s\nFields; %v\n", pv4.URL, pv4.Fields)
pv4, err := client.Bucket(bucketName).GenerateSignedPostPolicyV4(objectName, opts)
if err != nil {
// TODO: Handle error.
}
fmt.Printf("URL: %s\nFields; %v\n", pv4.URL, pv4.Fields)
Errors
# Credential requirements for signing
Errors returned by this client are often of the type googleapi.Error.
These errors can be introspected for more information by using errors.As
with the richer googleapi.Error type. For example:
If the GoogleAccessID and PrivateKey option fields are not provided, they will
be automatically detected by [BucketHandle.SignedURL] and
[BucketHandle.GenerateSignedPostPolicyV4] if any of the following are true:
- you are authenticated to the Storage Client with a service account's
downloaded private key, either directly in code or by setting the
GOOGLE_APPLICATION_CREDENTIALS environment variable (see [Other Environments]),
- your application is running on Google Compute Engine (GCE), or
- you are logged into [gcloud using application default credentials]
with [impersonation enabled].
Detecting GoogleAccessID may not be possible if you are authenticated using a
token source or using [option.WithHTTPClient]. In this case, you can provide a
service account email for GoogleAccessID and the client will attempt to sign
the URL or Post Policy using that service account.
To generate the signature, you must have:
- iam.serviceAccounts.signBlob permissions on the GoogleAccessID service
account, and
- the [IAM Service Account Credentials API] enabled (unless authenticating
with a downloaded private key).
# Errors
Errors returned by this client are often of the type [googleapi.Error].
These errors can be introspected for more information by using [errors.As]
with the richer [googleapi.Error] type. For example:
var e *googleapi.Error
if ok := errors.As(err, &e); ok {
if e.Code == 409 { ... }
}
See https://pkg.go.dev/google.golang.org/api/googleapi#Error for more information.
Retrying failed requests
# Retrying failed requests
Methods in this package may retry calls that fail with transient errors.
Retrying continues indefinitely unless the controlling context is canceled, the
@@ -271,12 +290,12 @@ continuing, use context timeouts or cancellation.
The retry strategy in this library follows best practices for Cloud Storage. By
default, operations are retried only if they are idempotent, and exponential
backoff with jitter is employed. In addition, errors are only retried if they
are defined as transient by the service. See
https://cloud.google.com/storage/docs/retry-strategy for more information.
are defined as transient by the service. See the [Cloud Storage retry docs]
for more information.
Users can configure non-default retry behavior for a single library call (using
BucketHandle.Retryer and ObjectHandle.Retryer) or for all calls made by a
client (using Client.SetRetry). For example:
[BucketHandle.Retryer] and [ObjectHandle.Retryer]) or for all calls made by a
client (using [Client.SetRetry]). For example:
o := client.Bucket(bucket).Object(object).Retryer(
// Use WithBackoff to change the timing of the exponential backoff.
@@ -297,5 +316,13 @@ client (using Client.SetRetry). For example:
if err := o.Delete(ctx); err != nil {
// Handle err.
}
[Cloud Storage IAM docs]: https://cloud.google.com/storage/docs/access-control/iam
[XML POST Object docs]: https://cloud.google.com/storage/docs/xml-api/post-object
[Cloud Storage retry docs]: https://cloud.google.com/storage/docs/retry-strategy
[Other Environments]: https://cloud.google.com/storage/docs/authentication#libauth
[gcloud using application default credentials]: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login
[impersonation enabled]: https://cloud.google.com/sdk/gcloud/reference#--impersonate-service-account
[IAM Service Account Credentials API]: https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview
*/
package storage // import "cloud.google.com/go/storage"
vendor/cloud.google.com/go/storage/grpc_client.go
Generated Vendored
+1466 -69
View File
File diff suppressed because it is too large Load Diff
vendor/cloud.google.com/go/storage/hmac.go
Generated Vendored
+55 -108
View File
@@ -20,6 +20,7 @@ import (
"fmt"
"time"
storagepb "cloud.google.com/go/storage/internal/apiv2/stubs"
"google.golang.org/api/iterator"
raw "google.golang.org/api/storage/v1"
)
@@ -90,7 +91,7 @@ type HMACKeyHandle struct {
projectID string
accessID string
retry *retryConfig
raw *raw.ProjectsHmacKeysService
tc storageClient
}
// HMACKeyHandle creates a handle that will be used for HMACKey operations.
@@ -101,7 +102,7 @@ func (c *Client) HMACKeyHandle(projectID, accessID string) *HMACKeyHandle {
projectID: projectID,
accessID: accessID,
retry: c.retry,
raw: raw.NewProjectsHmacKeysService(c.raw),
tc: c.tc,
}
}
@@ -113,32 +114,15 @@ func (c *Client) HMACKeyHandle(projectID, accessID string) *HMACKeyHandle {
//
// This method is EXPERIMENTAL and subject to change or removal without notice.
func (hkh *HMACKeyHandle) Get(ctx context.Context, opts ...HMACKeyOption) (*HMACKey, error) {
call := hkh.raw.Get(hkh.projectID, hkh.accessID)
desc := new(hmacKeyDesc)
for _, opt := range opts {
opt.withHMACKeyDesc(desc)
}
if desc.userProjectID != "" {
call = call.UserProject(desc.userProjectID)
}
setClientHeader(call.Header())
o := makeStorageOpts(true, hkh.retry, desc.userProjectID)
hk, err := hkh.tc.GetHMACKey(ctx, hkh.projectID, hkh.accessID, o...)
var metadata *raw.HmacKeyMetadata
var err error
err = run(ctx, func() error {
metadata, err = call.Context(ctx).Do()
return err
}, hkh.retry, true)
if err != nil {
return nil, err
}
hkPb := &raw.HmacKey{
Metadata: metadata,
}
return pbHmacKeyToHMACKey(hkPb, false)
return hk, err
}
// Delete invokes an RPC to delete the key referenced by accessID, on Google Cloud Storage.
@@ -147,49 +131,59 @@ func (hkh *HMACKeyHandle) Get(ctx context.Context, opts ...HMACKeyOption) (*HMAC
//
// This method is EXPERIMENTAL and subject to change or removal without notice.
func (hkh *HMACKeyHandle) Delete(ctx context.Context, opts ...HMACKeyOption) error {
delCall := hkh.raw.Delete(hkh.projectID, hkh.accessID)
desc := new(hmacKeyDesc)
for _, opt := range opts {
opt.withHMACKeyDesc(desc)
}
if desc.userProjectID != "" {
delCall = delCall.UserProject(desc.userProjectID)
}
setClientHeader(delCall.Header())
return run(ctx, func() error {
return delCall.Context(ctx).Do()
}, hkh.retry, true)
o := makeStorageOpts(true, hkh.retry, desc.userProjectID)
return hkh.tc.DeleteHMACKey(ctx, hkh.projectID, hkh.accessID, o...)
}
func pbHmacKeyToHMACKey(pb *raw.HmacKey, updatedTimeCanBeNil bool) (*HMACKey, error) {
pbmd := pb.Metadata
if pbmd == nil {
func toHMACKeyFromRaw(hk *raw.HmacKey, updatedTimeCanBeNil bool) (*HMACKey, error) {
hkmd := hk.Metadata
if hkmd == nil {
return nil, errors.New("field Metadata cannot be nil")
}
createdTime, err := time.Parse(time.RFC3339, pbmd.TimeCreated)
createdTime, err := time.Parse(time.RFC3339, hkmd.TimeCreated)
if err != nil {
return nil, fmt.Errorf("field CreatedTime: %v", err)
return nil, fmt.Errorf("field CreatedTime: %w", err)
}
updatedTime, err := time.Parse(time.RFC3339, pbmd.Updated)
updatedTime, err := time.Parse(time.RFC3339, hkmd.Updated)
if err != nil && !updatedTimeCanBeNil {
return nil, fmt.Errorf("field UpdatedTime: %v", err)
return nil, fmt.Errorf("field UpdatedTime: %w", err)
}
hmk := &HMACKey{
AccessID: pbmd.AccessId,
Secret: pb.Secret,
Etag: pbmd.Etag,
ID: pbmd.Id,
State: HMACState(pbmd.State),
ProjectID: pbmd.ProjectId,
hmKey := &HMACKey{
AccessID: hkmd.AccessId,
Secret: hk.Secret,
Etag: hkmd.Etag,
ID: hkmd.Id,
State: HMACState(hkmd.State),
ProjectID: hkmd.ProjectId,
CreatedTime: createdTime,
UpdatedTime: updatedTime,
ServiceAccountEmail: pbmd.ServiceAccountEmail,
ServiceAccountEmail: hkmd.ServiceAccountEmail,
}
return hmk, nil
return hmKey, nil
}
func toHMACKeyFromProto(pbmd *storagepb.HmacKeyMetadata) *HMACKey {
if pbmd == nil {
return nil
}
return &HMACKey{
AccessID: pbmd.GetAccessId(),
ID: pbmd.GetId(),
State: HMACState(pbmd.GetState()),
ProjectID: pbmd.GetProject(),
CreatedTime: convertProtoTime(pbmd.GetCreateTime()),
UpdatedTime: convertProtoTime(pbmd.GetUpdateTime()),
ServiceAccountEmail: pbmd.GetServiceAccountEmail(),
}
}
// CreateHMACKey invokes an RPC for Google Cloud Storage to create a new HMACKey.
@@ -203,29 +197,14 @@ func (c *Client) CreateHMACKey(ctx context.Context, projectID, serviceAccountEma
return nil, errors.New("storage: expecting a non-blank service account email")
}
svc := raw.NewProjectsHmacKeysService(c.raw)
call := svc.Create(projectID, serviceAccountEmail)
desc := new(hmacKeyDesc)
for _, opt := range opts {
opt.withHMACKeyDesc(desc)
}
if desc.userProjectID != "" {
call = call.UserProject(desc.userProjectID)
}
setClientHeader(call.Header())
var hkPb *raw.HmacKey
if err := run(ctx, func() error {
h, err := call.Context(ctx).Do()
hkPb = h
return err
}, c.retry, false); err != nil {
return nil, err
}
return pbHmacKeyToHMACKey(hkPb, true)
o := makeStorageOpts(false, c.retry, desc.userProjectID)
hk, err := c.tc.CreateHMACKey(ctx, projectID, serviceAccountEmail, o...)
return hk, err
}
// HMACKeyAttrsToUpdate defines the attributes of an HMACKey that will be updated.
@@ -247,35 +226,15 @@ func (h *HMACKeyHandle) Update(ctx context.Context, au HMACKeyAttrsToUpdate, opt
return nil, fmt.Errorf("storage: invalid state %q for update, must be either %q or %q", au.State, Active, Inactive)
}
call := h.raw.Update(h.projectID, h.accessID, &raw.HmacKeyMetadata{
Etag: au.Etag,
State: string(au.State),
})
desc := new(hmacKeyDesc)
for _, opt := range opts {
opt.withHMACKeyDesc(desc)
}
if desc.userProjectID != "" {
call = call.UserProject(desc.userProjectID)
}
setClientHeader(call.Header())
var metadata *raw.HmacKeyMetadata
var err error
isIdempotent := len(au.Etag) > 0
err = run(ctx, func() error {
metadata, err = call.Context(ctx).Do()
return err
}, h.retry, isIdempotent)
if err != nil {
return nil, err
}
hkPb := &raw.HmacKey{
Metadata: metadata,
}
return pbHmacKeyToHMACKey(hkPb, false)
o := makeStorageOpts(isIdempotent, h.retry, desc.userProjectID)
hk, err := h.tc.UpdateHMACKey(ctx, h.projectID, desc.forServiceAccountEmail, h.accessID, &au, o...)
return hk, err
}
// An HMACKeysIterator is an iterator over HMACKeys.
@@ -301,27 +260,13 @@ type HMACKeysIterator struct {
//
// This method is EXPERIMENTAL and subject to change or removal without notice.
func (c *Client) ListHMACKeys(ctx context.Context, projectID string, opts ...HMACKeyOption) *HMACKeysIterator {
it := &HMACKeysIterator{
ctx: ctx,
raw: raw.NewProjectsHmacKeysService(c.raw),
projectID: projectID,
retry: c.retry,
}
desc := new(hmacKeyDesc)
for _, opt := range opts {
opt.withHMACKeyDesc(&it.desc)
opt.withHMACKeyDesc(desc)
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.hmacKeys) - it.index },
func() interface{} {
prev := it.hmacKeys
it.hmacKeys = it.hmacKeys[:0]
it.index = 0
return prev
})
return it
o := makeStorageOpts(true, c.retry, desc.userProjectID)
return c.tc.ListHMACKeys(ctx, projectID, desc.forServiceAccountEmail, desc.showDeletedKeys, o...)
}
// Next returns the next result. Its second return value is iterator.Done if
@@ -350,6 +295,8 @@ func (it *HMACKeysIterator) Next() (*HMACKey, error) {
func (it *HMACKeysIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
func (it *HMACKeysIterator) fetch(pageSize int, pageToken string) (token string, err error) {
// TODO: Remove fetch method upon integration. This method is internalized into
// httpStorageClient.ListHMACKeys() as it is the only caller.
call := it.raw.List(it.projectID)
setClientHeader(call.Header())
if pageToken != "" {
@@ -373,16 +320,16 @@ func (it *HMACKeysIterator) fetch(pageSize int, pageToken string) (token string,
err = run(it.ctx, func() error {
resp, err = call.Context(ctx).Do()
return err
}, it.retry, true)
}, it.retry, true, setRetryHeaderHTTP(call))
if err != nil {
return "", err
}
for _, metadata := range resp.Items {
hkPb := &raw.HmacKey{
hk := &raw.HmacKey{
Metadata: metadata,
}
hkey, err := pbHmacKeyToHMACKey(hkPb, true)
hkey, err := toHMACKeyFromRaw(hk, true)
if err != nil {
return "", err
}
vendor/cloud.google.com/go/storage/http_client.go
Generated Vendored
+1035 -53
View File
File diff suppressed because it is too large Load Diff
vendor/cloud.google.com/go/storage/iam.go
Generated Vendored
+8 -40
View File
@@ -27,17 +27,17 @@ import (
// IAM provides access to IAM access control for the bucket.
func (b *BucketHandle) IAM() *iam.Handle {
return iam.InternalNewHandleClient(&iamClient{
raw: b.c.raw,
userProject: b.userProject,
retry: b.retry,
client: b.c,
}, b.name)
}
// iamClient implements the iam.client interface.
type iamClient struct {
raw *raw.Service
userProject string
retry *retryConfig
client *Client
}
func (c *iamClient) Get(ctx context.Context, resource string) (p *iampb.Policy, err error) {
@@ -48,57 +48,25 @@ func (c *iamClient) GetWithVersion(ctx context.Context, resource string, request
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.IAM.Get")
defer func() { trace.EndSpan(ctx, err) }()
call := c.raw.Buckets.GetIamPolicy(resource).OptionsRequestedPolicyVersion(int64(requestedPolicyVersion))
setClientHeader(call.Header())
if c.userProject != "" {
call.UserProject(c.userProject)
}
var rp *raw.Policy
err = run(ctx, func() error {
rp, err = call.Context(ctx).Do()
return err
}, c.retry, true)
if err != nil {
return nil, err
}
return iamFromStoragePolicy(rp), nil
o := makeStorageOpts(true, c.retry, c.userProject)
return c.client.tc.GetIamPolicy(ctx, resource, requestedPolicyVersion, o...)
}
func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) (err error) {
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.IAM.Set")
defer func() { trace.EndSpan(ctx, err) }()
rp := iamToStoragePolicy(p)
call := c.raw.Buckets.SetIamPolicy(resource, rp)
setClientHeader(call.Header())
if c.userProject != "" {
call.UserProject(c.userProject)
}
isIdempotent := len(p.Etag) > 0
return run(ctx, func() error {
_, err := call.Context(ctx).Do()
return err
}, c.retry, isIdempotent)
o := makeStorageOpts(isIdempotent, c.retry, c.userProject)
return c.client.tc.SetIamPolicy(ctx, resource, p, o...)
}
func (c *iamClient) Test(ctx context.Context, resource string, perms []string) (permissions []string, err error) {
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.IAM.Test")
defer func() { trace.EndSpan(ctx, err) }()
call := c.raw.Buckets.TestIamPermissions(resource, perms)
setClientHeader(call.Header())
if c.userProject != "" {
call.UserProject(c.userProject)
}
var res *raw.TestIamPermissionsResponse
err = run(ctx, func() error {
res, err = call.Context(ctx).Do()
return err
}, c.retry, true)
if err != nil {
return nil, err
}
return res.Permissions, nil
o := makeStorageOpts(true, c.retry, c.userProject)
return c.client.tc.TestIamPermissions(ctx, resource, perms, o...)
}
func iamToStoragePolicy(ip *iampb.Policy) *raw.Policy {
vendor/cloud.google.com/go/storage/internal/apiv2/doc.go
Generated Vendored
+41 -28
View File
@@ -1,4 +1,4 @@
// Copyright 2022 Google LLC
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -19,52 +19,65 @@
//
// Lets you store and retrieve potentially-large, immutable data objects.
//
// NOTE: This package is in alpha. It is not stable, and is likely to change.
// NOTE: This package is in alpha. It is not stable, and is likely to change.
//
// Example usage
// # General documentation
//
// For information about setting deadlines, reusing contexts, and more
// please visit https://pkg.go.dev/cloud.google.com/go.
//
// # Example usage
//
// To get started with this package, create a client.
// ctx := context.Background()
// c, err := storage.NewClient(ctx)
// if err != nil {
// // TODO: Handle error.
// }
// defer c.Close()
//
// ctx := context.Background()
// // This snippet has been automatically generated and should be regarded as a code template only.
// // It will require modifications to work:
// // - It may require correct/in-range values for request initialization.
// // - It may require specifying regional endpoints when creating the service client as shown in:
// // https://pkg.go.dev/cloud.google.com/go#hdr-Client_Options
// c, err := storage.NewClient(ctx)
// if err != nil {
// // TODO: Handle error.
// }
// defer c.Close()
//
// The client will use your default application credentials. Clients should be reused instead of created as needed.
// The methods of Client are safe for concurrent use by multiple goroutines.
// The returned client must be Closed when it is done being used.
//
// Using the Client
// # Using the Client
//
// The following is an example of making an API call with the newly created client.
//
// ctx := context.Background()
// c, err := storage.NewClient(ctx)
// if err != nil {
// // TODO: Handle error.
// }
// defer c.Close()
// ctx := context.Background()
// // This snippet has been automatically generated and should be regarded as a code template only.
// // It will require modifications to work:
// // - It may require correct/in-range values for request initialization.
// // - It may require specifying regional endpoints when creating the service client as shown in:
// // https://pkg.go.dev/cloud.google.com/go#hdr-Client_Options
// c, err := storage.NewClient(ctx)
// if err != nil {
// // TODO: Handle error.
// }
// defer c.Close()
//
// req := &storagepb.DeleteBucketRequest{
// // TODO: Fill request struct fields.
// // See https://pkg.go.dev/google.golang.org/genproto/googleapis/storage/v2#DeleteBucketRequest.
// }
// err = c.DeleteBucket(ctx, req)
// if err != nil {
// // TODO: Handle error.
// }
// req := &storagepb.DeleteBucketRequest{
// // TODO: Fill request struct fields.
// // See https://pkg.go.dev/cloud.google.com/go/storage/internal/apiv2/stubs#DeleteBucketRequest.
// }
// err = c.DeleteBucket(ctx, req)
// if err != nil {
// // TODO: Handle error.
// }
//
// Use of Context
// # Use of Context
//
// The ctx passed to NewClient is used for authentication requests and
// for creating the underlying connection, but is not used for subsequent calls.
// Individual methods on the client use the ctx given to them.
//
// To close the open connection, use the Close() method.
//
// For information about setting deadlines, reusing contexts, and more
// please visit https://pkg.go.dev/cloud.google.com/go.
package storage // import "cloud.google.com/go/storage/internal/apiv2"
import (
vendor/cloud.google.com/go/storage/internal/apiv2/gapic_metadata.json
Generated Vendored
+5
View File
@@ -10,6 +10,11 @@
"grpc": {
"libraryClient": "Client",
"rpcs": {
"CancelResumableWrite": {
"methods": [
"CancelResumableWrite"
]
},
"ComposeObject": {
"methods": [
"ComposeObject"
vendor/github.com/googleapis/go-type-adapters/adapters/month.go → vendor/cloud.google.com/go/storage/internal/apiv2/metadata.go
Generated Vendored
+7 -12
View File
@@ -1,4 +1,4 @@
// Copyright 2021 Google LLC
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,20 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package adapters
package storage
import (
"time"
"context"
mpb "google.golang.org/genproto/googleapis/type/month"
"google.golang.org/grpc/metadata"
)
// ToMonth converts a google.type.Month to a golang Month.
func ToMonth(m mpb.Month) time.Month {
return time.Month(m.Number())
}
// ToProtoMonth converts a golang Month to a google.type.Month.
func ToProtoMonth(m time.Month) mpb.Month {
return mpb.Month(m)
// InsertMetadata inserts the given gRPC metadata into the outgoing context.
func InsertMetadata(ctx context.Context, mds ...metadata.MD) context.Context {
return insertMetadata(ctx, mds...)
}
vendor/cloud.google.com/go/storage/internal/apiv2/storage_client.go
Generated Vendored
+470 -70
View File
@@ -1,4 +1,4 @@
// Copyright 2022 Google LLC
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -18,15 +18,19 @@ package storage
import (
"context"
"fmt"
"math"
"net/url"
"regexp"
"strings"
storagepb "cloud.google.com/go/storage/internal/apiv2/stubs"
gax "github.com/googleapis/gax-go/v2"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
"google.golang.org/api/option/internaloption"
gtransport "google.golang.org/api/transport/grpc"
iampb "google.golang.org/genproto/googleapis/iam/v1"
storagepb "google.golang.org/genproto/googleapis/storage/v2"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"google.golang.org/protobuf/proto"
@@ -51,6 +55,7 @@ type CallOptions struct {
ListNotifications []gax.CallOption
ComposeObject []gax.CallOption
DeleteObject []gax.CallOption
CancelResumableWrite []gax.CallOption
GetObject []gax.CallOption
ReadObject []gax.CallOption
UpdateObject []gax.CallOption
@@ -96,6 +101,7 @@ func defaultCallOptions() *CallOptions {
ListNotifications: []gax.CallOption{},
ComposeObject: []gax.CallOption{},
DeleteObject: []gax.CallOption{},
CancelResumableWrite: []gax.CallOption{},
GetObject: []gax.CallOption{},
ReadObject: []gax.CallOption{},
UpdateObject: []gax.CallOption{},
@@ -113,7 +119,7 @@ func defaultCallOptions() *CallOptions {
}
}
// internalClient is an interface that defines the methods availaible from Cloud Storage API.
// internalClient is an interface that defines the methods available from Cloud Storage API.
type internalClient interface {
Close() error
setGoogleClientInfo(...string)
@@ -133,6 +139,7 @@ type internalClient interface {
ListNotifications(context.Context, *storagepb.ListNotificationsRequest, ...gax.CallOption) *NotificationIterator
ComposeObject(context.Context, *storagepb.ComposeObjectRequest, ...gax.CallOption) (*storagepb.Object, error)
DeleteObject(context.Context, *storagepb.DeleteObjectRequest, ...gax.CallOption) error
CancelResumableWrite(context.Context, *storagepb.CancelResumableWriteRequest, ...gax.CallOption) (*storagepb.CancelResumableWriteResponse, error)
GetObject(context.Context, *storagepb.GetObjectRequest, ...gax.CallOption) (*storagepb.Object, error)
ReadObject(context.Context, *storagepb.ReadObjectRequest, ...gax.CallOption) (storagepb.Storage_ReadObjectClient, error)
UpdateObject(context.Context, *storagepb.UpdateObjectRequest, ...gax.CallOption) (*storagepb.Object, error)
@@ -158,22 +165,22 @@ type internalClient interface {
//
// Resources are named as follows:
//
// Projects are referred to as they are defined by the Resource Manager API,
// using strings like projects/123456 or projects/my-string-id.
// Projects are referred to as they are defined by the Resource Manager API,
// using strings like projects/123456 or projects/my-string-id.
//
// Buckets are named using string names of the form:
// projects/{project}/buckets/{bucket}
// For globally unique buckets, _ may be substituted for the project.
// Buckets are named using string names of the form:
// projects/{project}/buckets/{bucket}
// For globally unique buckets, _ may be substituted for the project.
//
// Objects are uniquely identified by their name along with the name of the
// bucket they belong to, as separate strings in this API. For example:
// Objects are uniquely identified by their name along with the name of the
// bucket they belong to, as separate strings in this API. For example:
//
// ReadObjectRequest {
// bucket: ‘projects/_/buckets/my-bucket’
// object: ‘my-object’
// }
// Note that object names can contain / characters, which are treated as
// any other character (no special directory semantics).
// ReadObjectRequest {
// bucket: ‘projects/_/buckets/my-bucket’
// object: ‘my-object’
// }
// Note that object names can contain / characters, which are treated as
// any other character (no special directory semantics).
type Client struct {
// The internal transport-dependent client.
internalClient internalClient
@@ -199,7 +206,8 @@ func (c *Client) setGoogleClientInfo(keyval ...string) {
// Connection returns a connection to the API service.
//
// Deprecated.
// Deprecated: Connections are now pooled so this method does not always
// return the same resource.
func (c *Client) Connection() *grpc.ClientConn {
return c.internalClient.Connection()
}
@@ -229,18 +237,27 @@ func (c *Client) LockBucketRetentionPolicy(ctx context.Context, req *storagepb.L
return c.internalClient.LockBucketRetentionPolicy(ctx, req, opts...)
}
// GetIamPolicy gets the IAM policy for a specified bucket.
// GetIamPolicy gets the IAM policy for a specified bucket or object.
// The resource field in the request should be
// projects//buckets/<bucket_name> for a bucket or
// projects//buckets/<bucket_name>/objects/<object_name> for an object.
func (c *Client) GetIamPolicy(ctx context.Context, req *iampb.GetIamPolicyRequest, opts ...gax.CallOption) (*iampb.Policy, error) {
return c.internalClient.GetIamPolicy(ctx, req, opts...)
}
// SetIamPolicy updates an IAM policy for the specified bucket.
// SetIamPolicy updates an IAM policy for the specified bucket or object.
// The resource field in the request should be
// projects//buckets/<bucket_name> for a bucket or
// projects//buckets/<bucket_name>/objects/<object_name> for an object.
func (c *Client) SetIamPolicy(ctx context.Context, req *iampb.SetIamPolicyRequest, opts ...gax.CallOption) (*iampb.Policy, error) {
return c.internalClient.SetIamPolicy(ctx, req, opts...)
}
// TestIamPermissions tests a set of permissions on the given bucket to see which, if
// TestIamPermissions tests a set of permissions on the given bucket or object to see which, if
// any, are held by the caller.
// The resource field in the request should be
// projects//buckets/<bucket_name> for a bucket or
// projects//buckets/<bucket_name>/objects/<object_name> for an object.
func (c *Client) TestIamPermissions(ctx context.Context, req *iampb.TestIamPermissionsRequest, opts ...gax.CallOption) (*iampb.TestIamPermissionsResponse, error) {
return c.internalClient.TestIamPermissions(ctx, req, opts...)
}
@@ -280,12 +297,16 @@ func (c *Client) ComposeObject(ctx context.Context, req *storagepb.ComposeObject
}
// DeleteObject deletes an object and its metadata. Deletions are permanent if versioning
// is not enabled for the bucket, or if the generation parameter
// is used.
// is not enabled for the bucket, or if the generation parameter is used.
func (c *Client) DeleteObject(ctx context.Context, req *storagepb.DeleteObjectRequest, opts ...gax.CallOption) error {
return c.internalClient.DeleteObject(ctx, req, opts...)
}
// CancelResumableWrite cancels an in-progress resumable upload.
func (c *Client) CancelResumableWrite(ctx context.Context, req *storagepb.CancelResumableWriteRequest, opts ...gax.CallOption) (*storagepb.CancelResumableWriteResponse, error) {
return c.internalClient.CancelResumableWrite(ctx, req, opts...)
}
// GetObject retrieves an object’s metadata.
func (c *Client) GetObject(ctx context.Context, req *storagepb.GetObjectRequest, opts ...gax.CallOption) (*storagepb.Object, error) {
return c.internalClient.GetObject(ctx, req, opts...)
@@ -312,13 +333,40 @@ func (c *Client) UpdateObject(ctx context.Context, req *storagepb.UpdateObjectRe
// true, or else it is an error.
//
// For a resumable write, the client should instead call
// StartResumableWrite() and provide that method an WriteObjectSpec.
// StartResumableWrite(), populating a WriteObjectSpec into that request.
// They should then attach the returned upload_id to the first message of
// each following call to Create. If there is an error or the connection is
// broken during the resumable Create(), the client should check the status
// of the Create() by calling QueryWriteStatus() and continue writing from
// the returned persisted_size. This may be less than the amount of data the
// client previously sent.
// each following call to WriteObject. If the stream is closed before
// finishing the upload (either explicitly by the client or due to a network
// error or an error response from the server), the client should do as
// follows:
//
// Check the result Status of the stream, to determine if writing can be
// resumed on this stream or must be restarted from scratch (by calling
// StartResumableWrite()). The resumable errors are DEADLINE_EXCEEDED,
// INTERNAL, and UNAVAILABLE. For each case, the client should use binary
// exponential backoff before retrying. Additionally, writes can be
// resumed after RESOURCE_EXHAUSTED errors, but only after taking
// appropriate measures, which may include reducing aggregate send rate
// across clients and/or requesting a quota increase for your project.
//
// If the call to WriteObject returns ABORTED, that indicates
// concurrent attempts to update the resumable write, caused either by
// multiple racing clients or by a single client where the previous
// request was timed out on the client side but nonetheless reached the
// server. In this case the client should take steps to prevent further
// concurrent writes (e.g., increase the timeouts, stop using more than
// one process to perform the upload, etc.), and then should follow the
// steps below for resuming the upload.
//
// For resumable errors, the client should call QueryWriteStatus() and
// then continue writing from the returned persisted_size. This may be
// less than the amount of data the client previously sent. Note also that
// it is acceptable to send data starting at an offset earlier than the
// returned persisted_size; in this case, the service will skip data at
// offsets that were already persisted (without checking that it matches
// the previously written data), and write only the data starting from the
// persisted offset. This behavior can make client-side handling simpler
// in some cases.
//
// The service will not view the object as complete until the client has
// sent a WriteObjectRequest with finish_write set to true. Sending any
@@ -326,6 +374,10 @@ func (c *Client) UpdateObject(ctx context.Context, req *storagepb.UpdateObjectRe
// true will cause an error. The client should check the response it
// receives to determine how much data the service was able to commit and
// whether the service views the object as complete.
//
// Attempting to resume an already finalized object will result in an OK
// status, with a WriteObjectResponse containing the finalized object’s
// metadata.
func (c *Client) WriteObject(ctx context.Context, opts ...gax.CallOption) (storagepb.Storage_WriteObjectClient, error) {
return c.internalClient.WriteObject(ctx, opts...)
}
@@ -424,22 +476,22 @@ type gRPCClient struct {
//
// Resources are named as follows:
//
// Projects are referred to as they are defined by the Resource Manager API,
// using strings like projects/123456 or projects/my-string-id.
// Projects are referred to as they are defined by the Resource Manager API,
// using strings like projects/123456 or projects/my-string-id.
//
// Buckets are named using string names of the form:
// projects/{project}/buckets/{bucket}
// For globally unique buckets, _ may be substituted for the project.
// Buckets are named using string names of the form:
// projects/{project}/buckets/{bucket}
// For globally unique buckets, _ may be substituted for the project.
//
// Objects are uniquely identified by their name along with the name of the
// bucket they belong to, as separate strings in this API. For example:
// Objects are uniquely identified by their name along with the name of the
// bucket they belong to, as separate strings in this API. For example:
//
// ReadObjectRequest {
// bucket: ‘projects/_/buckets/my-bucket’
// object: ‘my-object’
// }
// Note that object names can contain / characters, which are treated as
// any other character (no special directory semantics).
// ReadObjectRequest {
// bucket: ‘projects/_/buckets/my-bucket’
// object: ‘my-object’
// }
// Note that object names can contain / characters, which are treated as
// any other character (no special directory semantics).
func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) {
clientOpts := defaultGRPCClientOptions()
if newClientHook != nil {
@@ -476,7 +528,8 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error
// Connection returns a connection to the API service.
//
// Deprecated.
// Deprecated: Connections are now pooled so this method does not always
// return the same resource.
func (c *gRPCClient) Connection() *grpc.ClientConn {
return c.connPool.Conn()
}
@@ -497,7 +550,18 @@ func (c *gRPCClient) Close() error {
}
func (c *gRPCClient) DeleteBucket(ctx context.Context, req *storagepb.DeleteBucketRequest, opts ...gax.CallOption) error {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetName()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetName())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetName())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).DeleteBucket[0:len((*c.CallOptions).DeleteBucket):len((*c.CallOptions).DeleteBucket)], opts...)
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
@@ -508,7 +572,18 @@ func (c *gRPCClient) DeleteBucket(ctx context.Context, req *storagepb.DeleteBuck
}
func (c *gRPCClient) GetBucket(ctx context.Context, req *storagepb.GetBucketRequest, opts ...gax.CallOption) (*storagepb.Bucket, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetName()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetName())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetName())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).GetBucket[0:len((*c.CallOptions).GetBucket):len((*c.CallOptions).GetBucket)], opts...)
var resp *storagepb.Bucket
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -523,7 +598,18 @@ func (c *gRPCClient) GetBucket(ctx context.Context, req *storagepb.GetBucketRequ
}
func (c *gRPCClient) CreateBucket(ctx context.Context, req *storagepb.CreateBucketRequest, opts ...gax.CallOption) (*storagepb.Bucket, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<project>.*)"); reg.MatchString(req.GetParent()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetParent())[1])) > 0 {
routingHeadersMap["project"] = url.QueryEscape(reg.FindStringSubmatch(req.GetParent())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).CreateBucket[0:len((*c.CallOptions).CreateBucket):len((*c.CallOptions).CreateBucket)], opts...)
var resp *storagepb.Bucket
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -538,7 +624,18 @@ func (c *gRPCClient) CreateBucket(ctx context.Context, req *storagepb.CreateBuck
}
func (c *gRPCClient) ListBuckets(ctx context.Context, req *storagepb.ListBucketsRequest, opts ...gax.CallOption) *BucketIterator {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<project>.*)"); reg.MatchString(req.GetParent()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetParent())[1])) > 0 {
routingHeadersMap["project"] = url.QueryEscape(reg.FindStringSubmatch(req.GetParent())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).ListBuckets[0:len((*c.CallOptions).ListBuckets):len((*c.CallOptions).ListBuckets)], opts...)
it := &BucketIterator{}
req = proto.Clone(req).(*storagepb.ListBucketsRequest)
@@ -581,7 +678,18 @@ func (c *gRPCClient) ListBuckets(ctx context.Context, req *storagepb.ListBuckets
}
func (c *gRPCClient) LockBucketRetentionPolicy(ctx context.Context, req *storagepb.LockBucketRetentionPolicyRequest, opts ...gax.CallOption) (*storagepb.Bucket, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetBucket()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetBucket())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetBucket())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).LockBucketRetentionPolicy[0:len((*c.CallOptions).LockBucketRetentionPolicy):len((*c.CallOptions).LockBucketRetentionPolicy)], opts...)
var resp *storagepb.Bucket
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -596,7 +704,21 @@ func (c *gRPCClient) LockBucketRetentionPolicy(ctx context.Context, req *storage
}
func (c *gRPCClient) GetIamPolicy(ctx context.Context, req *iampb.GetIamPolicyRequest, opts ...gax.CallOption) (*iampb.Policy, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetResource()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetResource())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetResource())[1])
}
if reg := regexp.MustCompile("(?P<bucket>projects/[^/]+/buckets/[^/]+)/objects(?:/.*)?"); reg.MatchString(req.GetResource()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetResource())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetResource())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).GetIamPolicy[0:len((*c.CallOptions).GetIamPolicy):len((*c.CallOptions).GetIamPolicy)], opts...)
var resp *iampb.Policy
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -611,7 +733,21 @@ func (c *gRPCClient) GetIamPolicy(ctx context.Context, req *iampb.GetIamPolicyRe
}
func (c *gRPCClient) SetIamPolicy(ctx context.Context, req *iampb.SetIamPolicyRequest, opts ...gax.CallOption) (*iampb.Policy, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetResource()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetResource())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetResource())[1])
}
if reg := regexp.MustCompile("(?P<bucket>projects/[^/]+/buckets/[^/]+)/objects(?:/.*)?"); reg.MatchString(req.GetResource()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetResource())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetResource())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).SetIamPolicy[0:len((*c.CallOptions).SetIamPolicy):len((*c.CallOptions).SetIamPolicy)], opts...)
var resp *iampb.Policy
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -626,7 +762,21 @@ func (c *gRPCClient) SetIamPolicy(ctx context.Context, req *iampb.SetIamPolicyRe
}
func (c *gRPCClient) TestIamPermissions(ctx context.Context, req *iampb.TestIamPermissionsRequest, opts ...gax.CallOption) (*iampb.TestIamPermissionsResponse, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetResource()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetResource())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetResource())[1])
}
if reg := regexp.MustCompile("(?P<bucket>projects/[^/]+/buckets/[^/]+)/objects(?:/.*)?"); reg.MatchString(req.GetResource()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetResource())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetResource())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).TestIamPermissions[0:len((*c.CallOptions).TestIamPermissions):len((*c.CallOptions).TestIamPermissions)], opts...)
var resp *iampb.TestIamPermissionsResponse
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -641,7 +791,18 @@ func (c *gRPCClient) TestIamPermissions(ctx context.Context, req *iampb.TestIamP
}
func (c *gRPCClient) UpdateBucket(ctx context.Context, req *storagepb.UpdateBucketRequest, opts ...gax.CallOption) (*storagepb.Bucket, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetBucket().GetName()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetBucket().GetName())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetBucket().GetName())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).UpdateBucket[0:len((*c.CallOptions).UpdateBucket):len((*c.CallOptions).UpdateBucket)], opts...)
var resp *storagepb.Bucket
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -656,7 +817,18 @@ func (c *gRPCClient) UpdateBucket(ctx context.Context, req *storagepb.UpdateBuck
}
func (c *gRPCClient) DeleteNotification(ctx context.Context, req *storagepb.DeleteNotificationRequest, opts ...gax.CallOption) error {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>projects/[^/]+/buckets/[^/]+)(?:/.*)?"); reg.MatchString(req.GetName()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetName())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetName())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).DeleteNotification[0:len((*c.CallOptions).DeleteNotification):len((*c.CallOptions).DeleteNotification)], opts...)
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
@@ -667,7 +839,18 @@ func (c *gRPCClient) DeleteNotification(ctx context.Context, req *storagepb.Dele
}
func (c *gRPCClient) GetNotification(ctx context.Context, req *storagepb.GetNotificationRequest, opts ...gax.CallOption) (*storagepb.Notification, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>projects/[^/]+/buckets/[^/]+)(?:/.*)?"); reg.MatchString(req.GetName()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetName())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetName())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).GetNotification[0:len((*c.CallOptions).GetNotification):len((*c.CallOptions).GetNotification)], opts...)
var resp *storagepb.Notification
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -682,7 +865,18 @@ func (c *gRPCClient) GetNotification(ctx context.Context, req *storagepb.GetNoti
}
func (c *gRPCClient) CreateNotification(ctx context.Context, req *storagepb.CreateNotificationRequest, opts ...gax.CallOption) (*storagepb.Notification, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetParent()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetParent())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetParent())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).CreateNotification[0:len((*c.CallOptions).CreateNotification):len((*c.CallOptions).CreateNotification)], opts...)
var resp *storagepb.Notification
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -697,7 +891,18 @@ func (c *gRPCClient) CreateNotification(ctx context.Context, req *storagepb.Crea
}
func (c *gRPCClient) ListNotifications(ctx context.Context, req *storagepb.ListNotificationsRequest, opts ...gax.CallOption) *NotificationIterator {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetParent()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetParent())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetParent())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).ListNotifications[0:len((*c.CallOptions).ListNotifications):len((*c.CallOptions).ListNotifications)], opts...)
it := &NotificationIterator{}
req = proto.Clone(req).(*storagepb.ListNotificationsRequest)
@@ -740,7 +945,18 @@ func (c *gRPCClient) ListNotifications(ctx context.Context, req *storagepb.ListN
}
func (c *gRPCClient) ComposeObject(ctx context.Context, req *storagepb.ComposeObjectRequest, opts ...gax.CallOption) (*storagepb.Object, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetDestination().GetBucket()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetDestination().GetBucket())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetDestination().GetBucket())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).ComposeObject[0:len((*c.CallOptions).ComposeObject):len((*c.CallOptions).ComposeObject)], opts...)
var resp *storagepb.Object
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -755,7 +971,18 @@ func (c *gRPCClient) ComposeObject(ctx context.Context, req *storagepb.ComposeOb
}
func (c *gRPCClient) DeleteObject(ctx context.Context, req *storagepb.DeleteObjectRequest, opts ...gax.CallOption) error {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetBucket()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetBucket())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetBucket())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).DeleteObject[0:len((*c.CallOptions).DeleteObject):len((*c.CallOptions).DeleteObject)], opts...)
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
@@ -765,8 +992,45 @@ func (c *gRPCClient) DeleteObject(ctx context.Context, req *storagepb.DeleteObje
return err
}
func (c *gRPCClient) CancelResumableWrite(ctx context.Context, req *storagepb.CancelResumableWriteRequest, opts ...gax.CallOption) (*storagepb.CancelResumableWriteResponse, error) {
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>projects/[^/]+/buckets/[^/]+)(?:/.*)?"); reg.MatchString(req.GetUploadId()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetUploadId())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetUploadId())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).CancelResumableWrite[0:len((*c.CallOptions).CancelResumableWrite):len((*c.CallOptions).CancelResumableWrite)], opts...)
var resp *storagepb.CancelResumableWriteResponse
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
resp, err = c.client.CancelResumableWrite(ctx, req, settings.GRPC...)
return err
}, opts...)
if err != nil {
return nil, err
}
return resp, nil
}
func (c *gRPCClient) GetObject(ctx context.Context, req *storagepb.GetObjectRequest, opts ...gax.CallOption) (*storagepb.Object, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetBucket()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetBucket())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetBucket())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).GetObject[0:len((*c.CallOptions).GetObject):len((*c.CallOptions).GetObject)], opts...)
var resp *storagepb.Object
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -781,7 +1045,19 @@ func (c *gRPCClient) GetObject(ctx context.Context, req *storagepb.GetObjectRequ
}
func (c *gRPCClient) ReadObject(ctx context.Context, req *storagepb.ReadObjectRequest, opts ...gax.CallOption) (storagepb.Storage_ReadObjectClient, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetBucket()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetBucket())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetBucket())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).ReadObject[0:len((*c.CallOptions).ReadObject):len((*c.CallOptions).ReadObject)], opts...)
var resp storagepb.Storage_ReadObjectClient
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
@@ -795,7 +1071,18 @@ func (c *gRPCClient) ReadObject(ctx context.Context, req *storagepb.ReadObjectRe
}
func (c *gRPCClient) UpdateObject(ctx context.Context, req *storagepb.UpdateObjectRequest, opts ...gax.CallOption) (*storagepb.Object, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetObject().GetBucket()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetObject().GetBucket())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetObject().GetBucket())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).UpdateObject[0:len((*c.CallOptions).UpdateObject):len((*c.CallOptions).UpdateObject)], opts...)
var resp *storagepb.Object
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -825,7 +1112,18 @@ func (c *gRPCClient) WriteObject(ctx context.Context, opts ...gax.CallOption) (s
}
func (c *gRPCClient) ListObjects(ctx context.Context, req *storagepb.ListObjectsRequest, opts ...gax.CallOption) *ObjectIterator {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetParent()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetParent())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetParent())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).ListObjects[0:len((*c.CallOptions).ListObjects):len((*c.CallOptions).ListObjects)], opts...)
it := &ObjectIterator{}
req = proto.Clone(req).(*storagepb.ListObjectsRequest)
@@ -868,7 +1166,21 @@ func (c *gRPCClient) ListObjects(ctx context.Context, req *storagepb.ListObjects
}
func (c *gRPCClient) RewriteObject(ctx context.Context, req *storagepb.RewriteObjectRequest, opts ...gax.CallOption) (*storagepb.RewriteResponse, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(.*)"); reg.MatchString(req.GetSourceBucket()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetSourceBucket())[1])) > 0 {
routingHeadersMap["source_bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetSourceBucket())[1])
}
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetDestinationBucket()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetDestinationBucket())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetDestinationBucket())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).RewriteObject[0:len((*c.CallOptions).RewriteObject):len((*c.CallOptions).RewriteObject)], opts...)
var resp *storagepb.RewriteResponse
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -883,7 +1195,18 @@ func (c *gRPCClient) RewriteObject(ctx context.Context, req *storagepb.RewriteOb
}
func (c *gRPCClient) StartResumableWrite(ctx context.Context, req *storagepb.StartResumableWriteRequest, opts ...gax.CallOption) (*storagepb.StartResumableWriteResponse, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>.*)"); reg.MatchString(req.GetWriteObjectSpec().GetResource().GetBucket()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetWriteObjectSpec().GetResource().GetBucket())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetWriteObjectSpec().GetResource().GetBucket())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).StartResumableWrite[0:len((*c.CallOptions).StartResumableWrite):len((*c.CallOptions).StartResumableWrite)], opts...)
var resp *storagepb.StartResumableWriteResponse
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -898,7 +1221,18 @@ func (c *gRPCClient) StartResumableWrite(ctx context.Context, req *storagepb.Sta
}
func (c *gRPCClient) QueryWriteStatus(ctx context.Context, req *storagepb.QueryWriteStatusRequest, opts ...gax.CallOption) (*storagepb.QueryWriteStatusResponse, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<bucket>projects/[^/]+/buckets/[^/]+)(?:/.*)?"); reg.MatchString(req.GetUploadId()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetUploadId())[1])) > 0 {
routingHeadersMap["bucket"] = url.QueryEscape(reg.FindStringSubmatch(req.GetUploadId())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).QueryWriteStatus[0:len((*c.CallOptions).QueryWriteStatus):len((*c.CallOptions).QueryWriteStatus)], opts...)
var resp *storagepb.QueryWriteStatusResponse
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -913,7 +1247,18 @@ func (c *gRPCClient) QueryWriteStatus(ctx context.Context, req *storagepb.QueryW
}
func (c *gRPCClient) GetServiceAccount(ctx context.Context, req *storagepb.GetServiceAccountRequest, opts ...gax.CallOption) (*storagepb.ServiceAccount, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(.*)"); reg.MatchString(req.GetProject()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetProject())[1])) > 0 {
routingHeadersMap["project"] = url.QueryEscape(reg.FindStringSubmatch(req.GetProject())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).GetServiceAccount[0:len((*c.CallOptions).GetServiceAccount):len((*c.CallOptions).GetServiceAccount)], opts...)
var resp *storagepb.ServiceAccount
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -928,7 +1273,18 @@ func (c *gRPCClient) GetServiceAccount(ctx context.Context, req *storagepb.GetSe
}
func (c *gRPCClient) CreateHmacKey(ctx context.Context, req *storagepb.CreateHmacKeyRequest, opts ...gax.CallOption) (*storagepb.CreateHmacKeyResponse, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(.*)"); reg.MatchString(req.GetProject()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetProject())[1])) > 0 {
routingHeadersMap["project"] = url.QueryEscape(reg.FindStringSubmatch(req.GetProject())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).CreateHmacKey[0:len((*c.CallOptions).CreateHmacKey):len((*c.CallOptions).CreateHmacKey)], opts...)
var resp *storagepb.CreateHmacKeyResponse
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -943,7 +1299,18 @@ func (c *gRPCClient) CreateHmacKey(ctx context.Context, req *storagepb.CreateHma
}
func (c *gRPCClient) DeleteHmacKey(ctx context.Context, req *storagepb.DeleteHmacKeyRequest, opts ...gax.CallOption) error {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(.*)"); reg.MatchString(req.GetProject()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetProject())[1])) > 0 {
routingHeadersMap["project"] = url.QueryEscape(reg.FindStringSubmatch(req.GetProject())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).DeleteHmacKey[0:len((*c.CallOptions).DeleteHmacKey):len((*c.CallOptions).DeleteHmacKey)], opts...)
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
@@ -954,7 +1321,18 @@ func (c *gRPCClient) DeleteHmacKey(ctx context.Context, req *storagepb.DeleteHma
}
func (c *gRPCClient) GetHmacKey(ctx context.Context, req *storagepb.GetHmacKeyRequest, opts ...gax.CallOption) (*storagepb.HmacKeyMetadata, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(.*)"); reg.MatchString(req.GetProject()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetProject())[1])) > 0 {
routingHeadersMap["project"] = url.QueryEscape(reg.FindStringSubmatch(req.GetProject())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).GetHmacKey[0:len((*c.CallOptions).GetHmacKey):len((*c.CallOptions).GetHmacKey)], opts...)
var resp *storagepb.HmacKeyMetadata
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -969,7 +1347,18 @@ func (c *gRPCClient) GetHmacKey(ctx context.Context, req *storagepb.GetHmacKeyRe
}
func (c *gRPCClient) ListHmacKeys(ctx context.Context, req *storagepb.ListHmacKeysRequest, opts ...gax.CallOption) *HmacKeyMetadataIterator {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(.*)"); reg.MatchString(req.GetProject()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetProject())[1])) > 0 {
routingHeadersMap["project"] = url.QueryEscape(reg.FindStringSubmatch(req.GetProject())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).ListHmacKeys[0:len((*c.CallOptions).ListHmacKeys):len((*c.CallOptions).ListHmacKeys)], opts...)
it := &HmacKeyMetadataIterator{}
req = proto.Clone(req).(*storagepb.ListHmacKeysRequest)
@@ -1012,7 +1401,18 @@ func (c *gRPCClient) ListHmacKeys(ctx context.Context, req *storagepb.ListHmacKe
}
func (c *gRPCClient) UpdateHmacKey(ctx context.Context, req *storagepb.UpdateHmacKeyRequest, opts ...gax.CallOption) (*storagepb.HmacKeyMetadata, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
routingHeaders := ""
routingHeadersMap := make(map[string]string)
if reg := regexp.MustCompile("(?P<project>.*)"); reg.MatchString(req.GetHmacKey().GetProject()) && len(url.QueryEscape(reg.FindStringSubmatch(req.GetHmacKey().GetProject())[1])) > 0 {
routingHeadersMap["project"] = url.QueryEscape(reg.FindStringSubmatch(req.GetHmacKey().GetProject())[1])
}
for headerName, headerValue := range routingHeadersMap {
routingHeaders = fmt.Sprintf("%s%s=%s&", routingHeaders, headerName, headerValue)
}
routingHeaders = strings.TrimSuffix(routingHeaders, "&")
md := metadata.Pairs("x-goog-request-params", routingHeaders)
ctx = insertMetadata(ctx, c.xGoogMetadata, md)
opts = append((*c.CallOptions).UpdateHmacKey[0:len((*c.CallOptions).UpdateHmacKey):len((*c.CallOptions).UpdateHmacKey)], opts...)
var resp *storagepb.HmacKeyMetadata
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
vendor/google.golang.org/genproto/googleapis/storage/v2/storage.pb.go → vendor/cloud.google.com/go/storage/internal/apiv2/stubs/storage.pb.go
Generated Vendored
+2572 -2341
View File
File diff suppressed because it is too large Load Diff
vendor/cloud.google.com/go/storage/internal/apiv2/version.go
Generated Vendored
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2022 Google LLC
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
vendor/cloud.google.com/go/storage/internal/version.go
Generated Vendored
+1 -1
View File
@@ -15,4 +15,4 @@
package internal
// Version is the current tagged release of the library.
const Version = "1.22.0"
const Version = "1.29.0"
vendor/cloud.google.com/go/storage/invoke.go
Generated Vendored
+47 -6
View File
@@ -16,29 +16,39 @@ package storage
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
"cloud.google.com/go/internal"
"cloud.google.com/go/internal/version"
sinternal "cloud.google.com/go/storage/internal"
"github.com/google/uuid"
gax "github.com/googleapis/gax-go/v2"
"golang.org/x/xerrors"
"google.golang.org/api/googleapi"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
var defaultRetry *retryConfig = &retryConfig{}
var xGoogDefaultHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), sinternal.Version)
// run determines whether a retry is necessary based on the config and
// idempotency information. It then calls the function with or without retries
// as appropriate, using the configured settings.
func run(ctx context.Context, call func() error, retry *retryConfig, isIdempotent bool) error {
func run(ctx context.Context, call func() error, retry *retryConfig, isIdempotent bool, setHeader func(string, int)) error {
attempts := 1
invocationID := uuid.New().String()
if retry == nil {
retry = defaultRetry
}
if (retry.policy == RetryIdempotent && !isIdempotent) || retry.policy == RetryNever {
setHeader(invocationID, attempts)
return call()
}
bo := gax.Backoff{}
@@ -47,21 +57,52 @@ func run(ctx context.Context, call func() error, retry *retryConfig, isIdempoten
bo.Initial = retry.backoff.Initial
bo.Max = retry.backoff.Max
}
var errorFunc func(err error) bool = shouldRetry
var errorFunc func(err error) bool = ShouldRetry
if retry.shouldRetry != nil {
errorFunc = retry.shouldRetry
}
return internal.Retry(ctx, bo, func() (stop bool, err error) {
setHeader(invocationID, attempts)
err = call()
attempts++
return !errorFunc(err), err
})
}
func shouldRetry(err error) bool {
func setRetryHeaderHTTP(req interface{ Header() http.Header }) func(string, int) {
return func(invocationID string, attempts int) {
if req == nil {
return
}
header := req.Header()
invocationHeader := fmt.Sprintf("gccl-invocation-id/%v gccl-attempt-count/%v", invocationID, attempts)
xGoogHeader := strings.Join([]string{invocationHeader, xGoogDefaultHeader}, " ")
header.Set("x-goog-api-client", xGoogHeader)
}
}
// TODO: Implement method setting header via context for gRPC
func setRetryHeaderGRPC(_ context.Context) func(string, int) {
return func(_ string, _ int) {
return
}
}
// ShouldRetry returns true if an error is retryable, based on best practice
// guidance from GCS. See
// https://cloud.google.com/storage/docs/retry-strategy#go for more information
// on what errors are considered retryable.
//
// If you would like to customize retryable errors, use the WithErrorFunc to
// supply a RetryOption to your library calls. For example, to retry additional
// errors, you can write a custom func that wraps ShouldRetry and also specifies
// additional errors that should return true.
func ShouldRetry(err error) bool {
if err == nil {
return false
}
if xerrors.Is(err, io.ErrUnexpectedEOF) {
if errors.Is(err, io.ErrUnexpectedEOF) {
return true
}
@@ -99,7 +140,7 @@ func shouldRetry(err error) bool {
}
// Unwrap is only supported in go1.13.x+
if e, ok := err.(interface{ Unwrap() error }); ok {
return shouldRetry(e.Unwrap())
return ShouldRetry(e.Unwrap())
}
return false
}
vendor/cloud.google.com/go/storage/notifications.go
Generated Vendored
+41 -36
View File
@@ -21,6 +21,7 @@ import (
"regexp"
"cloud.google.com/go/internal/trace"
storagepb "cloud.google.com/go/storage/internal/apiv2/stubs"
raw "google.golang.org/api/storage/v1"
)
@@ -91,6 +92,30 @@ func toNotification(rn *raw.Notification) *Notification {
return n
}
func toNotificationFromProto(pbn *storagepb.Notification) *Notification {
n := &Notification{
ID: pbn.GetName(),
EventTypes: pbn.GetEventTypes(),
ObjectNamePrefix: pbn.GetObjectNamePrefix(),
CustomAttributes: pbn.GetCustomAttributes(),
PayloadFormat: pbn.GetPayloadFormat(),
}
n.TopicProjectID, n.TopicID = parseNotificationTopic(pbn.Topic)
return n
}
func toProtoNotification(n *Notification) *storagepb.Notification {
return &storagepb.Notification{
Name: n.ID,
Topic: fmt.Sprintf("//pubsub.googleapis.com/projects/%s/topics/%s",
n.TopicProjectID, n.TopicID),
EventTypes: n.EventTypes,
ObjectNamePrefix: n.ObjectNamePrefix,
CustomAttributes: n.CustomAttributes,
PayloadFormat: n.PayloadFormat,
}
}
var topicRE = regexp.MustCompile("^//pubsub.googleapis.com/projects/([^/]+)/topics/([^/]+)")
// parseNotificationTopic extracts the project and topic IDs from from the full
@@ -132,21 +157,10 @@ func (b *BucketHandle) AddNotification(ctx context.Context, n *Notification) (re
if n.TopicID == "" {
return nil, errors.New("storage: AddNotification: missing TopicID")
}
call := b.c.raw.Notifications.Insert(b.name, toRawNotification(n))
setClientHeader(call.Header())
if b.userProject != "" {
call.UserProject(b.userProject)
}
var rn *raw.Notification
err = run(ctx, func() error {
rn, err = call.Context(ctx).Do()
return err
}, b.retry, false)
if err != nil {
return nil, err
}
return toNotification(rn), nil
opts := makeStorageOpts(false, b.retry, b.userProject)
ret, err = b.c.tc.CreateNotification(ctx, b.name, n, opts...)
return ret, err
}
// Notifications returns all the Notifications configured for this bucket, as a map
@@ -155,20 +169,9 @@ func (b *BucketHandle) Notifications(ctx context.Context) (n map[string]*Notific
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Notifications")
defer func() { trace.EndSpan(ctx, err) }()
call := b.c.raw.Notifications.List(b.name)
setClientHeader(call.Header())
if b.userProject != "" {
call.UserProject(b.userProject)
}
var res *raw.Notifications
err = run(ctx, func() error {
res, err = call.Context(ctx).Do()
return err
}, b.retry, true)
if err != nil {
return nil, err
}
return notificationsToMap(res.Items), nil
opts := makeStorageOpts(true, b.retry, b.userProject)
n, err = b.c.tc.ListNotifications(ctx, b.name, opts...)
return n, err
}
func notificationsToMap(rns []*raw.Notification) map[string]*Notification {
@@ -179,17 +182,19 @@ func notificationsToMap(rns []*raw.Notification) map[string]*Notification {
return m
}
func notificationsToMapFromProto(ns []*storagepb.Notification) map[string]*Notification {
m := map[string]*Notification{}
for _, n := range ns {
m[n.Name] = toNotificationFromProto(n)
}
return m
}
// DeleteNotification deletes the notification with the given ID.
func (b *BucketHandle) DeleteNotification(ctx context.Context, id string) (err error) {
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.DeleteNotification")
defer func() { trace.EndSpan(ctx, err) }()
call := b.c.raw.Notifications.Delete(b.name, id)
setClientHeader(call.Header())
if b.userProject != "" {
call.UserProject(b.userProject)
}
return run(ctx, func() error {
return call.Context(ctx).Do()
}, b.retry, true)
opts := makeStorageOpts(true, b.retry, b.userProject)
return b.c.tc.DeleteNotification(ctx, b.name, id, opts...)
}
vendor/cloud.google.com/go/storage/post_policy_v4.go
Generated Vendored
+1 -1
View File
@@ -340,7 +340,7 @@ func GenerateSignedPostPolicyV4(bucket, object string, opts *PostPolicyV4Options
"expiration": opts.Expires.Format(time.RFC3339),
})
if err != nil {
return nil, fmt.Errorf("storage: PostPolicyV4 JSON serialization failed: %v", err)
return nil, fmt.Errorf("storage: PostPolicyV4 JSON serialization failed: %w", err)
}
b64Policy := base64.StdEncoding.EncodeToString(condsAsJSON)
vendor/cloud.google.com/go/storage/reader.go
Generated Vendored
+29 -488
View File
@@ -16,20 +16,15 @@ package storage
import (
"context"
"errors"
"fmt"
"hash/crc32"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"cloud.google.com/go/internal/trace"
"google.golang.org/api/googleapi"
storagepb "google.golang.org/genproto/googleapis/storage/v2"
)
var crc32cTable = crc32.MakeTable(crc32.Castagnoli)
@@ -95,10 +90,6 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.NewRangeReader")
defer func() { trace.EndSpan(ctx, err) }()
if o.c.gc != nil {
return o.newRangeReaderWithGRPC(ctx, offset, length)
}
if err := o.validate(); err != nil {
return nil, err
}
@@ -110,208 +101,31 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
return nil, err
}
}
u := &url.URL{
Scheme: o.c.scheme,
Host: o.c.readHost,
Path: fmt.Sprintf("/%s/%s", o.bucket, o.object),
}
verb := "GET"
if length == 0 {
verb = "HEAD"
}
req, err := http.NewRequest(verb, u.String(), nil)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
if o.userProject != "" {
req.Header.Set("X-Goog-User-Project", o.userProject)
}
if o.readCompressed {
req.Header.Set("Accept-Encoding", "gzip")
}
if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil {
return nil, err
opts := makeStorageOpts(true, o.retry, o.userProject)
params := &newRangeReaderParams{
bucket: o.bucket,
object: o.object,
gen: o.gen,
offset: offset,
length: length,
encryptionKey: o.encryptionKey,
conds: o.conds,
readCompressed: o.readCompressed,
}
gen := o.gen
r, err = o.c.tc.NewRangeReader(ctx, params, opts...)
// Define a function that initiates a Read with offset and length, assuming we
// have already read seen bytes.
reopen := func(seen int64) (*http.Response, error) {
// If the context has already expired, return immediately without making a
// call.
if err := ctx.Err(); err != nil {
return nil, err
}
start := offset + seen
if length < 0 && start < 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d", start))
} else if length < 0 && start > 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", start))
} else if length > 0 {
// The end character isn't affected by how many bytes we've seen.
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, offset+length-1))
}
// We wait to assign conditions here because the generation number can change in between reopen() runs.
if err := setConditionsHeaders(req.Header, o.conds); err != nil {
return nil, err
}
// If an object generation is specified, include generation as query string parameters.
if gen >= 0 {
req.URL.RawQuery = fmt.Sprintf("generation=%d", gen)
}
var res *http.Response
err = run(ctx, func() error {
res, err = o.c.hc.Do(req)
if err != nil {
return err
}
if res.StatusCode == http.StatusNotFound {
res.Body.Close()
return ErrObjectNotExist
}
if res.StatusCode < 200 || res.StatusCode > 299 {
body, _ := ioutil.ReadAll(res.Body)
res.Body.Close()
return &googleapi.Error{
Code: res.StatusCode,
Header: res.Header,
Body: string(body),
}
}
partialContentNotSatisfied :=
!decompressiveTranscoding(res) &&
start > 0 && length != 0 &&
res.StatusCode != http.StatusPartialContent
if partialContentNotSatisfied {
res.Body.Close()
return errors.New("storage: partial request not satisfied")
}
// With "Content-Encoding": "gzip" aka decompressive transcoding, GCS serves
// back the whole file regardless of the range count passed in as per:
// https://cloud.google.com/storage/docs/transcoding#range,
// thus we have to manually move the body forward by seen bytes.
if decompressiveTranscoding(res) && seen > 0 {
_, _ = io.CopyN(ioutil.Discard, res.Body, seen)
}
// If a generation hasn't been specified, and this is the first response we get, let's record the
// generation. In future requests we'll use this generation as a precondition to avoid data races.
if gen < 0 && res.Header.Get("X-Goog-Generation") != "" {
gen64, err := strconv.ParseInt(res.Header.Get("X-Goog-Generation"), 10, 64)
if err != nil {
return err
}
gen = gen64
}
return nil
}, o.retry, true)
if err != nil {
return nil, err
}
return res, nil
}
res, err := reopen(0)
if err != nil {
return nil, err
}
var (
size int64 // total size of object, even if a range was requested.
checkCRC bool
crc uint32
startOffset int64 // non-zero if range request.
)
if res.StatusCode == http.StatusPartialContent {
cr := strings.TrimSpace(res.Header.Get("Content-Range"))
if !strings.HasPrefix(cr, "bytes ") || !strings.Contains(cr, "/") {
return nil, fmt.Errorf("storage: invalid Content-Range %q", cr)
}
// Content range is formatted <first byte>-<last byte>/<total size>. We take
// the total size.
size, err = strconv.ParseInt(cr[strings.LastIndex(cr, "/")+1:], 10, 64)
if err != nil {
return nil, fmt.Errorf("storage: invalid Content-Range %q", cr)
}
dashIndex := strings.Index(cr, "-")
if dashIndex >= 0 {
startOffset, err = strconv.ParseInt(cr[len("bytes="):dashIndex], 10, 64)
if err != nil {
return nil, fmt.Errorf("storage: invalid Content-Range %q: %v", cr, err)
}
}
} else {
size = res.ContentLength
// Check the CRC iff all of the following hold:
// - We asked for content (length != 0).
// - We got all the content (status != PartialContent).
// - The server sent a CRC header.
// - The Go http stack did not uncompress the file.
// - We were not served compressed data that was uncompressed on download.
// The problem with the last two cases is that the CRC will not match -- GCS
// computes it on the compressed contents, but we compute it on the
// uncompressed contents.
if length != 0 && !res.Uncompressed && !uncompressedByServer(res) {
crc, checkCRC = parseCRC32c(res)
}
}
remain := res.ContentLength
body := res.Body
if length == 0 {
remain = 0
body.Close()
body = emptyBody
}
var metaGen int64
if res.Header.Get("X-Goog-Metageneration") != "" {
metaGen, err = strconv.ParseInt(res.Header.Get("X-Goog-Metageneration"), 10, 64)
if err != nil {
return nil, err
}
}
var lm time.Time
if res.Header.Get("Last-Modified") != "" {
lm, err = http.ParseTime(res.Header.Get("Last-Modified"))
if err != nil {
return nil, err
}
}
attrs := ReaderObjectAttrs{
Size: size,
ContentType: res.Header.Get("Content-Type"),
ContentEncoding: res.Header.Get("Content-Encoding"),
CacheControl: res.Header.Get("Cache-Control"),
LastModified: lm,
StartOffset: startOffset,
Generation: gen,
Metageneration: metaGen,
}
return &Reader{
Attrs: attrs,
body: body,
size: size,
remain: remain,
wantCRC: crc,
checkCRC: checkCRC,
reopen: reopen,
}, nil
return r, err
}
// decompressiveTranscoding returns true if the request was served decompressed
// and different than its original storage form. This happens when the "Content-Encoding"
// header is "gzip".
// See:
// * https://cloud.google.com/storage/docs/transcoding#transcoding_and_gzip
// * https://github.com/googleapis/google-cloud-go/issues/1800
// - https://cloud.google.com/storage/docs/transcoding#transcoding_and_gzip
// - https://github.com/googleapis/google-cloud-go/issues/1800
func decompressiveTranscoding(res *http.Response) bool {
// Decompressive Transcoding.
return res.Header.Get("Content-Encoding") == "gzip" ||
@@ -356,6 +170,16 @@ func setConditionsHeaders(headers http.Header, conds *Conditions) error {
return nil
}
// Wrap a request to look similar to an apiary library request, in order to
// be used by run().
type readerRequestWrapper struct {
req *http.Request
}
func (w *readerRequestWrapper) Header() http.Header {
return w.req.Header
}
var emptyBody = ioutil.NopCloser(strings.NewReader(""))
// Reader reads a Cloud Storage object.
@@ -366,42 +190,21 @@ var emptyBody = ioutil.NopCloser(strings.NewReader(""))
// is skipped if transcoding occurs. See https://cloud.google.com/storage/docs/transcoding.
type Reader struct {
Attrs ReaderObjectAttrs
body io.ReadCloser
seen, remain, size int64
checkCRC bool // should we check the CRC?
wantCRC uint32 // the CRC32c value the server sent in the header
gotCRC uint32 // running crc
reopen func(seen int64) (*http.Response, error)
// The following fields are only for use in the gRPC hybrid client.
stream storagepb.Storage_ReadObjectClient
reopenWithGRPC func(seen int64) (*readStreamResponse, context.CancelFunc, error)
leftovers []byte
cancelStream context.CancelFunc
}
type readStreamResponse struct {
stream storagepb.Storage_ReadObjectClient
response *storagepb.ReadObjectResponse
reader io.ReadCloser
}
// Close closes the Reader. It must be called when done reading.
func (r *Reader) Close() error {
if r.body != nil {
return r.body.Close()
}
r.closeStream()
return nil
return r.reader.Close()
}
func (r *Reader) Read(p []byte) (int, error) {
read := r.readWithRetry
if r.reopenWithGRPC != nil {
read = r.readWithGRPC
}
n, err := read(p)
n, err := r.reader.Read(p)
if r.remain != -1 {
r.remain -= int64(n)
}
@@ -420,268 +223,6 @@ func (r *Reader) Read(p []byte) (int, error) {
return n, err
}
// newRangeReaderWithGRPC creates a new Reader with the given range that uses
// gRPC to read Object content.
//
// This is an experimental API and not intended for public use.
func (o *ObjectHandle) newRangeReaderWithGRPC(ctx context.Context, offset, length int64) (r *Reader, err error) {
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.newRangeReaderWithGRPC")
defer func() { trace.EndSpan(ctx, err) }()
if o.c.gc == nil {
err = fmt.Errorf("handle doesn't have a gRPC client initialized")
return
}
if err = o.validate(); err != nil {
return
}
// A negative length means "read to the end of the object", but the
// read_limit field it corresponds to uses zero to mean the same thing. Thus
// we coerce the length to 0 to read to the end of the object.
if length < 0 {
length = 0
}
// For now, there are only globally unique buckets, and "_" is the alias
// project ID for such buckets.
b := bucketResourceName("_", o.bucket)
req := &storagepb.ReadObjectRequest{
Bucket: b,
Object: o.object,
}
// The default is a negative value, which means latest.
if o.gen >= 0 {
req.Generation = o.gen
}
// Define a function that initiates a Read with offset and length, assuming
// we have already read seen bytes.
reopen := func(seen int64) (*readStreamResponse, context.CancelFunc, error) {
// If the context has already expired, return immediately without making
// we call.
if err := ctx.Err(); err != nil {
return nil, nil, err
}
cc, cancel := context.WithCancel(ctx)
start := offset + seen
// Only set a ReadLimit if length is greater than zero, because zero
// means read it all.
if length > 0 {
req.ReadLimit = length - seen
}
req.ReadOffset = start
if err := applyCondsProto("reopenWithGRPC", o.gen, o.conds, req); err != nil {
cancel()
return nil, nil, err
}
var stream storagepb.Storage_ReadObjectClient
var msg *storagepb.ReadObjectResponse
var err error
err = run(cc, func() error {
stream, err = o.c.gc.ReadObject(cc, req)
if err != nil {
return err
}
msg, err = stream.Recv()
return err
}, o.retry, true)
if err != nil {
// Close the stream context we just created to ensure we don't leak
// resources.
cancel()
return nil, nil, err
}
return &readStreamResponse{stream, msg}, cancel, nil
}
res, cancel, err := reopen(0)
if err != nil {
return nil, err
}
r = &Reader{
stream: res.stream,
reopenWithGRPC: reopen,
cancelStream: cancel,
}
// The first message was Recv'd on stream open, use it to populate the
// object metadata.
msg := res.response
obj := msg.GetMetadata()
// This is the size of the entire object, even if only a range was requested.
size := obj.GetSize()
r.Attrs = ReaderObjectAttrs{
Size: size,
ContentType: obj.GetContentType(),
ContentEncoding: obj.GetContentEncoding(),
CacheControl: obj.GetCacheControl(),
LastModified: obj.GetUpdateTime().AsTime(),
Metageneration: obj.GetMetageneration(),
Generation: obj.GetGeneration(),
}
r.size = size
cr := msg.GetContentRange()
if cr != nil {
r.Attrs.StartOffset = cr.GetStart()
r.remain = cr.GetEnd() - cr.GetStart() + 1
} else {
r.remain = size
}
// Only support checksums when reading an entire object, not a range.
if checksums := msg.GetObjectChecksums(); checksums != nil && checksums.Crc32C != nil && offset == 0 && length == 0 {
r.wantCRC = checksums.GetCrc32C()
r.checkCRC = true
}
// Store the content from the first Recv in the client buffer for reading
// later.
r.leftovers = msg.GetChecksummedData().GetContent()
return r, nil
}
func (r *Reader) readWithRetry(p []byte) (int, error) {
n := 0
for len(p[n:]) > 0 {
m, err := r.body.Read(p[n:])
n += m
r.seen += int64(m)
if err == nil || err == io.EOF {
return n, err
}
// Read failed (likely due to connection issues), but we will try to reopen
// the pipe and continue. Send a ranged read request that takes into account
// the number of bytes we've already seen.
res, err := r.reopen(r.seen)
if err != nil {
// reopen already retries
return n, err
}
r.body.Close()
r.body = res.Body
}
return n, nil
}
// closeStream cancels a stream's context in order for it to be closed and
// collected.
//
// This is an experimental API and not intended for public use.
func (r *Reader) closeStream() {
if r.cancelStream != nil {
r.cancelStream()
}
r.stream = nil
}
// readWithGRPC reads bytes into the user's buffer from an open gRPC stream.
//
// This is an experimental API and not intended for public use.
func (r *Reader) readWithGRPC(p []byte) (int, error) {
// No stream to read from, either never initiliazed or Close was called.
// Note: There is a potential concurrency issue if multiple routines are
// using the same reader. One encounters an error and the stream is closed
// and then reopened while the other routine attempts to read from it.
if r.stream == nil {
return 0, fmt.Errorf("reader has been closed")
}
// The entire object has been read by this reader, return EOF.
if r.size != 0 && r.size == r.seen {
return 0, io.EOF
}
var n int
// Read leftovers and return what was available to conform to the Reader
// interface: https://pkg.go.dev/io#Reader.
if len(r.leftovers) > 0 {
n = copy(p, r.leftovers)
r.seen += int64(n)
r.leftovers = r.leftovers[n:]
return n, nil
}
// Attempt to Recv the next message on the stream.
msg, err := r.recv()
if err != nil {
return 0, err
}
// TODO: Determine if we need to capture incremental CRC32C for this
// chunk. The Object CRC32C checksum is captured when directed to read
// the entire Object. If directed to read a range, we may need to
// calculate the range's checksum for verification if the checksum is
// present in the response here.
// TODO: Figure out if we need to support decompressive transcoding
// https://cloud.google.com/storage/docs/transcoding.
content := msg.GetChecksummedData().GetContent()
n = copy(p[n:], content)
leftover := len(content) - n
if leftover > 0 {
// Wasn't able to copy all of the data in the message, store for
// future Read calls.
r.leftovers = content[n:]
}
r.seen += int64(n)
return n, nil
}
// recv attempts to Recv the next message on the stream. In the event
// that a retryable error is encountered, the stream will be closed, reopened,
// and Recv again. This will attempt to Recv until one of the following is true:
//
// * Recv is successful
// * A non-retryable error is encountered
// * The Reader's context is canceled
//
// The last error received is the one that is returned, which could be from
// an attempt to reopen the stream.
//
// This is an experimental API and not intended for public use.
func (r *Reader) recv() (*storagepb.ReadObjectResponse, error) {
msg, err := r.stream.Recv()
if err != nil && shouldRetry(err) {
// This will "close" the existing stream and immediately attempt to
// reopen the stream, but will backoff if further attempts are necessary.
// Reopening the stream Recvs the first message, so if retrying is
// successful, the next logical chunk will be returned.
msg, err = r.reopenStream(r.seen)
}
return msg, err
}
// reopenStream "closes" the existing stream and attempts to reopen a stream and
// sets the Reader's stream and cancelStream properties in the process.
//
// This is an experimental API and not intended for public use.
func (r *Reader) reopenStream(seen int64) (*storagepb.ReadObjectResponse, error) {
// Close existing stream and initialize new stream with updated offset.
r.closeStream()
res, cancel, err := r.reopenWithGRPC(r.seen)
if err != nil {
return nil, err
}
r.stream = res.stream
r.cancelStream = cancel
return res.response, nil
}
// Size returns the size of the object in bytes.
// The returned value is always the same and is not affected by
// calls to Read or Close.
vendor/cloud.google.com/go/storage/release-please-config.json
Generated Vendored
-11
View File
@@ -1,11 +0,0 @@
{
"release-type": "go-yoshi",
"separate-pull-requests": true,
"include-component-in-tag": true,
"tag-separator": "/",
"packages": {
"storage": {
"component": "storage"
}
}
}
vendor/cloud.google.com/go/storage/storage.go
Generated Vendored
+329 -220
View File
@@ -33,27 +33,26 @@ import (
"reflect"
"regexp"
"sort"
"strconv"
"strings"
"time"
"unicode/utf8"
"cloud.google.com/go/internal/optional"
"cloud.google.com/go/internal/trace"
"cloud.google.com/go/internal/version"
"cloud.google.com/go/storage/internal"
gapic "cloud.google.com/go/storage/internal/apiv2"
storagepb "cloud.google.com/go/storage/internal/apiv2/stubs"
"github.com/googleapis/gax-go/v2"
"golang.org/x/oauth2/google"
"golang.org/x/xerrors"
"google.golang.org/api/googleapi"
"google.golang.org/api/option"
"google.golang.org/api/option/internaloption"
raw "google.golang.org/api/storage/v1"
"google.golang.org/api/transport"
htransport "google.golang.org/api/transport/http"
storagepb "google.golang.org/genproto/googleapis/storage/v2"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/types/known/fieldmaskpb"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -86,12 +85,19 @@ const (
// ScopeReadWrite grants permissions to manage your
// data in Google Cloud Storage.
ScopeReadWrite = raw.DevstorageReadWriteScope
// aes256Algorithm is the AES256 encryption algorithm used with the
// Customer-Supplied Encryption Keys feature.
aes256Algorithm = "AES256"
// defaultGen indicates the latest object generation by default,
// using a negative value.
defaultGen = int64(-1)
)
var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), internal.Version)
// TODO: remove this once header with invocation ID is applied to all methods.
func setClientHeader(headers http.Header) {
headers.Set("x-goog-api-client", xGoogHeader)
headers.Set("x-goog-api-client", xGoogDefaultHeader)
}
// Client is a client for interacting with Google Cloud Storage.
@@ -109,10 +115,12 @@ type Client struct {
creds *google.Credentials
retry *retryConfig
// gc is an optional gRPC-based, GAPIC client.
//
// This is an experimental field and not intended for public use.
gc *gapic.Client
// tc is the transport-agnostic client implemented with either gRPC or HTTP.
tc storageClient
// useGRPC flags whether the client uses gRPC. This is needed while the
// integration piece is only partially complete.
// TODO: remove before merging to main.
useGRPC bool
}
// NewClient creates a new Google Cloud Storage client.
@@ -122,6 +130,13 @@ type Client struct {
// Clients should be reused instead of created as needed. The methods of Client
// are safe for concurrent use by multiple goroutines.
func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) {
// Use the experimental gRPC client if the env var is set.
// This is an experimental API and not intended for public use.
if withGRPC := os.Getenv("STORAGE_USE_GRPC"); withGRPC != "" {
return newGRPCClient(ctx, opts...)
}
var creds *google.Credentials
// In general, it is recommended to use raw.NewService instead of htransport.NewClient
@@ -173,17 +188,22 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error
// htransport selects the correct endpoint among WithEndpoint (user override), WithDefaultEndpoint, and WithDefaultMTLSEndpoint.
hc, ep, err := htransport.NewClient(ctx, opts...)
if err != nil {
return nil, fmt.Errorf("dialing: %v", err)
return nil, fmt.Errorf("dialing: %w", err)
}
// RawService should be created with the chosen endpoint to take account of user override.
rawService, err := raw.NewService(ctx, option.WithEndpoint(ep), option.WithHTTPClient(hc))
if err != nil {
return nil, fmt.Errorf("storage client: %v", err)
return nil, fmt.Errorf("storage client: %w", err)
}
// Update readHost and scheme with the chosen endpoint.
u, err := url.Parse(ep)
if err != nil {
return nil, fmt.Errorf("supplied endpoint %q is not valid: %v", ep, err)
return nil, fmt.Errorf("supplied endpoint %q is not valid: %w", ep, err)
}
tc, err := newHTTPStorageClient(ctx, withClientOptions(opts...))
if err != nil {
return nil, fmt.Errorf("storage: %w", err)
}
return &Client{
@@ -192,37 +212,22 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error
scheme: u.Scheme,
readHost: u.Host,
creds: creds,
tc: tc,
}, nil
}
// hybridClientOptions carries the set of client options for HTTP and gRPC clients.
type hybridClientOptions struct {
HTTPOpts []option.ClientOption
GRPCOpts []option.ClientOption
}
// newHybridClient creates a new Storage client that initializes a gRPC-based client
// for media upload and download operations.
// newGRPCClient creates a new Storage client that initializes a gRPC-based
// client. Calls that have not been implemented in gRPC will panic.
//
// This is an experimental API and not intended for public use.
func newHybridClient(ctx context.Context, opts *hybridClientOptions) (*Client, error) {
if opts == nil {
opts = &hybridClientOptions{}
}
opts.GRPCOpts = append(defaultGRPCOptions(), opts.GRPCOpts...)
c, err := NewClient(ctx, opts.HTTPOpts...)
func newGRPCClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) {
opts = append(defaultGRPCOptions(), opts...)
tc, err := newGRPCStorageClient(ctx, withClientOptions(opts...))
if err != nil {
return nil, err
}
g, err := gapic.NewClient(ctx, opts.GRPCOpts...)
if err != nil {
return nil, err
}
c.gc = g
return c, nil
return &Client{tc: tc, useGRPC: true}, nil
}
// Close closes the Client.
@@ -233,8 +238,8 @@ func (c *Client) Close() error {
c.hc = nil
c.raw = nil
c.creds = nil
if c.gc != nil {
return c.gc.Close()
if c.tc != nil {
return c.tc.Close()
}
return nil
}
@@ -516,13 +521,13 @@ func v2SanitizeHeaders(hdrs []string) []string {
// at https://cloud.google.com/storage/docs/authentication/canonical-requests#about-headers.
//
// V4 does a couple things differently from V2:
// - Headers get sorted by key, instead of by key:value. We do this in
// signedURLV4.
// - There's no canonical regexp: we simply split headers on :.
// - We don't exclude canonical headers.
// - We replace leading and trailing spaces in header values, like v2, but also
// all intermediate space duplicates get stripped. That is, there's only ever
// a single consecutive space.
// - Headers get sorted by key, instead of by key:value. We do this in
// signedURLV4.
// - There's no canonical regexp: we simply split headers on :.
// - We don't exclude canonical headers.
// - We replace leading and trailing spaces in header values, like v2, but also
// all intermediate space duplicates get stripped. That is, there's only ever
// a single consecutive space.
func v4SanitizeHeaders(hdrs []string) []string {
headerMap := map[string][]string{}
for _, hdr := range hdrs {
@@ -914,27 +919,8 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (attrs *ObjectAttrs, err error
if err := o.validate(); err != nil {
return nil, err
}
call := o.c.raw.Objects.Get(o.bucket, o.object).Projection("full").Context(ctx)
if err := applyConds("Attrs", o.gen, o.conds, call); err != nil {
return nil, err
}
if o.userProject != "" {
call.UserProject(o.userProject)
}
if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil {
return nil, err
}
var obj *raw.Object
setClientHeader(call.Header())
err = run(ctx, func() error { obj, err = call.Do(); return err }, o.retry, true)
var e *googleapi.Error
if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound {
return nil, ErrObjectNotExist
}
if err != nil {
return nil, err
}
return newObject(obj), nil
opts := makeStorageOpts(true, o.retry, o.userProject)
return o.c.tc.GetObject(ctx, o.bucket, o.object, o.gen, o.encryptionKey, o.conds, opts...)
}
// Update updates an object with the provided attributes. See
@@ -947,99 +933,9 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (
if err := o.validate(); err != nil {
return nil, err
}
var attrs ObjectAttrs
// Lists of fields to send, and set to null, in the JSON.
var forceSendFields, nullFields []string
if uattrs.ContentType != nil {
attrs.ContentType = optional.ToString(uattrs.ContentType)
// For ContentType, sending the empty string is a no-op.
// Instead we send a null.
if attrs.ContentType == "" {
nullFields = append(nullFields, "ContentType")
} else {
forceSendFields = append(forceSendFields, "ContentType")
}
}
if uattrs.ContentLanguage != nil {
attrs.ContentLanguage = optional.ToString(uattrs.ContentLanguage)
// For ContentLanguage it's an error to send the empty string.
// Instead we send a null.
if attrs.ContentLanguage == "" {
nullFields = append(nullFields, "ContentLanguage")
} else {
forceSendFields = append(forceSendFields, "ContentLanguage")
}
}
if uattrs.ContentEncoding != nil {
attrs.ContentEncoding = optional.ToString(uattrs.ContentEncoding)
forceSendFields = append(forceSendFields, "ContentEncoding")
}
if uattrs.ContentDisposition != nil {
attrs.ContentDisposition = optional.ToString(uattrs.ContentDisposition)
forceSendFields = append(forceSendFields, "ContentDisposition")
}
if uattrs.CacheControl != nil {
attrs.CacheControl = optional.ToString(uattrs.CacheControl)
forceSendFields = append(forceSendFields, "CacheControl")
}
if uattrs.EventBasedHold != nil {
attrs.EventBasedHold = optional.ToBool(uattrs.EventBasedHold)
forceSendFields = append(forceSendFields, "EventBasedHold")
}
if uattrs.TemporaryHold != nil {
attrs.TemporaryHold = optional.ToBool(uattrs.TemporaryHold)
forceSendFields = append(forceSendFields, "TemporaryHold")
}
if !uattrs.CustomTime.IsZero() {
attrs.CustomTime = uattrs.CustomTime
forceSendFields = append(forceSendFields, "CustomTime")
}
if uattrs.Metadata != nil {
attrs.Metadata = uattrs.Metadata
if len(attrs.Metadata) == 0 {
// Sending the empty map is a no-op. We send null instead.
nullFields = append(nullFields, "Metadata")
} else {
forceSendFields = append(forceSendFields, "Metadata")
}
}
if uattrs.ACL != nil {
attrs.ACL = uattrs.ACL
// It's an error to attempt to delete the ACL, so
// we don't append to nullFields here.
forceSendFields = append(forceSendFields, "Acl")
}
rawObj := attrs.toRawObject(o.bucket)
rawObj.ForceSendFields = forceSendFields
rawObj.NullFields = nullFields
call := o.c.raw.Objects.Patch(o.bucket, o.object, rawObj).Projection("full").Context(ctx)
if err := applyConds("Update", o.gen, o.conds, call); err != nil {
return nil, err
}
if o.userProject != "" {
call.UserProject(o.userProject)
}
if uattrs.PredefinedACL != "" {
call.PredefinedAcl(uattrs.PredefinedACL)
}
if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil {
return nil, err
}
var obj *raw.Object
setClientHeader(call.Header())
var isIdempotent bool
if o.conds != nil && o.conds.MetagenerationMatch != 0 {
isIdempotent = true
}
err = run(ctx, func() error { obj, err = call.Do(); return err }, o.retry, isIdempotent)
var e *googleapi.Error
if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound {
return nil, ErrObjectNotExist
}
if err != nil {
return nil, err
}
return newObject(obj), nil
isIdempotent := o.conds != nil && o.conds.MetagenerationMatch != 0
opts := makeStorageOpts(isIdempotent, o.retry, o.userProject)
return o.c.tc.UpdateObject(ctx, o.bucket, o.object, &uattrs, o.gen, o.encryptionKey, o.conds, opts...)
}
// BucketName returns the name of the bucket.
@@ -1059,11 +955,12 @@ func (o *ObjectHandle) ObjectName() string {
//
// For example, to change ContentType and delete ContentEncoding and
// Metadata, use
// ObjectAttrsToUpdate{
// ContentType: "text/html",
// ContentEncoding: "",
// Metadata: map[string]string{},
// }
//
// ObjectAttrsToUpdate{
// ContentType: "text/html",
// ContentEncoding: "",
// Metadata: map[string]string{},
// }
type ObjectAttrsToUpdate struct {
EventBasedHold optional.Bool
TemporaryHold optional.Bool
@@ -1086,27 +983,11 @@ func (o *ObjectHandle) Delete(ctx context.Context) error {
if err := o.validate(); err != nil {
return err
}
call := o.c.raw.Objects.Delete(o.bucket, o.object).Context(ctx)
if err := applyConds("Delete", o.gen, o.conds, call); err != nil {
return err
}
if o.userProject != "" {
call.UserProject(o.userProject)
}
// Encryption doesn't apply to Delete.
setClientHeader(call.Header())
var isIdempotent bool
// Delete is idempotent if GenerationMatch or Generation have been passed in.
// The default generation is negative to get the latest version of the object.
if (o.conds != nil && o.conds.GenerationMatch != 0) || o.gen >= 0 {
isIdempotent = true
}
err := run(ctx, func() error { return call.Do() }, o.retry, isIdempotent)
var e *googleapi.Error
if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound {
return ErrObjectNotExist
}
return err
isIdempotent := (o.conds != nil && o.conds.GenerationMatch != 0) || o.gen >= 0
opts := makeStorageOpts(isIdempotent, o.retry, o.userProject)
return o.c.tc.DeleteObject(ctx, o.bucket, o.object, o.gen, o.conds, opts...)
}
// ReadCompressed when true causes the read to happen without decompressing.
@@ -1129,6 +1010,9 @@ func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle {
// attribute is specified, the content type will be automatically sniffed
// using net/http.DetectContentType.
//
// Note that each Writer allocates an internal buffer of size Writer.ChunkSize.
// See the ChunkSize docs for more information.
//
// It is the caller's responsibility to call Close when writing is done. To
// stop writing without saving the data, cancel the context.
func (o *ObjectHandle) NewWriter(ctx context.Context) *Writer {
@@ -1206,14 +1090,12 @@ func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object {
// toProtoObject copies the editable attributes from o to the proto library's Object type.
func (o *ObjectAttrs) toProtoObject(b string) *storagepb.Object {
checksums := &storagepb.ObjectChecksums{Md5Hash: o.MD5}
if o.CRC32C > 0 {
checksums.Crc32C = proto.Uint32(o.CRC32C)
}
// For now, there are only globally unique buckets, and "_" is the alias
// project ID for such buckets.
b = bucketResourceName("_", b)
// project ID for such buckets. If the bucket is not provided, like in the
// destination ObjectAttrs of a Copy, do not attempt to format it.
if b != "" {
b = bucketResourceName(globalProjectAlias, b)
}
return &storagepb.Object{
Bucket: b,
@@ -1236,10 +1118,52 @@ func (o *ObjectAttrs) toProtoObject(b string) *storagepb.Object {
KmsKey: o.KMSKeyName,
Generation: o.Generation,
Size: o.Size,
Checksums: checksums,
}
}
// toProtoObject copies the attributes to update from uattrs to the proto library's Object type.
func (uattrs *ObjectAttrsToUpdate) toProtoObject(bucket, object string) *storagepb.Object {
o := &storagepb.Object{
Name: object,
Bucket: bucket,
}
if uattrs == nil {
return o
}
if uattrs.EventBasedHold != nil {
o.EventBasedHold = proto.Bool(optional.ToBool(uattrs.EventBasedHold))
}
if uattrs.TemporaryHold != nil {
o.TemporaryHold = optional.ToBool(uattrs.TemporaryHold)
}
if uattrs.ContentType != nil {
o.ContentType = optional.ToString(uattrs.ContentType)
}
if uattrs.ContentLanguage != nil {
o.ContentLanguage = optional.ToString(uattrs.ContentLanguage)
}
if uattrs.ContentEncoding != nil {
o.ContentEncoding = optional.ToString(uattrs.ContentEncoding)
}
if uattrs.ContentDisposition != nil {
o.ContentDisposition = optional.ToString(uattrs.ContentDisposition)
}
if uattrs.CacheControl != nil {
o.CacheControl = optional.ToString(uattrs.CacheControl)
}
if !uattrs.CustomTime.IsZero() {
o.CustomTime = toProtoTimestamp(uattrs.CustomTime)
}
if uattrs.ACL != nil {
o.Acl = toProtoObjectACL(uattrs.ACL)
}
// TODO(cathyo): Handle metadata. Pending b/230510191.
return o
}
// ObjectAttrs represents the metadata for a Google Cloud Storage (GCS) object.
type ObjectAttrs struct {
// Bucket is the name of the bucket containing this GCS object.
@@ -1321,6 +1245,10 @@ type ObjectAttrs struct {
// Metadata represents user-provided metadata, in key/value pairs.
// It can be nil if no metadata is provided.
//
// For object downloads using Reader, metadata keys are sent as headers.
// Therefore, avoid setting metadata keys using characters that are not valid
// for headers. See https://www.rfc-editor.org/rfc/rfc7230#section-3.2.6.
Metadata map[string]string
// Generation is the generation number of the object's content.
@@ -1387,6 +1315,11 @@ type ObjectAttrs struct {
// later value but not to an earlier one. For more information see
// https://cloud.google.com/storage/docs/metadata#custom-time .
CustomTime time.Time
// ComponentCount is the number of objects contained within a composite object.
// For non-composite objects, the value will be zero.
// This field is read-only.
ComponentCount int64
}
// convertTime converts a time in RFC3339 format to time.Time.
@@ -1457,6 +1390,7 @@ func newObject(o *raw.Object) *ObjectAttrs {
Updated: convertTime(o.Updated),
Etag: o.Etag,
CustomTime: convertTime(o.CustomTime),
ComponentCount: o.ComponentCount,
}
}
@@ -1484,12 +1418,14 @@ func newObjectFromProto(o *storagepb.Object) *ObjectAttrs {
Generation: o.Generation,
Metageneration: o.Metageneration,
StorageClass: o.StorageClass,
CustomerKeySHA256: string(o.GetCustomerEncryption().GetKeySha256Bytes()),
KMSKeyName: o.GetKmsKey(),
Created: convertProtoTime(o.GetCreateTime()),
Deleted: convertProtoTime(o.GetDeleteTime()),
Updated: convertProtoTime(o.GetUpdateTime()),
CustomTime: convertProtoTime(o.GetCustomTime()),
// CustomerKeySHA256 needs to be presented as base64 encoded, but the response from gRPC is not.
CustomerKeySHA256: base64.StdEncoding.EncodeToString(o.GetCustomerEncryption().GetKeySha256Bytes()),
KMSKeyName: o.GetKmsKey(),
Created: convertProtoTime(o.GetCreateTime()),
Deleted: convertProtoTime(o.GetDeleteTime()),
Updated: convertProtoTime(o.GetUpdateTime()),
CustomTime: convertProtoTime(o.GetCustomTime()),
ComponentCount: int64(o.ComponentCount),
}
}
@@ -1556,10 +1492,11 @@ type Query struct {
// object will be included in the results.
Versions bool
// fieldSelection is used to select only specific fields to be returned by
// the query. It's used internally and is populated for the user by
// calling Query.SetAttrSelection
fieldSelection string
// attrSelection is used to select only specific fields to be returned by
// the query. It is set by the user calling calling SetAttrSelection. These
// are used by toFieldMask and toFieldSelection for gRPC and HTTP/JSON
// clients repsectively.
attrSelection []string
// StartOffset is used to filter results to objects whose names are
// lexicographically equal to or after startOffset. If endOffset is also set,
@@ -1617,6 +1554,41 @@ var attrToFieldMap = map[string]string{
"Updated": "updated",
"Etag": "etag",
"CustomTime": "customTime",
"ComponentCount": "componentCount",
}
// attrToProtoFieldMap maps the field names of ObjectAttrs to the underlying field
// names in the protobuf Object message.
var attrToProtoFieldMap = map[string]string{
"Name": "name",
"Bucket": "bucket",
"Etag": "etag",
"Generation": "generation",
"Metageneration": "metageneration",
"StorageClass": "storage_class",
"Size": "size",
"ContentEncoding": "content_encoding",
"ContentDisposition": "content_disposition",
"CacheControl": "cache_control",
"ACL": "acl",
"ContentLanguage": "content_language",
"Deleted": "delete_time",
"ContentType": "content_type",
"Created": "create_time",
"CRC32C": "checksums.crc32c",
"MD5": "checksums.md5_hash",
"Updated": "update_time",
"KMSKeyName": "kms_key",
"TemporaryHold": "temporary_hold",
"RetentionExpirationTime": "retention_expire_time",
"Metadata": "metadata",
"EventBasedHold": "event_based_hold",
"Owner": "owner",
"CustomerKeySHA256": "customer_encryption",
"CustomTime": "custom_time",
"ComponentCount": "component_count",
// MediaLink was explicitly excluded from the proto as it is an HTTP-ism.
// "MediaLink": "mediaLink",
}
// SetAttrSelection makes the query populate only specific attributes of
@@ -1627,16 +1599,42 @@ var attrToFieldMap = map[string]string{
// optimization; for more information, see
// https://cloud.google.com/storage/docs/json_api/v1/how-tos/performance
func (q *Query) SetAttrSelection(attrs []string) error {
// Validate selections.
for _, attr := range attrs {
// If the attr is acceptable for one of the two sets, then it is OK.
// If it is not acceptable for either, then return an error.
// The respective masking implementations ignore unknown attrs which
// makes switching between transports a little easier.
_, okJSON := attrToFieldMap[attr]
_, okGRPC := attrToProtoFieldMap[attr]
if !okJSON && !okGRPC {
return fmt.Errorf("storage: attr %v is not valid", attr)
}
}
q.attrSelection = attrs
return nil
}
func (q *Query) toFieldSelection() string {
if q == nil || len(q.attrSelection) == 0 {
return ""
}
fieldSet := make(map[string]bool)
for _, attr := range attrs {
for _, attr := range q.attrSelection {
field, ok := attrToFieldMap[attr]
if !ok {
return fmt.Errorf("storage: attr %v is not valid", attr)
// Future proofing, skip unknown fields, let SetAttrSelection handle
// error modes.
continue
}
fieldSet[field] = true
}
var s string
if len(fieldSet) > 0 {
var b bytes.Buffer
b.WriteString("prefixes,items(")
@@ -1649,9 +1647,50 @@ func (q *Query) SetAttrSelection(attrs []string) error {
b.WriteString(field)
}
b.WriteString(")")
q.fieldSelection = b.String()
s = b.String()
}
return nil
return s
}
func (q *Query) toFieldMask() *fieldmaskpb.FieldMask {
// The default behavior with no Query is ProjectionDefault (i.e. ProjectionFull).
if q == nil {
return &fieldmaskpb.FieldMask{Paths: []string{"*"}}
}
// User selected attributes via q.SetAttrSeleciton. This takes precedence
// over the Projection.
if numSelected := len(q.attrSelection); numSelected > 0 {
protoFieldPaths := make([]string, 0, numSelected)
for _, attr := range q.attrSelection {
pf, ok := attrToProtoFieldMap[attr]
if !ok {
// Future proofing, skip unknown fields, let SetAttrSelection
// handle error modes.
continue
}
protoFieldPaths = append(protoFieldPaths, pf)
}
return &fieldmaskpb.FieldMask{Paths: protoFieldPaths}
}
// ProjectDefault == ProjectionFull which means all fields.
fm := &fieldmaskpb.FieldMask{Paths: []string{"*"}}
if q.Projection == ProjectionNoACL {
paths := make([]string, 0, len(attrToProtoFieldMap)-2) // omitting two fields
for _, f := range attrToProtoFieldMap {
// Skip the acl and owner fields for "NoACL".
if f == "acl" || f == "owner" {
continue
}
paths = append(paths, f)
}
fm.Paths = paths
}
return fm
}
// Conditions constrain methods to act on specific generations of
@@ -1795,6 +1834,33 @@ func applySourceConds(gen int64, conds *Conditions, call *raw.ObjectsRewriteCall
return nil
}
func applySourceCondsProto(gen int64, conds *Conditions, call *storagepb.RewriteObjectRequest) error {
if gen >= 0 {
call.SourceGeneration = gen
}
if conds == nil {
return nil
}
if err := conds.validate("CopyTo source"); err != nil {
return err
}
switch {
case conds.GenerationMatch != 0:
call.IfSourceGenerationMatch = proto.Int64(conds.GenerationMatch)
case conds.GenerationNotMatch != 0:
call.IfSourceGenerationNotMatch = proto.Int64(conds.GenerationNotMatch)
case conds.DoesNotExist:
call.IfSourceGenerationMatch = proto.Int64(0)
}
switch {
case conds.MetagenerationMatch != 0:
call.IfSourceMetagenerationMatch = proto.Int64(conds.MetagenerationMatch)
case conds.MetagenerationNotMatch != 0:
call.IfSourceMetagenerationNotMatch = proto.Int64(conds.MetagenerationNotMatch)
}
return nil
}
// setConditionField sets a field on a *raw.WhateverCall.
// We can't use anonymous interfaces because the return type is
// different, since the field setters are builders.
@@ -1916,8 +1982,8 @@ func (ws *withPolicy) apply(config *retryConfig) {
// WithErrorFunc allows users to pass a custom function to the retryer. Errors
// will be retried if and only if `shouldRetry(err)` returns true.
// By default, the following errors are retried (see invoke.go for the default
// shouldRetry function):
// By default, the following errors are retried (see ShouldRetry for the default
// function):
//
// - HTTP responses with codes 408, 429, 502, 503, and 504.
//
@@ -1928,7 +1994,8 @@ func (ws *withPolicy) apply(config *retryConfig) {
// - Wrapped versions of these errors.
//
// This option can be used to retry on a different set of errors than the
// default.
// default. Users can use the default ShouldRetry function inside their custom
// function if they only want to make minor modifications to default behavior.
func WithErrorFunc(shouldRetry func(err error) bool) RetryOption {
return &withErrorFunc{
shouldRetry: shouldRetry,
@@ -2001,26 +2068,50 @@ func setEncryptionHeaders(headers http.Header, key []byte, copySource bool) erro
if copySource {
cs = "copy-source-"
}
headers.Set("x-goog-"+cs+"encryption-algorithm", "AES256")
headers.Set("x-goog-"+cs+"encryption-algorithm", aes256Algorithm)
headers.Set("x-goog-"+cs+"encryption-key", base64.StdEncoding.EncodeToString(key))
keyHash := sha256.Sum256(key)
headers.Set("x-goog-"+cs+"encryption-key-sha256", base64.StdEncoding.EncodeToString(keyHash[:]))
return nil
}
// toProtoCommonObjectRequestParams sets customer-supplied encryption to the proto library's CommonObjectRequestParams.
func toProtoCommonObjectRequestParams(key []byte) *storagepb.CommonObjectRequestParams {
if key == nil {
return nil
}
keyHash := sha256.Sum256(key)
return &storagepb.CommonObjectRequestParams{
EncryptionAlgorithm: aes256Algorithm,
EncryptionKeyBytes: key,
EncryptionKeySha256Bytes: keyHash[:],
}
}
func toProtoChecksums(sendCRC32C bool, attrs *ObjectAttrs) *storagepb.ObjectChecksums {
var checksums *storagepb.ObjectChecksums
if sendCRC32C {
checksums = &storagepb.ObjectChecksums{
Crc32C: proto.Uint32(attrs.CRC32C),
}
}
if len(attrs.MD5) != 0 {
if checksums == nil {
checksums = &storagepb.ObjectChecksums{
Md5Hash: attrs.MD5,
}
} else {
checksums.Md5Hash = attrs.MD5
}
}
return checksums
}
// ServiceAccount fetches the email address of the given project's Google Cloud Storage service account.
func (c *Client) ServiceAccount(ctx context.Context, projectID string) (string, error) {
r := c.raw.Projects.ServiceAccount.Get(projectID)
var res *raw.ServiceAccount
var err error
err = run(ctx, func() error {
res, err = r.Context(ctx).Do()
return err
}, c.retry, true)
if err != nil {
return "", err
}
return res.EmailAddress, nil
o := makeStorageOpts(true, c.retry, "")
return c.tc.GetServiceAccount(ctx, projectID, o...)
}
// bucketResourceName formats the given project ID and bucketResourceName ID
@@ -2037,6 +2128,24 @@ func parseBucketName(b string) string {
return b[sep+1:]
}
// parseProjectNumber consume the given resource name and parses out the project
// number if one is present i.e. it is not a project ID.
func parseProjectNumber(r string) uint64 {
projectID := regexp.MustCompile(`projects\/([0-9]+)\/?`)
if matches := projectID.FindStringSubmatch(r); len(matches) > 0 {
// Capture group follows the matched segment. For example:
// input: projects/123/bars/456
// output: [projects/123/, 123]
number, err := strconv.ParseUint(matches[1], 10, 64)
if err != nil {
return 0
}
return number
}
return 0
}
// toProjectResource accepts a project ID and formats it as a Project resource
// name.
func toProjectResource(project string) string {
vendor/cloud.google.com/go/storage/writer.go
Generated Vendored
+53 -474
View File
@@ -16,29 +16,12 @@ package storage
import (
"context"
"encoding/base64"
"errors"
"fmt"
"io"
"sync"
"time"
"unicode/utf8"
"github.com/golang/protobuf/proto"
"golang.org/x/xerrors"
"google.golang.org/api/googleapi"
raw "google.golang.org/api/storage/v1"
storagepb "google.golang.org/genproto/googleapis/storage/v2"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
// Maximum amount of content that can be sent per WriteObjectRequest message.
// A buffer reaching this amount will precipitate a flush of the buffer.
//
// This is only used for the gRPC-based Writer.
maxPerMessageWriteSize int = int(storagepb.ServiceConstants_MAX_WRITE_CHUNK_BYTES)
)
// A Writer writes a Cloud Storage object.
@@ -62,21 +45,26 @@ type Writer struct {
// ChunkSize controls the maximum number of bytes of the object that the
// Writer will attempt to send to the server in a single request. Objects
// smaller than the size will be sent in a single request, while larger
// objects will be split over multiple requests. The size will be rounded up
// to the nearest multiple of 256K.
// objects will be split over multiple requests. The value will be rounded up
// to the nearest multiple of 256K. The default ChunkSize is 16MiB.
//
// ChunkSize will default to a reasonable value. If you perform many
// concurrent writes of small objects (under ~8MB), you may wish set ChunkSize
// to a value that matches your objects' sizes to avoid consuming large
// amounts of memory. See
// Each Writer will internally allocate a buffer of size ChunkSize. This is
// used to buffer input data and allow for the input to be sent again if a
// request must be retried.
//
// If you upload small objects (< 16MiB), you should set ChunkSize
// to a value slightly larger than the objects' sizes to avoid memory bloat.
// This is especially important if you are uploading many small objects
// concurrently. See
// https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload#size
// for more information about performance trade-offs related to ChunkSize.
//
// If ChunkSize is set to zero, chunking will be disabled and the object will
// be uploaded in a single request without the use of a buffer. This will
// further reduce memory used during uploads, but will also prevent the writer
// from retrying in case of a transient error from the server, since a buffer
// is required in order to retry the failed request.
// from retrying in case of a transient error from the server or resuming an
// upload that fails midway through, since the buffer is required in order to
// retry the failed request.
//
// ChunkSize must be set before the first Write call.
ChunkSize int
@@ -119,112 +107,6 @@ type Writer struct {
mu sync.Mutex
err error
// The gRPC client-stream used for sending buffers.
//
// This is an experimental API and not intended for public use.
stream storagepb.Storage_WriteObjectClient
// The Resumable Upload ID started by a gRPC-based Writer.
//
// This is an experimental API and not intended for public use.
upid string
}
func (w *Writer) open() error {
if err := w.validateWriteAttrs(); err != nil {
return err
}
pr, pw := io.Pipe()
w.pw = pw
w.opened = true
go w.monitorCancel()
attrs := w.ObjectAttrs
mediaOpts := []googleapi.MediaOption{
googleapi.ChunkSize(w.ChunkSize),
}
if c := attrs.ContentType; c != "" {
mediaOpts = append(mediaOpts, googleapi.ContentType(c))
}
if w.ChunkRetryDeadline != 0 {
mediaOpts = append(mediaOpts, googleapi.ChunkRetryDeadline(w.ChunkRetryDeadline))
}
go func() {
defer close(w.donec)
rawObj := attrs.toRawObject(w.o.bucket)
if w.SendCRC32C {
rawObj.Crc32c = encodeUint32(attrs.CRC32C)
}
if w.MD5 != nil {
rawObj.Md5Hash = base64.StdEncoding.EncodeToString(w.MD5)
}
call := w.o.c.raw.Objects.Insert(w.o.bucket, rawObj).
Media(pr, mediaOpts...).
Projection("full").
Context(w.ctx).
Name(w.o.object)
if w.ProgressFunc != nil {
call.ProgressUpdater(func(n, _ int64) { w.ProgressFunc(n) })
}
if attrs.KMSKeyName != "" {
call.KmsKeyName(attrs.KMSKeyName)
}
if attrs.PredefinedACL != "" {
call.PredefinedAcl(attrs.PredefinedACL)
}
if err := setEncryptionHeaders(call.Header(), w.o.encryptionKey, false); err != nil {
w.mu.Lock()
w.err = err
w.mu.Unlock()
pr.CloseWithError(err)
return
}
var resp *raw.Object
err := applyConds("NewWriter", w.o.gen, w.o.conds, call)
if err == nil {
if w.o.userProject != "" {
call.UserProject(w.o.userProject)
}
setClientHeader(call.Header())
// The internals that perform call.Do automatically retry both the initial
// call to set up the upload as well as calls to upload individual chunks
// for a resumable upload (as long as the chunk size is non-zero). Hence
// there is no need to add retries here.
// Retry only when the operation is idempotent or the retry policy is RetryAlways.
isIdempotent := w.o.conds != nil && (w.o.conds.GenerationMatch >= 0 || w.o.conds.DoesNotExist == true)
var useRetry bool
if (w.o.retry == nil || w.o.retry.policy == RetryIdempotent) && isIdempotent {
useRetry = true
} else if w.o.retry != nil && w.o.retry.policy == RetryAlways {
useRetry = true
}
if useRetry {
if w.o.retry != nil {
call.WithRetry(w.o.retry.backoff, w.o.retry.shouldRetry)
} else {
call.WithRetry(nil, nil)
}
}
resp, err = call.Do()
}
if err != nil {
w.mu.Lock()
w.err = err
w.mu.Unlock()
pr.CloseWithError(err)
return
}
w.obj = newObject(resp)
}()
return nil
}
// Write appends to w. It implements the io.Writer interface.
@@ -244,12 +126,7 @@ func (w *Writer) Write(p []byte) (n int, err error) {
return 0, werr
}
if !w.opened {
// gRPC client has been initialized - use gRPC to upload.
if w.o.c.gc != nil {
if err := w.openGRPC(); err != nil {
return 0, err
}
} else if err := w.open(); err != nil {
if err := w.openWriter(); err != nil {
return 0, err
}
}
@@ -261,7 +138,7 @@ func (w *Writer) Write(p []byte) (n int, err error) {
// Preserve existing functionality that when context is canceled, Write will return
// context.Canceled instead of "io: read/write on closed pipe". This hides the
// pipe implementation detail from users and makes Write seem as though it's an RPC.
if xerrors.Is(werr, context.Canceled) || xerrors.Is(werr, context.DeadlineExceeded) {
if errors.Is(werr, context.Canceled) || errors.Is(werr, context.DeadlineExceeded) {
return n, werr
}
}
@@ -273,7 +150,7 @@ func (w *Writer) Write(p []byte) (n int, err error) {
// can be retrieved by calling Attrs.
func (w *Writer) Close() error {
if !w.opened {
if err := w.open(); err != nil {
if err := w.openWriter(); err != nil {
return err
}
}
@@ -289,6 +166,43 @@ func (w *Writer) Close() error {
return w.err
}
func (w *Writer) openWriter() (err error) {
if err := w.validateWriteAttrs(); err != nil {
return err
}
if w.o.gen != defaultGen {
return fmt.Errorf("storage: generation not supported on Writer, got %v", w.o.gen)
}
isIdempotent := w.o.conds != nil && (w.o.conds.GenerationMatch >= 0 || w.o.conds.DoesNotExist == true)
opts := makeStorageOpts(isIdempotent, w.o.retry, w.o.userProject)
params := &openWriterParams{
ctx: w.ctx,
chunkSize: w.ChunkSize,
chunkRetryDeadline: w.ChunkRetryDeadline,
bucket: w.o.bucket,
attrs: &w.ObjectAttrs,
conds: w.o.conds,
encryptionKey: w.o.encryptionKey,
sendCRC32C: w.SendCRC32C,
donec: w.donec,
setError: w.error,
progress: w.progress,
setObj: func(o *ObjectAttrs) { w.obj = o },
}
if err := w.ctx.Err(); err != nil {
return err // short-circuit
}
w.pw, err = w.o.c.tc.OpenWriter(params, opts...)
if err != nil {
return err
}
w.opened = true
go w.monitorCancel()
return nil
}
// monitorCancel is intended to be used as a background goroutine. It monitors the
// context, and when it observes that the context has been canceled, it manually
// closes things that do not take a context.
@@ -357,338 +271,3 @@ func (w *Writer) error(err error) {
w.err = err
w.mu.Unlock()
}
// openGRPC initializes a pipe for the user to write data to, and a routine to
// read from that pipe and upload the data to GCS via gRPC.
//
// This is an experimental API and not intended for public use.
func (w *Writer) openGRPC() error {
if err := w.validateWriteAttrs(); err != nil {
return err
}
pr, pw := io.Pipe()
w.pw = pw
w.opened = true
go w.monitorCancel()
bufSize := w.ChunkSize
if w.ChunkSize == 0 {
// TODO: Should we actually use the minimum of 256 KB here when the user
// indicates they want minimal memory usage? We cannot do a zero-copy,
// bufferless upload like HTTP/JSON can.
// TODO: We need to determine if we can avoid starting a
// resumable upload when the user *plans* to send more than bufSize but
// with a bufferless upload.
bufSize = maxPerMessageWriteSize
}
buf := make([]byte, bufSize)
var offset int64
// This function reads the data sent to the pipe and sends sets of messages
// on the gRPC client-stream as the buffer is filled.
go func() {
defer close(w.donec)
// Loop until there is an error or the Object has been finalized.
for {
// Note: This blocks until either the buffer is full or EOF is read.
recvd, doneReading, err := read(pr, buf)
if err != nil {
err = checkCanceled(err)
w.error(err)
pr.CloseWithError(err)
return
}
toWrite := buf[:recvd]
// TODO: Figure out how to set up encryption via CommonObjectRequestParams.
// The chunk buffer is full, but there is no end in sight. This
// means that a resumable upload will need to be used to send
// multiple chunks, until we are done reading data. Start a
// resumable upload if it has not already been started.
// Otherwise, all data will be sent over a single gRPC stream.
if !doneReading && w.upid == "" {
err = w.startResumableUpload()
if err != nil {
err = checkCanceled(err)
w.error(err)
pr.CloseWithError(err)
return
}
}
o, off, finalized, err := w.uploadBuffer(toWrite, recvd, offset, doneReading)
if err != nil {
err = checkCanceled(err)
w.error(err)
pr.CloseWithError(err)
return
}
// At this point, the current buffer has been uploaded. Capture the
// committed offset here in case the upload was not finalized and
// another chunk is to be uploaded.
offset = off
w.progress(offset)
// When we are done reading data and the chunk has been finalized,
// we are done.
if doneReading && finalized {
// Build Object from server's response.
w.obj = newObjectFromProto(o)
return
}
}
}()
return nil
}
// startResumableUpload initializes a Resumable Upload with gRPC and sets the
// upload ID on the Writer.
//
// This is an experimental API and not intended for public use.
func (w *Writer) startResumableUpload() error {
var common *storagepb.CommonRequestParams
if w.o.userProject != "" {
common = &storagepb.CommonRequestParams{UserProject: w.o.userProject}
}
spec, err := w.writeObjectSpec()
if err != nil {
return err
}
upres, err := w.o.c.gc.StartResumableWrite(w.ctx, &storagepb.StartResumableWriteRequest{
WriteObjectSpec: spec,
CommonRequestParams: common,
})
w.upid = upres.GetUploadId()
return err
}
// queryProgress is a helper that queries the status of the resumable upload
// associated with the given upload ID.
//
// This is an experimental API and not intended for public use.
func (w *Writer) queryProgress() (int64, error) {
q, err := w.o.c.gc.QueryWriteStatus(w.ctx, &storagepb.QueryWriteStatusRequest{UploadId: w.upid})
// q.GetCommittedSize() will return 0 if q is nil.
return q.GetPersistedSize(), err
}
// uploadBuffer opens a Write stream and uploads the buffer at the given offset (if
// uploading a chunk for a resumable uploadBuffer), and will mark the write as
// finished if we are done receiving data from the user. The resulting write
// offset after uploading the buffer is returned, as well as a boolean
// indicating if the Object has been finalized. If it has been finalized, the
// final Object will be returned as well. Finalizing the upload is primarily
// important for Resumable Uploads. A simple or multi-part upload will always
// be finalized once the entire buffer has been written.
//
// This is an experimental API and not intended for public use.
func (w *Writer) uploadBuffer(buf []byte, recvd int, start int64, doneReading bool) (*storagepb.Object, int64, bool, error) {
var err error
var finishWrite bool
var sent, limit int = 0, maxPerMessageWriteSize
offset := start
for {
first := sent == 0
// This indicates that this is the last message and the remaining
// data fits in one message.
belowLimit := recvd-sent <= limit
if belowLimit {
limit = recvd - sent
}
if belowLimit && doneReading {
finishWrite = true
}
// Prepare chunk section for upload.
data := buf[sent : sent+limit]
req := &storagepb.WriteObjectRequest{
Data: &storagepb.WriteObjectRequest_ChecksummedData{
ChecksummedData: &storagepb.ChecksummedData{
Content: data,
},
},
WriteOffset: offset,
FinishWrite: finishWrite,
}
// Open a new stream and set the first_message field on the request.
// The first message on the WriteObject stream must either be the
// Object or the Resumable Upload ID.
if first {
w.stream, err = w.o.c.gc.WriteObject(w.ctx)
if err != nil {
return nil, 0, false, err
}
if w.upid != "" {
req.FirstMessage = &storagepb.WriteObjectRequest_UploadId{UploadId: w.upid}
} else {
spec, err := w.writeObjectSpec()
if err != nil {
return nil, 0, false, err
}
req.FirstMessage = &storagepb.WriteObjectRequest_WriteObjectSpec{
WriteObjectSpec: spec,
}
}
// TODO: Currently the checksums are only sent on the first message
// of the stream, but in the future, we must also support sending it
// on the *last* message of the stream (instead of the first).
if w.SendCRC32C {
req.ObjectChecksums = &storagepb.ObjectChecksums{
Crc32C: proto.Uint32(w.CRC32C),
Md5Hash: w.MD5,
}
}
}
err = w.stream.Send(req)
if err == io.EOF {
// err was io.EOF. The client-side of a stream only gets an EOF on Send
// when the backend closes the stream and wants to return an error
// status. Closing the stream receives the status as an error.
_, err = w.stream.CloseAndRecv()
// Retriable errors mean we should start over and attempt to
// resend the entire buffer via a new stream.
// If not retriable, falling through will return the error received
// from closing the stream.
if shouldRetry(err) {
sent = 0
finishWrite = false
// TODO: Add test case for failure modes of querying progress.
offset, err = w.determineOffset(start)
if err == nil {
continue
}
}
}
if err != nil {
return nil, 0, false, err
}
// Update the immediate stream's sent total and the upload offset with
// the data sent.
sent += len(data)
offset += int64(len(data))
// Not done sending data, do not attempt to commit it yet, loop around
// and send more data.
if recvd-sent > 0 {
continue
}
// Done sending data. Close the stream to "commit" the data sent.
resp, finalized, err := w.commit()
// Retriable errors mean we should start over and attempt to
// resend the entire buffer via a new stream.
// If not retriable, falling through will return the error received
// from closing the stream.
if shouldRetry(err) {
sent = 0
finishWrite = false
offset, err = w.determineOffset(start)
if err == nil {
continue
}
}
if err != nil {
return nil, 0, false, err
}
return resp.GetResource(), offset, finalized, nil
}
}
// determineOffset either returns the offset given to it in the case of a simple
// upload, or queries the write status in the case a resumable upload is being
// used.
//
// This is an experimental API and not intended for public use.
func (w *Writer) determineOffset(offset int64) (int64, error) {
// For a Resumable Upload, we must start from however much data
// was committed.
if w.upid != "" {
committed, err := w.queryProgress()
if err != nil {
return 0, err
}
offset = committed
}
return offset, nil
}
// commit closes the stream to commit the data sent and potentially receive
// the finalized object if finished uploading. If the last request sent
// indicated that writing was finished, the Object will be finalized and
// returned. If not, then the Object will be nil, and the boolean returned will
// be false.
//
// This is an experimental API and not intended for public use.
func (w *Writer) commit() (*storagepb.WriteObjectResponse, bool, error) {
finalized := true
resp, err := w.stream.CloseAndRecv()
if err == io.EOF {
// Closing a stream for a resumable upload finish_write = false results
// in an EOF which can be ignored, as we aren't done uploading yet.
finalized = false
err = nil
}
// Drop the stream reference as it has been closed.
w.stream = nil
return resp, finalized, err
}
// writeObjectSpec constructs a WriteObjectSpec proto using the Writer's
// ObjectAttrs and applies its Conditions. This is only used for gRPC.
//
// This is an experimental API and not intended for public use.
func (w *Writer) writeObjectSpec() (*storagepb.WriteObjectSpec, error) {
spec := &storagepb.WriteObjectSpec{
Resource: w.ObjectAttrs.toProtoObject(w.o.bucket),
}
// WriteObject doesn't support the generation condition, so use -1.
if err := applyCondsProto("WriteObject", -1, w.o.conds, spec); err != nil {
return nil, err
}
return spec, nil
}
// read copies the data in the reader to the given buffer and reports how much
// data was read into the buffer and if there is no more data to read (EOF).
//
// This is an experimental API and not intended for public use.
func read(r io.Reader, buf []byte) (int, bool, error) {
// Set n to -1 to start the Read loop.
var n, recvd int = -1, 0
var err error
for err == nil && n != 0 {
// The routine blocks here until data is received.
n, err = r.Read(buf[recvd:])
recvd += n
}
var done bool
if err == io.EOF {
done = true
err = nil
}
return recvd, done, err
}
func checkCanceled(err error) error {
if status.Code(err) == codes.Canceled {
return context.Canceled
}
return err
}
vendor/cloud.google.com/go/testing.md
Generated Vendored
-236
View File
@@ -1,236 +0,0 @@
# Testing Code that depends on Go Client Libraries
The Go client libraries generated as a part of `cloud.google.com/go` all take
the approach of returning concrete types instead of interfaces. That way, new
fields and methods can be added to the libraries without breaking users. This
document will go over some patterns that can be used to test code that depends
on the Go client libraries.
## Testing gRPC services using fakes
*Note*: You can see the full
[example code using a fake here](https://github.com/googleapis/google-cloud-go/tree/main/internal/examples/fake).
The clients found in `cloud.google.com/go` are gRPC based, with a couple of
notable exceptions being the [`storage`](https://pkg.go.dev/cloud.google.com/go/storage)
and [`bigquery`](https://pkg.go.dev/cloud.google.com/go/bigquery) clients.
Interactions with gRPC services can be faked by serving up your own in-memory
server within your test. One benefit of using this approach is that you don’t
need to define an interface in your runtime code; you can keep using
concrete struct types. You instead define a fake server in your test code. For
example, take a look at the following function:
```go
import (
"context"
"fmt"
"log"
"os"
translate "cloud.google.com/go/translate/apiv3"
"github.com/googleapis/gax-go/v2"
translatepb "google.golang.org/genproto/googleapis/cloud/translate/v3"
)
func TranslateTextWithConcreteClient(client *translate.TranslationClient, text string, targetLang string) (string, error) {
ctx := context.Background()
log.Printf("Translating %q to %q", text, targetLang)
req := &translatepb.TranslateTextRequest{
Parent: fmt.Sprintf("projects/%s/locations/global", os.Getenv("GOOGLE_CLOUD_PROJECT")),
TargetLanguageCode: "en-US",
Contents: []string{text},
}
resp, err := client.TranslateText(ctx, req)
if err != nil {
return "", fmt.Errorf("unable to translate text: %v", err)
}
translations := resp.GetTranslations()
if len(translations) != 1 {
return "", fmt.Errorf("expected only one result, got %d", len(translations))
}
return translations[0].TranslatedText, nil
}
```
Here is an example of what a fake server implementation would look like for
faking the interactions above:
```go
import (
"context"
translatepb "google.golang.org/genproto/googleapis/cloud/translate/v3"
)
type fakeTranslationServer struct {
translatepb.UnimplementedTranslationServiceServer
}
func (f *fakeTranslationServer) TranslateText(ctx context.Context, req *translatepb.TranslateTextRequest) (*translatepb.TranslateTextResponse, error) {
resp := &translatepb.TranslateTextResponse{
Translations: []*translatepb.Translation{
&translatepb.Translation{
TranslatedText: "Hello World",
},
},
}
return resp, nil
}
```
All of the generated protobuf code found in [google.golang.org/genproto](https://pkg.go.dev/google.golang.org/genproto)
contains a similar `package.UnimplmentedFooServer` type that is useful for
creating fakes. By embedding the unimplemented server in the
`fakeTranslationServer`, the fake will “inherit” all of the RPCs the server
exposes. Then, by providing our own `fakeTranslationServer.TranslateText`
method you can “override” the default unimplemented behavior of the one RPC that
you would like to be faked.
The test itself does require a little bit of setup: start up a `net.Listener`,
register the server, and tell the client library to call the server:
```go
import (
"context"
"net"
"testing"
translate "cloud.google.com/go/translate/apiv3"
"google.golang.org/api/option"
translatepb "google.golang.org/genproto/googleapis/cloud/translate/v3"
"google.golang.org/grpc"
)
func TestTranslateTextWithConcreteClient(t *testing.T) {
ctx := context.Background()
// Setup the fake server.
fakeTranslationServer := &fakeTranslationServer{}
l, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatal(err)
}
gsrv := grpc.NewServer()
translatepb.RegisterTranslationServiceServer(gsrv, fakeTranslationServer)
fakeServerAddr := l.Addr().String()
go func() {
if err := gsrv.Serve(l); err != nil {
panic(err)
}
}()
// Create a client.
client, err := translate.NewTranslationClient(ctx,
option.WithEndpoint(fakeServerAddr),
option.WithoutAuthentication(),
option.WithGRPCDialOption(grpc.WithInsecure()),
)
if err != nil {
t.Fatal(err)
}
// Run the test.
text, err := TranslateTextWithConcreteClient(client, "Hola Mundo", "en-US")
if err != nil {
t.Fatal(err)
}
if text != "Hello World" {
t.Fatalf("got %q, want Hello World", text)
}
}
```
## Testing using mocks
*Note*: You can see the full
[example code using a mock here](https://github.com/googleapis/google-cloud-go/tree/main/internal/examples/mock).
When mocking code you need to work with interfaces. Let’s create an interface
for the `cloud.google.com/go/translate/apiv3` client used in the
`TranslateTextWithConcreteClient` function mentioned in the previous section.
The `translate.Client` has over a dozen methods but this code only uses one of
them. Here is an interface that satisfies the interactions of the
`translate.Client` in this function.
```go
type TranslationClient interface {
TranslateText(ctx context.Context, req *translatepb.TranslateTextRequest, opts ...gax.CallOption) (*translatepb.TranslateTextResponse, error)
}
```
Now that we have an interface that satisfies the method being used we can
rewrite the function signature to take the interface instead of the concrete
type.
```go
func TranslateTextWithInterfaceClient(client TranslationClient, text string, targetLang string) (string, error) {
// ...
}
```
This allows a real `translate.Client` to be passed to the method in production
and for a mock implementation to be passed in during testing. This pattern can
be applied to any Go code, not just `cloud.google.com/go`. This is because
interfaces in Go are implicitly satisfied. Structs in the client libraries can
implicitly implement interfaces defined in your codebase. Let’s take a look at
what it might look like to define a lightweight mock for the `TranslationClient`
interface.
```go
import (
"context"
"testing"
"github.com/googleapis/gax-go/v2"
translatepb "google.golang.org/genproto/googleapis/cloud/translate/v3"
)
type mockClient struct{}
func (*mockClient) TranslateText(_ context.Context, req *translatepb.TranslateTextRequest, opts ...gax.CallOption) (*translatepb.TranslateTextResponse, error) {
resp := &translatepb.TranslateTextResponse{
Translations: []*translatepb.Translation{
&translatepb.Translation{
TranslatedText: "Hello World",
},
},
}
return resp, nil
}
func TestTranslateTextWithAbstractClient(t *testing.T) {
client := &mockClient{}
text, err := TranslateTextWithInterfaceClient(client, "Hola Mundo", "en-US")
if err != nil {
t.Fatal(err)
}
if text != "Hello World" {
t.Fatalf("got %q, want Hello World", text)
}
}
```
If you prefer to not write your own mocks there are mocking frameworks such as
[golang/mock](https://github.com/golang/mock) which can generate mocks for you
from an interface. As a word of caution though, try to not
[overuse mocks](https://testing.googleblog.com/2013/05/testing-on-toilet-dont-overuse-mocks.html).
## Testing using emulators
Some of the client libraries provided in `cloud.google.com/go` support running
against a service emulator. The concept is similar to that of using fakes,
mentioned above, but the server is managed for you. You just need to start it up
and instruct the client library to talk to the emulator by setting a service
specific emulator environment variable. Current services/environment-variables
are:
- bigtable: `BIGTABLE_EMULATOR_HOST`
- datastore: `DATASTORE_EMULATOR_HOST`
- firestore: `FIRESTORE_EMULATOR_HOST`
- pubsub: `PUBSUB_EMULATOR_HOST`
- spanner: `SPANNER_EMULATOR_HOST`
- storage: `STORAGE_EMULATOR_HOST`
- Although the storage client supports an emulator environment variable there is no official emulator provided by gcloud.
For more information on emulators please refer to the
[gcloud documentation](https://cloud.google.com/sdk/gcloud/reference/beta/emulators).
vendor/github.com/Microsoft/go-winio/.gitattributes
Generated
+1
View File
@@ -0,0 +1 @@
* text=auto eol=lf
vendor/github.com/Microsoft/go-winio/.gitignore
Generated
+9
View File
@@ -1 +1,10 @@
.vscode/
*.exe
# testing
testdata
# go workspaces
go.work
go.work.sum
vendor/github.com/Microsoft/go-winio/.golangci.yml
Generated Vendored
+144
View File
@@ -0,0 +1,144 @@
run:
skip-dirs:
- pkg/etw/sample
linters:
enable:
# style
- containedctx # struct contains a context
- dupl # duplicate code
- errname # erorrs are named correctly
- goconst # strings that should be constants
- godot # comments end in a period
- misspell
- nolintlint # "//nolint" directives are properly explained
- revive # golint replacement
- stylecheck # golint replacement, less configurable than revive
- unconvert # unnecessary conversions
- wastedassign
# bugs, performance, unused, etc ...
- contextcheck # function uses a non-inherited context
- errorlint # errors not wrapped for 1.13
- exhaustive # check exhaustiveness of enum switch statements
- gofmt # files are gofmt'ed
- gosec # security
- nestif # deeply nested ifs
- nilerr # returns nil even with non-nil error
- prealloc # slices that can be pre-allocated
- structcheck # unused struct fields
- unparam # unused function params
issues:
exclude-rules:
# err is very often shadowed in nested scopes
- linters:
- govet
text: '^shadow: declaration of "err" shadows declaration'
# ignore long lines for skip autogen directives
- linters:
- revive
text: "^line-length-limit: "
source: "^//(go:generate|sys) "
# allow unjustified ignores of error checks in defer statements
- linters:
- nolintlint
text: "^directive `//nolint:errcheck` should provide explanation"
source: '^\s*defer '
# allow unjustified ignores of error lints for io.EOF
- linters:
- nolintlint
text: "^directive `//nolint:errorlint` should provide explanation"
source: '[=|!]= io.EOF'
linters-settings:
govet:
enable-all: true
disable:
# struct order is often for Win32 compat
# also, ignore pointer bytes/GC issues for now until performance becomes an issue
- fieldalignment
check-shadowing: true
nolintlint:
allow-leading-space: false
require-explanation: true
require-specific: true
revive:
# revive is more configurable than static check, so likely the preferred alternative to static-check
# (once the perf issue is solved: https://github.com/golangci/golangci-lint/issues/2997)
enable-all-rules:
true
# https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md
rules:
# rules with required arguments
- name: argument-limit
disabled: true
- name: banned-characters
disabled: true
- name: cognitive-complexity
disabled: true
- name: cyclomatic
disabled: true
- name: file-header
disabled: true
- name: function-length
disabled: true
- name: function-result-limit
disabled: true
- name: max-public-structs
disabled: true
# geneally annoying rules
- name: add-constant # complains about any and all strings and integers
disabled: true
- name: confusing-naming # we frequently use "Foo()" and "foo()" together
disabled: true
- name: flag-parameter # excessive, and a common idiom we use
disabled: true
# general config
- name: line-length-limit
arguments:
- 140
- name: var-naming
arguments:
- []
- - CID
- CRI
- CTRD
- DACL
- DLL
- DOS
- ETW
- FSCTL
- GCS
- GMSA
- HCS
- HV
- IO
- LCOW
- LDAP
- LPAC
- LTSC
- MMIO
- NT
- OCI
- PMEM
- PWSH
- RX
- SACl
- SID
- SMB
- TX
- VHD
- VHDX
- VMID
- VPCI
- WCOW
- WIM
stylecheck:
checks:
- "all"
- "-ST1003" # use revive's var naming
vendor/github.com/Microsoft/go-winio/README.md
Generated Vendored
+63 -11
View File
@@ -13,16 +13,60 @@ Please see the LICENSE file for licensing information.
## Contributing
This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA)
declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.microsoft.com.
This project welcomes contributions and suggestions.
Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that
you have the right to, and actually do, grant us the rights to use your contribution.
For details, visit [Microsoft CLA](https://cla.microsoft.com).
When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR
appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA.
When you submit a pull request, a CLA-bot will automatically determine whether you need to
provide a CLA and decorate the PR appropriately (e.g., label, comment).
Simply follow the instructions provided by the bot.
You will only need to do this once across all repos using our CLA.
We also require that contributors sign their commits using git commit -s or git commit --signoff to certify they either authored the work themselves
or otherwise have permission to use it in this project. Please see https://developercertificate.org/ for more info, as well as to make sure that you can
attest to the rules listed. Our CI uses the DCO Github app to ensure that all commits in a given PR are signed-off.
Additionally, the pull request pipeline requires the following steps to be performed before
mergining.
### Code Sign-Off
We require that contributors sign their commits using [`git commit --signoff`][git-commit-s]
to certify they either authored the work themselves or otherwise have permission to use it in this project.
A range of commits can be signed off using [`git rebase --signoff`][git-rebase-s].
Please see [the developer certificate](https://developercertificate.org) for more info,
as well as to make sure that you can attest to the rules listed.
Our CI uses the DCO Github app to ensure that all commits in a given PR are signed-off.
### Linting
Code must pass a linting stage, which uses [`golangci-lint`][lint].
The linting settings are stored in [`.golangci.yaml`](./.golangci.yaml), and can be run
automatically with VSCode by adding the following to your workspace or folder settings:
```json
"go.lintTool": "golangci-lint",
"go.lintOnSave": "package",
```
Additional editor [integrations options are also available][lint-ide].
Alternatively, `golangci-lint` can be [installed locally][lint-install] and run from the repo root:
```shell
# use . or specify a path to only lint a package
# to show all lint errors, use flags "--max-issues-per-linter=0 --max-same-issues=0"
> golangci-lint run ./...
```
### Go Generate
The pipeline checks that auto-generated code, via `go generate`, are up to date.
This can be done for the entire repo:
```shell
> go generate ./...
```
## Code of Conduct
@@ -30,8 +74,16 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
## Special Thanks
Thanks to natefinch for the inspiration for this library. See https://github.com/natefinch/npipe
for another named pipe implementation.
Thanks to [natefinch][natefinch] for the inspiration for this library.
See [npipe](https://github.com/natefinch/npipe) for another named pipe implementation.
[lint]: https://golangci-lint.run/
[lint-ide]: https://golangci-lint.run/usage/integrations/#editor-integration
[lint-install]: https://golangci-lint.run/usage/install/#local-installation
[git-commit-s]: https://git-scm.com/docs/git-commit#Documentation/git-commit.txt--s
[git-rebase-s]: https://git-scm.com/docs/git-rebase#Documentation/git-rebase.txt---signoff
[natefinch]: https://github.com/natefinch
vendor/github.com/Microsoft/go-winio/SECURITY.md
Generated Vendored
+41
View File
@@ -0,0 +1,41 @@
<!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
## Security
Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
## Reporting Security Issues
**Please do not report security vulnerabilities through public GitHub issues.**
Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc).
Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
* Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
* Full paths of source file(s) related to the manifestation of the issue
* The location of the affected source code (tag/branch/commit or direct URL)
* Any special configuration required to reproduce the issue
* Step-by-step instructions to reproduce the issue
* Proof-of-concept or exploit code (if possible)
* Impact of the issue, including how an attacker might exploit the issue
This information will help us triage your report more quickly.
If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
## Preferred Languages
We prefer all communications to be in English.
## Policy
Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
<!-- END MICROSOFT SECURITY.MD BLOCK -->
vendor/github.com/Microsoft/go-winio/backup.go
Generated Vendored
+29 -19
View File
@@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@@ -7,11 +8,12 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"runtime"
"syscall"
"unicode/utf16"
"golang.org/x/sys/windows"
)
//sys backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupRead
@@ -24,7 +26,7 @@ const (
BackupAlternateData
BackupLink
BackupPropertyData
BackupObjectId
BackupObjectId //revive:disable-line:var-naming ID, not Id
BackupReparseData
BackupSparseBlock
BackupTxfsData
@@ -34,14 +36,16 @@ const (
StreamSparseAttributes = uint32(8)
)
//nolint:revive // var-naming: ALL_CAPS
const (
WRITE_DAC = 0x40000
WRITE_OWNER = 0x80000
ACCESS_SYSTEM_SECURITY = 0x1000000
WRITE_DAC = windows.WRITE_DAC
WRITE_OWNER = windows.WRITE_OWNER
ACCESS_SYSTEM_SECURITY = windows.ACCESS_SYSTEM_SECURITY
)
// BackupHeader represents a backup stream of a file.
type BackupHeader struct {
//revive:disable-next-line:var-naming ID, not Id
Id uint32 // The backup stream ID
Attributes uint32 // Stream attributes
Size int64 // The size of the stream in bytes
@@ -49,8 +53,8 @@ type BackupHeader struct {
Offset int64 // The offset of the stream in the file (for BackupSparseBlock only).
}
type win32StreamId struct {
StreamId uint32
type win32StreamID struct {
StreamID uint32
Attributes uint32
Size uint64
NameSize uint32
@@ -71,7 +75,7 @@ func NewBackupStreamReader(r io.Reader) *BackupStreamReader {
// Next returns the next backup stream and prepares for calls to Read(). It skips the remainder of the current stream if
// it was not completely read.
func (r *BackupStreamReader) Next() (*BackupHeader, error) {
if r.bytesLeft > 0 {
if r.bytesLeft > 0 { //nolint:nestif // todo: flatten this
if s, ok := r.r.(io.Seeker); ok {
// Make sure Seek on io.SeekCurrent sometimes succeeds
// before trying the actual seek.
@@ -82,16 +86,16 @@ func (r *BackupStreamReader) Next() (*BackupHeader, error) {
r.bytesLeft = 0
}
}
if _, err := io.Copy(ioutil.Discard, r); err != nil {
if _, err := io.Copy(io.Discard, r); err != nil {
return nil, err
}
}
var wsi win32StreamId
var wsi win32StreamID
if err := binary.Read(r.r, binary.LittleEndian, &wsi); err != nil {
return nil, err
}
hdr := &BackupHeader{
Id: wsi.StreamId,
Id: wsi.StreamID,
Attributes: wsi.Attributes,
Size: int64(wsi.Size),
}
@@ -102,7 +106,7 @@ func (r *BackupStreamReader) Next() (*BackupHeader, error) {
}
hdr.Name = syscall.UTF16ToString(name)
}
if wsi.StreamId == BackupSparseBlock {
if wsi.StreamID == BackupSparseBlock {
if err := binary.Read(r.r, binary.LittleEndian, &hdr.Offset); err != nil {
return nil, err
}
@@ -147,8 +151,8 @@ func (w *BackupStreamWriter) WriteHeader(hdr *BackupHeader) error {
return fmt.Errorf("missing %d bytes", w.bytesLeft)
}
name := utf16.Encode([]rune(hdr.Name))
wsi := win32StreamId{
StreamId: hdr.Id,
wsi := win32StreamID{
StreamID: hdr.Id,
Attributes: hdr.Attributes,
Size: uint64(hdr.Size),
NameSize: uint32(len(name) * 2),
@@ -203,7 +207,7 @@ func (r *BackupFileReader) Read(b []byte) (int, error) {
var bytesRead uint32
err := backupRead(syscall.Handle(r.f.Fd()), b, &bytesRead, false, r.includeSecurity, &r.ctx)
if err != nil {
return 0, &os.PathError{"BackupRead", r.f.Name(), err}
return 0, &os.PathError{Op: "BackupRead", Path: r.f.Name(), Err: err}
}
runtime.KeepAlive(r.f)
if bytesRead == 0 {
@@ -216,7 +220,7 @@ func (r *BackupFileReader) Read(b []byte) (int, error) {
// the underlying file.
func (r *BackupFileReader) Close() error {
if r.ctx != 0 {
backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
_ = backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
runtime.KeepAlive(r.f)
r.ctx = 0
}
@@ -242,7 +246,7 @@ func (w *BackupFileWriter) Write(b []byte) (int, error) {
var bytesWritten uint32
err := backupWrite(syscall.Handle(w.f.Fd()), b, &bytesWritten, false, w.includeSecurity, &w.ctx)
if err != nil {
return 0, &os.PathError{"BackupWrite", w.f.Name(), err}
return 0, &os.PathError{Op: "BackupWrite", Path: w.f.Name(), Err: err}
}
runtime.KeepAlive(w.f)
if int(bytesWritten) != len(b) {
@@ -255,7 +259,7 @@ func (w *BackupFileWriter) Write(b []byte) (int, error) {
// close the underlying file.
func (w *BackupFileWriter) Close() error {
if w.ctx != 0 {
backupWrite(syscall.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx)
_ = backupWrite(syscall.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx)
runtime.KeepAlive(w.f)
w.ctx = 0
}
@@ -271,7 +275,13 @@ func OpenForBackup(path string, access uint32, share uint32, createmode uint32)
if err != nil {
return nil, err
}
h, err := syscall.CreateFile(&winPath[0], access, share, nil, createmode, syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OPEN_REPARSE_POINT, 0)
h, err := syscall.CreateFile(&winPath[0],
access,
share,
nil,
createmode,
syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OPEN_REPARSE_POINT,
0)
if err != nil {
err = &os.PathError{Op: "open", Path: path, Err: err}
return nil, err
vendor/github.com/Microsoft/go-winio/doc.go
Generated Vendored
+22
View File
@@ -0,0 +1,22 @@
// This package provides utilities for efficiently performing Win32 IO operations in Go.
// Currently, this package is provides support for genreal IO and management of
// - named pipes
// - files
// - [Hyper-V sockets]
//
// This code is similar to Go's [net] package, and uses IO completion ports to avoid
// blocking IO on system threads, allowing Go to reuse the thread to schedule other goroutines.
//
// This limits support to Windows Vista and newer operating systems.
//
// Additionally, this package provides support for:
// - creating and managing GUIDs
// - writing to [ETW]
// - opening and manageing VHDs
// - parsing [Windows Image files]
// - auto-generating Win32 API code
//
// [Hyper-V sockets]: https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/make-integration-service
// [ETW]: https://docs.microsoft.com/en-us/windows-hardware/drivers/devtest/event-tracing-for-windows--etw-
// [Windows Image files]: https://docs.microsoft.com/en-us/windows-hardware/manufacture/desktop/work-with-windows-images
package winio
vendor/github.com/Microsoft/go-winio/ea.go
Generated Vendored
+4 -4
View File
@@ -33,7 +33,7 @@ func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info)
if err != nil {
err = errInvalidEaBuffer
return
return ea, nb, err
}
nameOffset := fileFullEaInformationSize
@@ -43,7 +43,7 @@ func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
nextOffset := int(info.NextEntryOffset)
if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) {
err = errInvalidEaBuffer
return
return ea, nb, err
}
ea.Name = string(b[nameOffset : nameOffset+nameLen])
@@ -52,7 +52,7 @@ func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
if info.NextEntryOffset != 0 {
nb = b[info.NextEntryOffset:]
}
return
return ea, nb, err
}
// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION
@@ -67,7 +67,7 @@ func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) {
eas = append(eas, ea)
b = nb
}
return
return eas, err
}
func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error {
vendor/github.com/Microsoft/go-winio/file.go
Generated Vendored
+34 -32
View File
@@ -11,6 +11,8 @@ import (
"sync/atomic"
"syscall"
"time"
"golang.org/x/sys/windows"
)
//sys cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) = CancelIoEx
@@ -24,6 +26,8 @@ type atomicBool int32
func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 }
func (b *atomicBool) setFalse() { atomic.StoreInt32((*int32)(b), 0) }
func (b *atomicBool) setTrue() { atomic.StoreInt32((*int32)(b), 1) }
//revive:disable-next-line:predeclared Keep "new" to maintain consistency with "atomic" pkg
func (b *atomicBool) swap(new bool) bool {
var newInt int32
if new {
@@ -32,11 +36,6 @@ func (b *atomicBool) swap(new bool) bool {
return atomic.SwapInt32((*int32)(b), newInt) == 1
}
const (
cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS = 1
cFILE_SKIP_SET_EVENT_ON_HANDLE = 2
)
var (
ErrFileClosed = errors.New("file has already been closed")
ErrTimeout = &timeoutError{}
@@ -44,28 +43,28 @@ var (
type timeoutError struct{}
func (e *timeoutError) Error() string { return "i/o timeout" }
func (e *timeoutError) Timeout() bool { return true }
func (e *timeoutError) Temporary() bool { return true }
func (*timeoutError) Error() string { return "i/o timeout" }
func (*timeoutError) Timeout() bool { return true }
func (*timeoutError) Temporary() bool { return true }
type timeoutChan chan struct{}
var ioInitOnce sync.Once
var ioCompletionPort syscall.Handle
// ioResult contains the result of an asynchronous IO operation
// ioResult contains the result of an asynchronous IO operation.
type ioResult struct {
bytes uint32
err error
}
// ioOperation represents an outstanding asynchronous Win32 IO
// ioOperation represents an outstanding asynchronous Win32 IO.
type ioOperation struct {
o syscall.Overlapped
ch chan ioResult
}
func initIo() {
func initIO() {
h, err := createIoCompletionPort(syscall.InvalidHandle, 0, 0, 0xffffffff)
if err != nil {
panic(err)
@@ -94,15 +93,15 @@ type deadlineHandler struct {
timedout atomicBool
}
// makeWin32File makes a new win32File from an existing file handle
// makeWin32File makes a new win32File from an existing file handle.
func makeWin32File(h syscall.Handle) (*win32File, error) {
f := &win32File{handle: h}
ioInitOnce.Do(initIo)
ioInitOnce.Do(initIO)
_, err := createIoCompletionPort(h, ioCompletionPort, 0, 0xffffffff)
if err != nil {
return nil, err
}
err = setFileCompletionNotificationModes(h, cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS|cFILE_SKIP_SET_EVENT_ON_HANDLE)
err = setFileCompletionNotificationModes(h, windows.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS|windows.FILE_SKIP_SET_EVENT_ON_HANDLE)
if err != nil {
return nil, err
}
@@ -121,14 +120,14 @@ func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) {
return f, nil
}
// closeHandle closes the resources associated with a Win32 handle
// closeHandle closes the resources associated with a Win32 handle.
func (f *win32File) closeHandle() {
f.wgLock.Lock()
// Atomically set that we are closing, releasing the resources only once.
if !f.closing.swap(true) {
f.wgLock.Unlock()
// cancel all IO and wait for it to complete
cancelIoEx(f.handle, nil)
_ = cancelIoEx(f.handle, nil)
f.wg.Wait()
// at this point, no new IO can start
syscall.Close(f.handle)
@@ -144,14 +143,14 @@ func (f *win32File) Close() error {
return nil
}
// IsClosed checks if the file has been closed
// IsClosed checks if the file has been closed.
func (f *win32File) IsClosed() bool {
return f.closing.isSet()
}
// prepareIo prepares for a new IO operation.
// prepareIO prepares for a new IO operation.
// The caller must call f.wg.Done() when the IO is finished, prior to Close() returning.
func (f *win32File) prepareIo() (*ioOperation, error) {
func (f *win32File) prepareIO() (*ioOperation, error) {
f.wgLock.RLock()
if f.closing.isSet() {
f.wgLock.RUnlock()
@@ -164,7 +163,7 @@ func (f *win32File) prepareIo() (*ioOperation, error) {
return c, nil
}
// ioCompletionProcessor processes completed async IOs forever
// ioCompletionProcessor processes completed async IOs forever.
func ioCompletionProcessor(h syscall.Handle) {
for {
var bytes uint32
@@ -178,15 +177,17 @@ func ioCompletionProcessor(h syscall.Handle) {
}
}
// asyncIo processes the return value from ReadFile or WriteFile, blocking until
// todo: helsaawy - create an asyncIO version that takes a context
// asyncIO processes the return value from ReadFile or WriteFile, blocking until
// the operation has actually completed.
func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
if err != syscall.ERROR_IO_PENDING {
func (f *win32File) asyncIO(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
if err != syscall.ERROR_IO_PENDING { //nolint:errorlint // err is Errno
return int(bytes), err
}
if f.closing.isSet() {
cancelIoEx(f.handle, &c.o)
_ = cancelIoEx(f.handle, &c.o)
}
var timeout timeoutChan
@@ -200,7 +201,7 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
select {
case r = <-c.ch:
err = r.err
if err == syscall.ERROR_OPERATION_ABORTED {
if err == syscall.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno
if f.closing.isSet() {
err = ErrFileClosed
}
@@ -210,10 +211,10 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
err = wsaGetOverlappedResult(f.handle, &c.o, &bytes, false, &flags)
}
case <-timeout:
cancelIoEx(f.handle, &c.o)
_ = cancelIoEx(f.handle, &c.o)
r = <-c.ch
err = r.err
if err == syscall.ERROR_OPERATION_ABORTED {
if err == syscall.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno
err = ErrTimeout
}
}
@@ -221,13 +222,14 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
// runtime.KeepAlive is needed, as c is passed via native
// code to ioCompletionProcessor, c must remain alive
// until the channel read is complete.
// todo: (de)allocate *ioOperation via win32 heap functions, instead of needing to KeepAlive?
runtime.KeepAlive(c)
return int(r.bytes), err
}
// Read reads from a file handle.
func (f *win32File) Read(b []byte) (int, error) {
c, err := f.prepareIo()
c, err := f.prepareIO()
if err != nil {
return 0, err
}
@@ -239,13 +241,13 @@ func (f *win32File) Read(b []byte) (int, error) {
var bytes uint32
err = syscall.ReadFile(f.handle, b, &bytes, &c.o)
n, err := f.asyncIo(c, &f.readDeadline, bytes, err)
n, err := f.asyncIO(c, &f.readDeadline, bytes, err)
runtime.KeepAlive(b)
// Handle EOF conditions.
if err == nil && n == 0 && len(b) != 0 {
return 0, io.EOF
} else if err == syscall.ERROR_BROKEN_PIPE {
} else if err == syscall.ERROR_BROKEN_PIPE { //nolint:errorlint // err is Errno
return 0, io.EOF
} else {
return n, err
@@ -254,7 +256,7 @@ func (f *win32File) Read(b []byte) (int, error) {
// Write writes to a file handle.
func (f *win32File) Write(b []byte) (int, error) {
c, err := f.prepareIo()
c, err := f.prepareIO()
if err != nil {
return 0, err
}
@@ -266,7 +268,7 @@ func (f *win32File) Write(b []byte) (int, error) {
var bytes uint32
err = syscall.WriteFile(f.handle, b, &bytes, &c.o)
n, err := f.asyncIo(c, &f.writeDeadline, bytes, err)
n, err := f.asyncIO(c, &f.writeDeadline, bytes, err)
runtime.KeepAlive(b)
return n, err
}
vendor/github.com/Microsoft/go-winio/fileinfo.go
Generated Vendored
+24 -5
View File
@@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@@ -14,13 +15,18 @@ import (
type FileBasicInfo struct {
CreationTime, LastAccessTime, LastWriteTime, ChangeTime windows.Filetime
FileAttributes uint32
pad uint32 // padding
_ uint32 // padding
}
// GetFileBasicInfo retrieves times and attributes for a file.
func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
bi := &FileBasicInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
if err := windows.GetFileInformationByHandleEx(
windows.Handle(f.Fd()),
windows.FileBasicInfo,
(*byte)(unsafe.Pointer(bi)),
uint32(unsafe.Sizeof(*bi)),
); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
@@ -29,7 +35,12 @@ func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
// SetFileBasicInfo sets times and attributes for a file.
func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error {
if err := windows.SetFileInformationByHandle(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
if err := windows.SetFileInformationByHandle(
windows.Handle(f.Fd()),
windows.FileBasicInfo,
(*byte)(unsafe.Pointer(bi)),
uint32(unsafe.Sizeof(*bi)),
); err != nil {
return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
@@ -48,7 +59,10 @@ type FileStandardInfo struct {
// GetFileStandardInfo retrieves ended information for the file.
func GetFileStandardInfo(f *os.File) (*FileStandardInfo, error) {
si := &FileStandardInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileStandardInfo, (*byte)(unsafe.Pointer(si)), uint32(unsafe.Sizeof(*si))); err != nil {
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()),
windows.FileStandardInfo,
(*byte)(unsafe.Pointer(si)),
uint32(unsafe.Sizeof(*si))); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
@@ -65,7 +79,12 @@ type FileIDInfo struct {
// GetFileID retrieves the unique (volume, file ID) pair for a file.
func GetFileID(f *os.File) (*FileIDInfo, error) {
fileID := &FileIDInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileIdInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
if err := windows.GetFileInformationByHandleEx(
windows.Handle(f.Fd()),
windows.FileIdInfo,
(*byte)(unsafe.Pointer(fileID)),
uint32(unsafe.Sizeof(*fileID)),
); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
vendor/github.com/Microsoft/go-winio/hvsock.go
Generated Vendored
+302 -43
View File
@@ -4,6 +4,8 @@
package winio
import (
"context"
"errors"
"fmt"
"io"
"net"
@@ -12,16 +14,87 @@ import (
"time"
"unsafe"
"golang.org/x/sys/windows"
"github.com/Microsoft/go-winio/internal/socket"
"github.com/Microsoft/go-winio/pkg/guid"
)
//sys bind(s syscall.Handle, name unsafe.Pointer, namelen int32) (err error) [failretval==socketError] = ws2_32.bind
const afHVSock = 34 // AF_HYPERV
const (
afHvSock = 34 // AF_HYPERV
// Well known Service and VM IDs
//https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/make-integration-service#vmid-wildcards
socketError = ^uintptr(0)
)
// HvsockGUIDWildcard is the wildcard VmId for accepting connections from all partitions.
func HvsockGUIDWildcard() guid.GUID { // 00000000-0000-0000-0000-000000000000
return guid.GUID{}
}
// HvsockGUIDBroadcast is the wildcard VmId for broadcasting sends to all partitions.
func HvsockGUIDBroadcast() guid.GUID { //ffffffff-ffff-ffff-ffff-ffffffffffff
return guid.GUID{
Data1: 0xffffffff,
Data2: 0xffff,
Data3: 0xffff,
Data4: [8]uint8{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
}
}
// HvsockGUIDLoopback is the Loopback VmId for accepting connections to the same partition as the connector.
func HvsockGUIDLoopback() guid.GUID { // e0e16197-dd56-4a10-9195-5ee7a155a838
return guid.GUID{
Data1: 0xe0e16197,
Data2: 0xdd56,
Data3: 0x4a10,
Data4: [8]uint8{0x91, 0x95, 0x5e, 0xe7, 0xa1, 0x55, 0xa8, 0x38},
}
}
// HvsockGUIDSiloHost is the address of a silo's host partition:
// - The silo host of a hosted silo is the utility VM.
// - The silo host of a silo on a physical host is the physical host.
func HvsockGUIDSiloHost() guid.GUID { // 36bd0c5c-7276-4223-88ba-7d03b654c568
return guid.GUID{
Data1: 0x36bd0c5c,
Data2: 0x7276,
Data3: 0x4223,
Data4: [8]byte{0x88, 0xba, 0x7d, 0x03, 0xb6, 0x54, 0xc5, 0x68},
}
}
// HvsockGUIDChildren is the wildcard VmId for accepting connections from the connector's child partitions.
func HvsockGUIDChildren() guid.GUID { // 90db8b89-0d35-4f79-8ce9-49ea0ac8b7cd
return guid.GUID{
Data1: 0x90db8b89,
Data2: 0xd35,
Data3: 0x4f79,
Data4: [8]uint8{0x8c, 0xe9, 0x49, 0xea, 0xa, 0xc8, 0xb7, 0xcd},
}
}
// HvsockGUIDParent is the wildcard VmId for accepting connections from the connector's parent partition.
// Listening on this VmId accepts connection from:
// - Inside silos: silo host partition.
// - Inside hosted silo: host of the VM.
// - Inside VM: VM host.
// - Physical host: Not supported.
func HvsockGUIDParent() guid.GUID { // a42e7cda-d03f-480c-9cc2-a4de20abb878
return guid.GUID{
Data1: 0xa42e7cda,
Data2: 0xd03f,
Data3: 0x480c,
Data4: [8]uint8{0x9c, 0xc2, 0xa4, 0xde, 0x20, 0xab, 0xb8, 0x78},
}
}
// hvsockVsockServiceTemplate is the Service GUID used for the VSOCK protocol.
func hvsockVsockServiceTemplate() guid.GUID { // 00000000-facb-11e6-bd58-64006a7986d3
return guid.GUID{
Data2: 0xfacb,
Data3: 0x11e6,
Data4: [8]uint8{0xbd, 0x58, 0x64, 0x00, 0x6a, 0x79, 0x86, 0xd3},
}
}
// An HvsockAddr is an address for a AF_HYPERV socket.
type HvsockAddr struct {
@@ -36,8 +109,10 @@ type rawHvsockAddr struct {
ServiceID guid.GUID
}
var _ socket.RawSockaddr = &rawHvsockAddr{}
// Network returns the address's network name, "hvsock".
func (addr *HvsockAddr) Network() string {
func (*HvsockAddr) Network() string {
return "hvsock"
}
@@ -47,14 +122,14 @@ func (addr *HvsockAddr) String() string {
// VsockServiceID returns an hvsock service ID corresponding to the specified AF_VSOCK port.
func VsockServiceID(port uint32) guid.GUID {
g, _ := guid.FromString("00000000-facb-11e6-bd58-64006a7986d3")
g := hvsockVsockServiceTemplate() // make a copy
g.Data1 = port
return g
}
func (addr *HvsockAddr) raw() rawHvsockAddr {
return rawHvsockAddr{
Family: afHvSock,
Family: afHVSock,
VMID: addr.VMID,
ServiceID: addr.ServiceID,
}
@@ -65,20 +140,48 @@ func (addr *HvsockAddr) fromRaw(raw *rawHvsockAddr) {
addr.ServiceID = raw.ServiceID
}
// Sockaddr returns a pointer to and the size of this struct.
//
// Implements the [socket.RawSockaddr] interface, and allows use in
// [socket.Bind] and [socket.ConnectEx].
func (r *rawHvsockAddr) Sockaddr() (unsafe.Pointer, int32, error) {
return unsafe.Pointer(r), int32(unsafe.Sizeof(rawHvsockAddr{})), nil
}
// Sockaddr interface allows use with `sockets.Bind()` and `.ConnectEx()`.
func (r *rawHvsockAddr) FromBytes(b []byte) error {
n := int(unsafe.Sizeof(rawHvsockAddr{}))
if len(b) < n {
return fmt.Errorf("got %d, want %d: %w", len(b), n, socket.ErrBufferSize)
}
copy(unsafe.Slice((*byte)(unsafe.Pointer(r)), n), b[:n])
if r.Family != afHVSock {
return fmt.Errorf("got %d, want %d: %w", r.Family, afHVSock, socket.ErrAddrFamily)
}
return nil
}
// HvsockListener is a socket listener for the AF_HYPERV address family.
type HvsockListener struct {
sock *win32File
addr HvsockAddr
}
var _ net.Listener = &HvsockListener{}
// HvsockConn is a connected socket of the AF_HYPERV address family.
type HvsockConn struct {
sock *win32File
local, remote HvsockAddr
}
func newHvSocket() (*win32File, error) {
fd, err := syscall.Socket(afHvSock, syscall.SOCK_STREAM, 1)
var _ net.Conn = &HvsockConn{}
func newHVSocket() (*win32File, error) {
fd, err := syscall.Socket(afHVSock, syscall.SOCK_STREAM, 1)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
@@ -94,12 +197,12 @@ func newHvSocket() (*win32File, error) {
// ListenHvsock listens for connections on the specified hvsock address.
func ListenHvsock(addr *HvsockAddr) (_ *HvsockListener, err error) {
l := &HvsockListener{addr: *addr}
sock, err := newHvSocket()
sock, err := newHVSocket()
if err != nil {
return nil, l.opErr("listen", err)
}
sa := addr.raw()
err = bind(sock.handle, unsafe.Pointer(&sa), int32(unsafe.Sizeof(sa)))
err = socket.Bind(windows.Handle(sock.handle), &sa)
if err != nil {
return nil, l.opErr("listen", os.NewSyscallError("socket", err))
}
@@ -121,7 +224,7 @@ func (l *HvsockListener) Addr() net.Addr {
// Accept waits for the next connection and returns it.
func (l *HvsockListener) Accept() (_ net.Conn, err error) {
sock, err := newHvSocket()
sock, err := newHVSocket()
if err != nil {
return nil, l.opErr("accept", err)
}
@@ -130,27 +233,42 @@ func (l *HvsockListener) Accept() (_ net.Conn, err error) {
sock.Close()
}
}()
c, err := l.sock.prepareIo()
c, err := l.sock.prepareIO()
if err != nil {
return nil, l.opErr("accept", err)
}
defer l.sock.wg.Done()
// AcceptEx, per documentation, requires an extra 16 bytes per address.
//
// https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-acceptex
const addrlen = uint32(16 + unsafe.Sizeof(rawHvsockAddr{}))
var addrbuf [addrlen * 2]byte
var bytes uint32
err = syscall.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0, addrlen, addrlen, &bytes, &c.o)
_, err = l.sock.asyncIo(c, nil, bytes, err)
if err != nil {
err = syscall.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0 /*rxdatalen*/, addrlen, addrlen, &bytes, &c.o)
if _, err = l.sock.asyncIO(c, nil, bytes, err); err != nil {
return nil, l.opErr("accept", os.NewSyscallError("acceptex", err))
}
conn := &HvsockConn{
sock: sock,
}
// The local address returned in the AcceptEx buffer is the same as the Listener socket's
// address. However, the service GUID reported by GetSockName is different from the Listeners
// socket, and is sometimes the same as the local address of the socket that dialed the
// address, with the service GUID.Data1 incremented, but othertimes is different.
// todo: does the local address matter? is the listener's address or the actual address appropriate?
conn.local.fromRaw((*rawHvsockAddr)(unsafe.Pointer(&addrbuf[0])))
conn.remote.fromRaw((*rawHvsockAddr)(unsafe.Pointer(&addrbuf[addrlen])))
// initialize the accepted socket and update its properties with those of the listening socket
if err = windows.Setsockopt(windows.Handle(sock.handle),
windows.SOL_SOCKET, windows.SO_UPDATE_ACCEPT_CONTEXT,
(*byte)(unsafe.Pointer(&l.sock.handle)), int32(unsafe.Sizeof(l.sock.handle))); err != nil {
return nil, conn.opErr("accept", os.NewSyscallError("setsockopt", err))
}
sock = nil
return conn, nil
}
@@ -160,43 +278,171 @@ func (l *HvsockListener) Close() error {
return l.sock.Close()
}
/* Need to finish ConnectEx handling
func DialHvsock(ctx context.Context, addr *HvsockAddr) (*HvsockConn, error) {
sock, err := newHvSocket()
// HvsockDialer configures and dials a Hyper-V Socket (ie, [HvsockConn]).
type HvsockDialer struct {
// Deadline is the time the Dial operation must connect before erroring.
Deadline time.Time
// Retries is the number of additional connects to try if the connection times out, is refused,
// or the host is unreachable
Retries uint
// RetryWait is the time to wait after a connection error to retry
RetryWait time.Duration
rt *time.Timer // redial wait timer
}
// Dial the Hyper-V socket at addr.
//
// See [HvsockDialer.Dial] for more information.
func Dial(ctx context.Context, addr *HvsockAddr) (conn *HvsockConn, err error) {
return (&HvsockDialer{}).Dial(ctx, addr)
}
// Dial attempts to connect to the Hyper-V socket at addr, and returns a connection if successful.
// Will attempt (HvsockDialer).Retries if dialing fails, waiting (HvsockDialer).RetryWait between
// retries.
//
// Dialing can be cancelled either by providing (HvsockDialer).Deadline, or cancelling ctx.
func (d *HvsockDialer) Dial(ctx context.Context, addr *HvsockAddr) (conn *HvsockConn, err error) {
op := "dial"
// create the conn early to use opErr()
conn = &HvsockConn{
remote: *addr,
}
if !d.Deadline.IsZero() {
var cancel context.CancelFunc
ctx, cancel = context.WithDeadline(ctx, d.Deadline)
defer cancel()
}
// preemptive timeout/cancellation check
if err = ctx.Err(); err != nil {
return nil, conn.opErr(op, err)
}
sock, err := newHVSocket()
if err != nil {
return nil, err
return nil, conn.opErr(op, err)
}
defer func() {
if sock != nil {
sock.Close()
}
}()
c, err := sock.prepareIo()
sa := addr.raw()
err = socket.Bind(windows.Handle(sock.handle), &sa)
if err != nil {
return nil, err
return nil, conn.opErr(op, os.NewSyscallError("bind", err))
}
c, err := sock.prepareIO()
if err != nil {
return nil, conn.opErr(op, err)
}
defer sock.wg.Done()
var bytes uint32
err = windows.ConnectEx(windows.Handle(sock.handle), sa, nil, 0, &bytes, &c.o)
_, err = sock.asyncIo(ctx, c, nil, bytes, err)
for i := uint(0); i <= d.Retries; i++ {
err = socket.ConnectEx(
windows.Handle(sock.handle),
&sa,
nil, // sendBuf
0, // sendDataLen
&bytes,
(*windows.Overlapped)(unsafe.Pointer(&c.o)))
_, err = sock.asyncIO(c, nil, bytes, err)
if i < d.Retries && canRedial(err) {
if err = d.redialWait(ctx); err == nil {
continue
}
}
break
}
if err != nil {
return nil, err
return nil, conn.opErr(op, os.NewSyscallError("connectex", err))
}
conn := &HvsockConn{
sock: sock,
remote: *addr,
// update the connection properties, so shutdown can be used
if err = windows.Setsockopt(
windows.Handle(sock.handle),
windows.SOL_SOCKET,
windows.SO_UPDATE_CONNECT_CONTEXT,
nil, // optvalue
0, // optlen
); err != nil {
return nil, conn.opErr(op, os.NewSyscallError("setsockopt", err))
}
// get the local name
var sal rawHvsockAddr
err = socket.GetSockName(windows.Handle(sock.handle), &sal)
if err != nil {
return nil, conn.opErr(op, os.NewSyscallError("getsockname", err))
}
conn.local.fromRaw(&sal)
// one last check for timeout, since asyncIO doesn't check the context
if err = ctx.Err(); err != nil {
return nil, conn.opErr(op, err)
}
conn.sock = sock
sock = nil
return conn, nil
}
*/
// redialWait waits before attempting to redial, resetting the timer as appropriate.
func (d *HvsockDialer) redialWait(ctx context.Context) (err error) {
if d.RetryWait == 0 {
return nil
}
if d.rt == nil {
d.rt = time.NewTimer(d.RetryWait)
} else {
// should already be stopped and drained
d.rt.Reset(d.RetryWait)
}
select {
case <-ctx.Done():
case <-d.rt.C:
return nil
}
// stop and drain the timer
if !d.rt.Stop() {
<-d.rt.C
}
return ctx.Err()
}
// assumes error is a plain, unwrapped syscall.Errno provided by direct syscall.
func canRedial(err error) bool {
//nolint:errorlint // guaranteed to be an Errno
switch err {
case windows.WSAECONNREFUSED, windows.WSAENETUNREACH, windows.WSAETIMEDOUT,
windows.ERROR_CONNECTION_REFUSED, windows.ERROR_CONNECTION_UNAVAIL:
return true
default:
return false
}
}
func (conn *HvsockConn) opErr(op string, err error) error {
// translate from "file closed" to "socket closed"
if errors.Is(err, ErrFileClosed) {
err = socket.ErrSocketClosed
}
return &net.OpError{Op: op, Net: "hvsock", Source: &conn.local, Addr: &conn.remote, Err: err}
}
func (conn *HvsockConn) Read(b []byte) (int, error) {
c, err := conn.sock.prepareIo()
c, err := conn.sock.prepareIO()
if err != nil {
return 0, conn.opErr("read", err)
}
@@ -204,10 +450,11 @@ func (conn *HvsockConn) Read(b []byte) (int, error) {
buf := syscall.WSABuf{Buf: &b[0], Len: uint32(len(b))}
var flags, bytes uint32
err = syscall.WSARecv(conn.sock.handle, &buf, 1, &bytes, &flags, &c.o, nil)
n, err := conn.sock.asyncIo(c, &conn.sock.readDeadline, bytes, err)
n, err := conn.sock.asyncIO(c, &conn.sock.readDeadline, bytes, err)
if err != nil {
if _, ok := err.(syscall.Errno); ok {
err = os.NewSyscallError("wsarecv", err)
var eno windows.Errno
if errors.As(err, &eno) {
err = os.NewSyscallError("wsarecv", eno)
}
return 0, conn.opErr("read", err)
} else if n == 0 {
@@ -230,7 +477,7 @@ func (conn *HvsockConn) Write(b []byte) (int, error) {
}
func (conn *HvsockConn) write(b []byte) (int, error) {
c, err := conn.sock.prepareIo()
c, err := conn.sock.prepareIO()
if err != nil {
return 0, conn.opErr("write", err)
}
@@ -238,10 +485,11 @@ func (conn *HvsockConn) write(b []byte) (int, error) {
buf := syscall.WSABuf{Buf: &b[0], Len: uint32(len(b))}
var bytes uint32
err = syscall.WSASend(conn.sock.handle, &buf, 1, &bytes, 0, &c.o, nil)
n, err := conn.sock.asyncIo(c, &conn.sock.writeDeadline, bytes, err)
n, err := conn.sock.asyncIO(c, &conn.sock.writeDeadline, bytes, err)
if err != nil {
if _, ok := err.(syscall.Errno); ok {
err = os.NewSyscallError("wsasend", err)
var eno windows.Errno
if errors.As(err, &eno) {
err = os.NewSyscallError("wsasend", eno)
}
return 0, conn.opErr("write", err)
}
@@ -257,13 +505,19 @@ func (conn *HvsockConn) IsClosed() bool {
return conn.sock.IsClosed()
}
// shutdown disables sending or receiving on a socket.
func (conn *HvsockConn) shutdown(how int) error {
if conn.IsClosed() {
return ErrFileClosed
return socket.ErrSocketClosed
}
err := syscall.Shutdown(conn.sock.handle, how)
if err != nil {
// If the connection was closed, shutdowns fail with "not connected"
if errors.Is(err, windows.WSAENOTCONN) ||
errors.Is(err, windows.WSAESHUTDOWN) {
err = socket.ErrSocketClosed
}
return os.NewSyscallError("shutdown", err)
}
return nil
@@ -273,7 +527,7 @@ func (conn *HvsockConn) shutdown(how int) error {
func (conn *HvsockConn) CloseRead() error {
err := conn.shutdown(syscall.SHUT_RD)
if err != nil {
return conn.opErr("close", err)
return conn.opErr("closeread", err)
}
return nil
}
@@ -283,7 +537,7 @@ func (conn *HvsockConn) CloseRead() error {
func (conn *HvsockConn) CloseWrite() error {
err := conn.shutdown(syscall.SHUT_WR)
if err != nil {
return conn.opErr("close", err)
return conn.opErr("closewrite", err)
}
return nil
}
@@ -300,8 +554,13 @@ func (conn *HvsockConn) RemoteAddr() net.Addr {
// SetDeadline implements the net.Conn SetDeadline method.
func (conn *HvsockConn) SetDeadline(t time.Time) error {
conn.SetReadDeadline(t)
conn.SetWriteDeadline(t)
// todo: implement `SetDeadline` for `win32File`
if err := conn.SetReadDeadline(t); err != nil {
return fmt.Errorf("set read deadline: %w", err)
}
if err := conn.SetWriteDeadline(t); err != nil {
return fmt.Errorf("set write deadline: %w", err)
}
return nil
}
vendor/github.com/Microsoft/go-winio/internal/socket/rawaddr.go
Generated Vendored
+20
View File
@@ -0,0 +1,20 @@
package socket
import (
"unsafe"
)
// RawSockaddr allows structs to be used with [Bind] and [ConnectEx]. The
// struct must meet the Win32 sockaddr requirements specified here:
// https://docs.microsoft.com/en-us/windows/win32/winsock/sockaddr-2
//
// Specifically, the struct size must be least larger than an int16 (unsigned short)
// for the address family.
type RawSockaddr interface {
// Sockaddr returns a pointer to the RawSockaddr and its struct size, allowing
// for the RawSockaddr's data to be overwritten by syscalls (if necessary).
//
// It is the callers responsibility to validate that the values are valid; invalid
// pointers or size can cause a panic.
Sockaddr() (unsafe.Pointer, int32, error)
}
vendor/github.com/Microsoft/go-winio/internal/socket/socket.go
Generated Vendored
+179
View File
@@ -0,0 +1,179 @@
//go:build windows
package socket
import (
"errors"
"fmt"
"net"
"sync"
"syscall"
"unsafe"
"github.com/Microsoft/go-winio/pkg/guid"
"golang.org/x/sys/windows"
)
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go socket.go
//sys getsockname(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) [failretval==socketError] = ws2_32.getsockname
//sys getpeername(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) [failretval==socketError] = ws2_32.getpeername
//sys bind(s windows.Handle, name unsafe.Pointer, namelen int32) (err error) [failretval==socketError] = ws2_32.bind
const socketError = uintptr(^uint32(0))
var (
// todo(helsaawy): create custom error types to store the desired vs actual size and addr family?
ErrBufferSize = errors.New("buffer size")
ErrAddrFamily = errors.New("address family")
ErrInvalidPointer = errors.New("invalid pointer")
ErrSocketClosed = fmt.Errorf("socket closed: %w", net.ErrClosed)
)
// todo(helsaawy): replace these with generics, ie: GetSockName[S RawSockaddr](s windows.Handle) (S, error)
// GetSockName writes the local address of socket s to the [RawSockaddr] rsa.
// If rsa is not large enough, the [windows.WSAEFAULT] is returned.
func GetSockName(s windows.Handle, rsa RawSockaddr) error {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
// although getsockname returns WSAEFAULT if the buffer is too small, it does not set
// &l to the correct size, so--apart from doubling the buffer repeatedly--there is no remedy
return getsockname(s, ptr, &l)
}
// GetPeerName returns the remote address the socket is connected to.
//
// See [GetSockName] for more information.
func GetPeerName(s windows.Handle, rsa RawSockaddr) error {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
return getpeername(s, ptr, &l)
}
func Bind(s windows.Handle, rsa RawSockaddr) (err error) {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
return bind(s, ptr, l)
}
// "golang.org/x/sys/windows".ConnectEx and .Bind only accept internal implementations of the
// their sockaddr interface, so they cannot be used with HvsockAddr
// Replicate functionality here from
// https://cs.opensource.google/go/x/sys/+/master:windows/syscall_windows.go
// The function pointers to `AcceptEx`, `ConnectEx` and `GetAcceptExSockaddrs` must be loaded at
// runtime via a WSAIoctl call:
// https://docs.microsoft.com/en-us/windows/win32/api/Mswsock/nc-mswsock-lpfn_connectex#remarks
type runtimeFunc struct {
id guid.GUID
once sync.Once
addr uintptr
err error
}
func (f *runtimeFunc) Load() error {
f.once.Do(func() {
var s windows.Handle
s, f.err = windows.Socket(windows.AF_INET, windows.SOCK_STREAM, windows.IPPROTO_TCP)
if f.err != nil {
return
}
defer windows.CloseHandle(s) //nolint:errcheck
var n uint32
f.err = windows.WSAIoctl(s,
windows.SIO_GET_EXTENSION_FUNCTION_POINTER,
(*byte)(unsafe.Pointer(&f.id)),
uint32(unsafe.Sizeof(f.id)),
(*byte)(unsafe.Pointer(&f.addr)),
uint32(unsafe.Sizeof(f.addr)),
&n,
nil, //overlapped
0, //completionRoutine
)
})
return f.err
}
var (
// todo: add `AcceptEx` and `GetAcceptExSockaddrs`
WSAID_CONNECTEX = guid.GUID{ //revive:disable-line:var-naming ALL_CAPS
Data1: 0x25a207b9,
Data2: 0xddf3,
Data3: 0x4660,
Data4: [8]byte{0x8e, 0xe9, 0x76, 0xe5, 0x8c, 0x74, 0x06, 0x3e},
}
connectExFunc = runtimeFunc{id: WSAID_CONNECTEX}
)
func ConnectEx(
fd windows.Handle,
rsa RawSockaddr,
sendBuf *byte,
sendDataLen uint32,
bytesSent *uint32,
overlapped *windows.Overlapped,
) error {
if err := connectExFunc.Load(); err != nil {
return fmt.Errorf("failed to load ConnectEx function pointer: %w", err)
}
ptr, n, err := rsa.Sockaddr()
if err != nil {
return err
}
return connectEx(fd, ptr, n, sendBuf, sendDataLen, bytesSent, overlapped)
}
// BOOL LpfnConnectex(
// [in] SOCKET s,
// [in] const sockaddr *name,
// [in] int namelen,
// [in, optional] PVOID lpSendBuffer,
// [in] DWORD dwSendDataLength,
// [out] LPDWORD lpdwBytesSent,
// [in] LPOVERLAPPED lpOverlapped
// )
func connectEx(
s windows.Handle,
name unsafe.Pointer,
namelen int32,
sendBuf *byte,
sendDataLen uint32,
bytesSent *uint32,
overlapped *windows.Overlapped,
) (err error) {
// todo: after upgrading to 1.18, switch from syscall.Syscall9 to syscall.SyscallN
r1, _, e1 := syscall.Syscall9(connectExFunc.addr,
7,
uintptr(s),
uintptr(name),
uintptr(namelen),
uintptr(unsafe.Pointer(sendBuf)),
uintptr(sendDataLen),
uintptr(unsafe.Pointer(bytesSent)),
uintptr(unsafe.Pointer(overlapped)),
0,
0)
if r1 == 0 {
if e1 != 0 {
err = error(e1)
} else {
err = syscall.EINVAL
}
}
return err
}
vendor/github.com/Microsoft/go-winio/internal/socket/zsyscall_windows.go
Generated Vendored
+72
View File
@@ -0,0 +1,72 @@
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package socket
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var _ unsafe.Pointer
// Do the interface allocations only once for common
// Errno values.
const (
errnoERROR_IO_PENDING = 997
)
var (
errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
errERROR_EINVAL error = syscall.EINVAL
)
// errnoErr returns common boxed Errno values, to prevent
// allocations at runtime.
func errnoErr(e syscall.Errno) error {
switch e {
case 0:
return errERROR_EINVAL
case errnoERROR_IO_PENDING:
return errERROR_IO_PENDING
}
// TODO: add more here, after collecting data on the common
// error values see on Windows. (perhaps when running
// all.bat?)
return e
}
var (
modws2_32 = windows.NewLazySystemDLL("ws2_32.dll")
procbind = modws2_32.NewProc("bind")
procgetpeername = modws2_32.NewProc("getpeername")
procgetsockname = modws2_32.NewProc("getsockname")
)
func bind(s windows.Handle, name unsafe.Pointer, namelen int32) (err error) {
r1, _, e1 := syscall.Syscall(procbind.Addr(), 3, uintptr(s), uintptr(name), uintptr(namelen))
if r1 == socketError {
err = errnoErr(e1)
}
return
}
func getpeername(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) {
r1, _, e1 := syscall.Syscall(procgetpeername.Addr(), 3, uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen)))
if r1 == socketError {
err = errnoErr(e1)
}
return
}
func getsockname(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) {
r1, _, e1 := syscall.Syscall(procgetsockname.Addr(), 3, uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen)))
if r1 == socketError {
err = errnoErr(e1)
}
return
}
vendor/github.com/Microsoft/go-winio/pipe.go
Generated Vendored
+64 -60
View File
@@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@@ -13,6 +14,8 @@ import (
"syscall"
"time"
"unsafe"
"golang.org/x/sys/windows"
)
//sys connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) = ConnectNamedPipe
@@ -21,10 +24,10 @@ import (
//sys getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo
//sys getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW
//sys localAlloc(uFlags uint32, length uint32) (ptr uintptr) = LocalAlloc
//sys ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntstatus) = ntdll.NtCreateNamedPipeFile
//sys rtlNtStatusToDosError(status ntstatus) (winerr error) = ntdll.RtlNtStatusToDosErrorNoTeb
//sys rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntstatus) = ntdll.RtlDosPathNameToNtPathName_U
//sys rtlDefaultNpAcl(dacl *uintptr) (status ntstatus) = ntdll.RtlDefaultNpAcl
//sys ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) = ntdll.NtCreateNamedPipeFile
//sys rtlNtStatusToDosError(status ntStatus) (winerr error) = ntdll.RtlNtStatusToDosErrorNoTeb
//sys rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntStatus) = ntdll.RtlDosPathNameToNtPathName_U
//sys rtlDefaultNpAcl(dacl *uintptr) (status ntStatus) = ntdll.RtlDefaultNpAcl
type ioStatusBlock struct {
Status, Information uintptr
@@ -51,45 +54,22 @@ type securityDescriptor struct {
Control uint16
Owner uintptr
Group uintptr
Sacl uintptr
Dacl uintptr
Sacl uintptr //revive:disable-line:var-naming SACL, not Sacl
Dacl uintptr //revive:disable-line:var-naming DACL, not Dacl
}
type ntstatus int32
type ntStatus int32
func (status ntstatus) Err() error {
func (status ntStatus) Err() error {
if status >= 0 {
return nil
}
return rtlNtStatusToDosError(status)
}
const (
cERROR_PIPE_BUSY = syscall.Errno(231)
cERROR_NO_DATA = syscall.Errno(232)
cERROR_PIPE_CONNECTED = syscall.Errno(535)
cERROR_SEM_TIMEOUT = syscall.Errno(121)
cSECURITY_SQOS_PRESENT = 0x100000
cSECURITY_ANONYMOUS = 0
cPIPE_TYPE_MESSAGE = 4
cPIPE_READMODE_MESSAGE = 2
cFILE_OPEN = 1
cFILE_CREATE = 2
cFILE_PIPE_MESSAGE_TYPE = 1
cFILE_PIPE_REJECT_REMOTE_CLIENTS = 2
cSE_DACL_PRESENT = 4
)
var (
// ErrPipeListenerClosed is returned for pipe operations on listeners that have been closed.
// This error should match net.errClosing since docker takes a dependency on its text.
ErrPipeListenerClosed = errors.New("use of closed network connection")
ErrPipeListenerClosed = net.ErrClosed
errPipeWriteClosed = errors.New("pipe has been closed for write")
)
@@ -116,9 +96,10 @@ func (f *win32Pipe) RemoteAddr() net.Addr {
}
func (f *win32Pipe) SetDeadline(t time.Time) error {
f.SetReadDeadline(t)
f.SetWriteDeadline(t)
return nil
if err := f.SetReadDeadline(t); err != nil {
return err
}
return f.SetWriteDeadline(t)
}
// CloseWrite closes the write side of a message pipe in byte mode.
@@ -157,14 +138,14 @@ func (f *win32MessageBytePipe) Read(b []byte) (int, error) {
return 0, io.EOF
}
n, err := f.win32File.Read(b)
if err == io.EOF {
if err == io.EOF { //nolint:errorlint
// If this was the result of a zero-byte read, then
// it is possible that the read was due to a zero-size
// message. Since we are simulating CloseWrite with a
// zero-byte message, ensure that all future Read() calls
// also return EOF.
f.readEOF = true
} else if err == syscall.ERROR_MORE_DATA {
} else if err == syscall.ERROR_MORE_DATA { //nolint:errorlint // err is Errno
// ERROR_MORE_DATA indicates that the pipe's read mode is message mode
// and the message still has more bytes. Treat this as a success, since
// this package presents all named pipes as byte streams.
@@ -173,7 +154,7 @@ func (f *win32MessageBytePipe) Read(b []byte) (int, error) {
return n, err
}
func (s pipeAddress) Network() string {
func (pipeAddress) Network() string {
return "pipe"
}
@@ -184,16 +165,21 @@ func (s pipeAddress) String() string {
// tryDialPipe attempts to dial the pipe at `path` until `ctx` cancellation or timeout.
func tryDialPipe(ctx context.Context, path *string, access uint32) (syscall.Handle, error) {
for {
select {
case <-ctx.Done():
return syscall.Handle(0), ctx.Err()
default:
h, err := createFile(*path, access, 0, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_OVERLAPPED|cSECURITY_SQOS_PRESENT|cSECURITY_ANONYMOUS, 0)
h, err := createFile(*path,
access,
0,
nil,
syscall.OPEN_EXISTING,
windows.FILE_FLAG_OVERLAPPED|windows.SECURITY_SQOS_PRESENT|windows.SECURITY_ANONYMOUS,
0)
if err == nil {
return h, nil
}
if err != cERROR_PIPE_BUSY {
if err != windows.ERROR_PIPE_BUSY { //nolint:errorlint // err is Errno
return h, &os.PathError{Err: err, Op: "open", Path: *path}
}
// Wait 10 msec and try again. This is a rather simplistic
@@ -213,9 +199,10 @@ func DialPipe(path string, timeout *time.Duration) (net.Conn, error) {
} else {
absTimeout = time.Now().Add(2 * time.Second)
}
ctx, _ := context.WithDeadline(context.Background(), absTimeout)
ctx, cancel := context.WithDeadline(context.Background(), absTimeout)
defer cancel()
conn, err := DialPipeContext(ctx, path)
if err == context.DeadlineExceeded {
if errors.Is(err, context.DeadlineExceeded) {
return nil, ErrTimeout
}
return conn, err
@@ -251,7 +238,7 @@ func DialPipeAccess(ctx context.Context, path string, access uint32) (net.Conn,
// If the pipe is in message mode, return a message byte pipe, which
// supports CloseWrite().
if flags&cPIPE_TYPE_MESSAGE != 0 {
if flags&windows.PIPE_TYPE_MESSAGE != 0 {
return &win32MessageBytePipe{
win32Pipe: win32Pipe{win32File: f, path: path},
}, nil
@@ -283,7 +270,11 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
oa.Length = unsafe.Sizeof(oa)
var ntPath unicodeString
if err := rtlDosPathNameToNtPathName(&path16[0], &ntPath, 0, 0).Err(); err != nil {
if err := rtlDosPathNameToNtPathName(&path16[0],
&ntPath,
0,
0,
).Err(); err != nil {
return 0, &os.PathError{Op: "open", Path: path, Err: err}
}
defer localFree(ntPath.Buffer)
@@ -292,8 +283,8 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
// The security descriptor is only needed for the first pipe.
if first {
if sd != nil {
len := uint32(len(sd))
sdb := localAlloc(0, len)
l := uint32(len(sd))
sdb := localAlloc(0, l)
defer localFree(sdb)
copy((*[0xffff]byte)(unsafe.Pointer(sdb))[:], sd)
oa.SecurityDescriptor = (*securityDescriptor)(unsafe.Pointer(sdb))
@@ -301,28 +292,28 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
// Construct the default named pipe security descriptor.
var dacl uintptr
if err := rtlDefaultNpAcl(&dacl).Err(); err != nil {
return 0, fmt.Errorf("getting default named pipe ACL: %s", err)
return 0, fmt.Errorf("getting default named pipe ACL: %w", err)
}
defer localFree(dacl)
sdb := &securityDescriptor{
Revision: 1,
Control: cSE_DACL_PRESENT,
Control: windows.SE_DACL_PRESENT,
Dacl: dacl,
}
oa.SecurityDescriptor = sdb
}
}
typ := uint32(cFILE_PIPE_REJECT_REMOTE_CLIENTS)
typ := uint32(windows.FILE_PIPE_REJECT_REMOTE_CLIENTS)
if c.MessageMode {
typ |= cFILE_PIPE_MESSAGE_TYPE
typ |= windows.FILE_PIPE_MESSAGE_TYPE
}
disposition := uint32(cFILE_OPEN)
disposition := uint32(windows.FILE_OPEN)
access := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE | syscall.SYNCHRONIZE)
if first {
disposition = cFILE_CREATE
disposition = windows.FILE_CREATE
// By not asking for read or write access, the named pipe file system
// will put this pipe into an initially disconnected state, blocking
// client connections until the next call with first == false.
@@ -335,7 +326,20 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
h syscall.Handle
iosb ioStatusBlock
)
err = ntCreateNamedPipeFile(&h, access, &oa, &iosb, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, disposition, 0, typ, 0, 0, 0xffffffff, uint32(c.InputBufferSize), uint32(c.OutputBufferSize), &timeout).Err()
err = ntCreateNamedPipeFile(&h,
access,
&oa,
&iosb,
syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE,
disposition,
0,
typ,
0,
0,
0xffffffff,
uint32(c.InputBufferSize),
uint32(c.OutputBufferSize),
&timeout).Err()
if err != nil {
return 0, &os.PathError{Op: "open", Path: path, Err: err}
}
@@ -380,7 +384,7 @@ func (l *win32PipeListener) makeConnectedServerPipe() (*win32File, error) {
p.Close()
p = nil
err = <-ch
if err == nil || err == ErrFileClosed {
if err == nil || err == ErrFileClosed { //nolint:errorlint // err is Errno
err = ErrPipeListenerClosed
}
}
@@ -402,12 +406,12 @@ func (l *win32PipeListener) listenerRoutine() {
p, err = l.makeConnectedServerPipe()
// If the connection was immediately closed by the client, try
// again.
if err != cERROR_NO_DATA {
if err != windows.ERROR_NO_DATA { //nolint:errorlint // err is Errno
break
}
}
responseCh <- acceptResponse{p, err}
closed = err == ErrPipeListenerClosed
closed = err == ErrPipeListenerClosed //nolint:errorlint // err is Errno
}
}
syscall.Close(l.firstHandle)
@@ -469,15 +473,15 @@ func ListenPipe(path string, c *PipeConfig) (net.Listener, error) {
}
func connectPipe(p *win32File) error {
c, err := p.prepareIo()
c, err := p.prepareIO()
if err != nil {
return err
}
defer p.wg.Done()
err = connectNamedPipe(p.handle, &c.o)
_, err = p.asyncIo(c, nil, 0, err)
if err != nil && err != cERROR_PIPE_CONNECTED {
_, err = p.asyncIO(c, nil, 0, err)
if err != nil && err != windows.ERROR_PIPE_CONNECTED { //nolint:errorlint // err is Errno
return err
}
return nil
vendor/github.com/Microsoft/go-winio/pkg/guid/guid.go
Generated Vendored
+10 -6
View File
@@ -1,5 +1,3 @@
// +build windows
// Package guid provides a GUID type. The backing structure for a GUID is
// identical to that used by the golang.org/x/sys/windows GUID type.
// There are two main binary encodings used for a GUID, the big-endian encoding,
@@ -9,24 +7,26 @@ package guid
import (
"crypto/rand"
"crypto/sha1"
"crypto/sha1" //nolint:gosec // not used for secure application
"encoding"
"encoding/binary"
"fmt"
"strconv"
)
//go:generate go run golang.org/x/tools/cmd/stringer -type=Variant -trimprefix=Variant -linecomment
// Variant specifies which GUID variant (or "type") of the GUID. It determines
// how the entirety of the rest of the GUID is interpreted.
type Variant uint8
// The variants specified by RFC 4122.
// The variants specified by RFC 4122 section 4.1.1.
const (
// VariantUnknown specifies a GUID variant which does not conform to one of
// the variant encodings specified in RFC 4122.
VariantUnknown Variant = iota
VariantNCS
VariantRFC4122
VariantRFC4122 // RFC 4122
VariantMicrosoft
VariantFuture
)
@@ -36,6 +36,10 @@ const (
// hash of an input string.
type Version uint8
func (v Version) String() string {
return strconv.FormatUint(uint64(v), 10)
}
var _ = (encoding.TextMarshaler)(GUID{})
var _ = (encoding.TextUnmarshaler)(&GUID{})
@@ -61,7 +65,7 @@ func NewV4() (GUID, error) {
// big-endian UTF16 stream of bytes. If that is desired, the string can be
// encoded as such before being passed to this function.
func NewV5(namespace GUID, name []byte) (GUID, error) {
b := sha1.New()
b := sha1.New() //nolint:gosec // not used for secure application
namespaceBytes := namespace.ToArray()
b.Write(namespaceBytes[:])
b.Write(name)
vendor/github.com/Microsoft/go-winio/pkg/guid/guid_nonwindows.go
Generated Vendored
+1
View File
@@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package guid
vendor/github.com/Microsoft/go-winio/pkg/guid/guid_windows.go
Generated Vendored
+3
View File
@@ -1,3 +1,6 @@
//go:build windows
// +build windows
package guid
import "golang.org/x/sys/windows"
vendor/github.com/Microsoft/go-winio/pkg/guid/variant_string.go
Generated Vendored
+27
View File
@@ -0,0 +1,27 @@
// Code generated by "stringer -type=Variant -trimprefix=Variant -linecomment"; DO NOT EDIT.
package guid
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[VariantUnknown-0]
_ = x[VariantNCS-1]
_ = x[VariantRFC4122-2]
_ = x[VariantMicrosoft-3]
_ = x[VariantFuture-4]
}
const _Variant_name = "UnknownNCSRFC 4122MicrosoftFuture"
var _Variant_index = [...]uint8{0, 7, 10, 18, 27, 33}
func (i Variant) String() string {
if i >= Variant(len(_Variant_index)-1) {
return "Variant(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _Variant_name[_Variant_index[i]:_Variant_index[i+1]]
}
vendor/github.com/Microsoft/go-winio/pkg/security/grantvmgroupaccess.go
Generated Vendored
+21 -12
View File
@@ -1,3 +1,4 @@
//go:build windows
// +build windows
package security
@@ -20,6 +21,7 @@ type (
trusteeForm uint32
trusteeType uint32
//nolint:structcheck // structcheck thinks fields are unused, but the are used to pass data to OS
explicitAccess struct {
accessPermissions accessMask
accessMode accessMode
@@ -27,6 +29,7 @@ type (
trustee trustee
}
//nolint:structcheck,unused // structcheck thinks fields are unused, but the are used to pass data to OS
trustee struct {
multipleTrustee *trustee
multipleTrusteeOperation int32
@@ -44,6 +47,7 @@ const (
desiredAccessReadControl desiredAccess = 0x20000
desiredAccessWriteDac desiredAccess = 0x40000
//cspell:disable-next-line
gvmga = "GrantVmGroupAccess:"
inheritModeNoInheritance inheritMode = 0x0
@@ -56,9 +60,9 @@ const (
shareModeRead shareMode = 0x1
shareModeWrite shareMode = 0x2
sidVmGroup = "S-1-5-83-0"
sidVMGroup = "S-1-5-83-0"
trusteeFormIsSid trusteeForm = 0
trusteeFormIsSID trusteeForm = 0
trusteeTypeWellKnownGroup trusteeType = 5
)
@@ -67,6 +71,8 @@ const (
// include Grant ACE entries for the VM Group SID. This is a golang re-
// implementation of the same function in vmcompute, just not exported in
// RS5. Which kind of sucks. Sucks a lot :/
//
//revive:disable-next-line:var-naming VM, not Vm
func GrantVmGroupAccess(name string) error {
// Stat (to determine if `name` is a directory).
s, err := os.Stat(name)
@@ -79,7 +85,7 @@ func GrantVmGroupAccess(name string) error {
if err != nil {
return err // Already wrapped
}
defer syscall.CloseHandle(fd)
defer syscall.CloseHandle(fd) //nolint:errcheck
// Get the current DACL and Security Descriptor. Must defer LocalFree on success.
ot := objectTypeFileObject
@@ -89,7 +95,7 @@ func GrantVmGroupAccess(name string) error {
if err := getSecurityInfo(fd, uint32(ot), uint32(si), nil, nil, &origDACL, nil, &sd); err != nil {
return fmt.Errorf("%s GetSecurityInfo %s: %w", gvmga, name, err)
}
defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(sd)))
defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(sd))) //nolint:errcheck
// Generate a new DACL which is the current DACL with the required ACEs added.
// Must defer LocalFree on success.
@@ -97,7 +103,7 @@ func GrantVmGroupAccess(name string) error {
if err != nil {
return err // Already wrapped
}
defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(newDACL)))
defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(newDACL))) //nolint:errcheck
// And finally use SetSecurityInfo to apply the updated DACL.
if err := setSecurityInfo(fd, uint32(ot), uint32(si), uintptr(0), uintptr(0), newDACL, uintptr(0)); err != nil {
@@ -110,16 +116,19 @@ func GrantVmGroupAccess(name string) error {
// createFile is a helper function to call [Nt]CreateFile to get a handle to
// the file or directory.
func createFile(name string, isDir bool) (syscall.Handle, error) {
namep := syscall.StringToUTF16(name)
namep, err := syscall.UTF16FromString(name)
if err != nil {
return syscall.InvalidHandle, fmt.Errorf("could not convernt name to UTF-16: %w", err)
}
da := uint32(desiredAccessReadControl | desiredAccessWriteDac)
sm := uint32(shareModeRead | shareModeWrite)
fa := uint32(syscall.FILE_ATTRIBUTE_NORMAL)
if isDir {
fa = uint32(fa | syscall.FILE_FLAG_BACKUP_SEMANTICS)
fa |= syscall.FILE_FLAG_BACKUP_SEMANTICS
}
fd, err := syscall.CreateFile(&namep[0], da, sm, nil, syscall.OPEN_EXISTING, fa, 0)
if err != nil {
return 0, fmt.Errorf("%s syscall.CreateFile %s: %w", gvmga, name, err)
return syscall.InvalidHandle, fmt.Errorf("%s syscall.CreateFile %s: %w", gvmga, name, err)
}
return fd, nil
}
@@ -128,9 +137,9 @@ func createFile(name string, isDir bool) (syscall.Handle, error) {
// The caller is responsible for LocalFree of the returned DACL on success.
func generateDACLWithAcesAdded(name string, isDir bool, origDACL uintptr) (uintptr, error) {
// Generate pointers to the SIDs based on the string SIDs
sid, err := syscall.StringToSid(sidVmGroup)
sid, err := syscall.StringToSid(sidVMGroup)
if err != nil {
return 0, fmt.Errorf("%s syscall.StringToSid %s %s: %w", gvmga, name, sidVmGroup, err)
return 0, fmt.Errorf("%s syscall.StringToSid %s %s: %w", gvmga, name, sidVMGroup, err)
}
inheritance := inheritModeNoInheritance
@@ -139,12 +148,12 @@ func generateDACLWithAcesAdded(name string, isDir bool, origDACL uintptr) (uintp
}
eaArray := []explicitAccess{
explicitAccess{
{
accessPermissions: accessMaskDesiredPermission,
accessMode: accessModeGrant,
inheritance: inheritance,
trustee: trustee{
trusteeForm: trusteeFormIsSid,
trusteeForm: trusteeFormIsSID,
trusteeType: trusteeTypeWellKnownGroup,
name: uintptr(unsafe.Pointer(sid)),
},
vendor/github.com/Microsoft/go-winio/pkg/security/syscall_windows.go
Generated Vendored
+1 -1
View File
@@ -1,6 +1,6 @@
package security
//go:generate go run mksyscall_windows.go -output zsyscall_windows.go syscall_windows.go
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go syscall_windows.go
//sys getSecurityInfo(handle syscall.Handle, objectType uint32, si uint32, ppsidOwner **uintptr, ppsidGroup **uintptr, ppDacl *uintptr, ppSacl *uintptr, ppSecurityDescriptor *uintptr) (win32err error) = advapi32.GetSecurityInfo
//sys setSecurityInfo(handle syscall.Handle, objectType uint32, si uint32, psidOwner uintptr, psidGroup uintptr, pDacl uintptr, pSacl uintptr) (win32err error) = advapi32.SetSecurityInfo
vendor/github.com/Microsoft/go-winio/pkg/security/zsyscall_windows.go
Generated Vendored
+3 -1
View File
@@ -1,4 +1,6 @@
// Code generated by 'go generate'; DO NOT EDIT.
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package security
vendor/github.com/Microsoft/go-winio/privilege.go
Generated Vendored
+13 -19
View File
@@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@@ -24,22 +25,17 @@ import (
//sys lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) = advapi32.LookupPrivilegeDisplayNameW
const (
SE_PRIVILEGE_ENABLED = 2
//revive:disable-next-line:var-naming ALL_CAPS
SE_PRIVILEGE_ENABLED = windows.SE_PRIVILEGE_ENABLED
ERROR_NOT_ALL_ASSIGNED syscall.Errno = 1300
//revive:disable-next-line:var-naming ALL_CAPS
ERROR_NOT_ALL_ASSIGNED syscall.Errno = windows.ERROR_NOT_ALL_ASSIGNED
SeBackupPrivilege = "SeBackupPrivilege"
SeRestorePrivilege = "SeRestorePrivilege"
SeSecurityPrivilege = "SeSecurityPrivilege"
)
const (
securityAnonymous = iota
securityIdentification
securityImpersonation
securityDelegation
)
var (
privNames = make(map[string]uint64)
privNameMutex sync.Mutex
@@ -51,11 +47,9 @@ type PrivilegeError struct {
}
func (e *PrivilegeError) Error() string {
s := ""
s := "Could not enable privilege "
if len(e.privileges) > 1 {
s = "Could not enable privileges "
} else {
s = "Could not enable privilege "
}
for i, p := range e.privileges {
if i != 0 {
@@ -94,7 +88,7 @@ func RunWithPrivileges(names []string, fn func() error) error {
}
func mapPrivileges(names []string) ([]uint64, error) {
var privileges []uint64
privileges := make([]uint64, 0, len(names))
privNameMutex.Lock()
defer privNameMutex.Unlock()
for _, name := range names {
@@ -127,7 +121,7 @@ func enableDisableProcessPrivilege(names []string, action uint32) error {
return err
}
p, _ := windows.GetCurrentProcess()
p := windows.CurrentProcess()
var token windows.Token
err = windows.OpenProcessToken(p, windows.TOKEN_ADJUST_PRIVILEGES|windows.TOKEN_QUERY, &token)
if err != nil {
@@ -140,10 +134,10 @@ func enableDisableProcessPrivilege(names []string, action uint32) error {
func adjustPrivileges(token windows.Token, privileges []uint64, action uint32) error {
var b bytes.Buffer
binary.Write(&b, binary.LittleEndian, uint32(len(privileges)))
_ = binary.Write(&b, binary.LittleEndian, uint32(len(privileges)))
for _, p := range privileges {
binary.Write(&b, binary.LittleEndian, p)
binary.Write(&b, binary.LittleEndian, action)
_ = binary.Write(&b, binary.LittleEndian, p)
_ = binary.Write(&b, binary.LittleEndian, action)
}
prevState := make([]byte, b.Len())
reqSize := uint32(0)
@@ -151,7 +145,7 @@ func adjustPrivileges(token windows.Token, privileges []uint64, action uint32) e
if !success {
return err
}
if err == ERROR_NOT_ALL_ASSIGNED {
if err == ERROR_NOT_ALL_ASSIGNED { //nolint:errorlint // err is Errno
return &PrivilegeError{privileges}
}
return nil
@@ -177,7 +171,7 @@ func getPrivilegeName(luid uint64) string {
}
func newThreadToken() (windows.Token, error) {
err := impersonateSelf(securityImpersonation)
err := impersonateSelf(windows.SecurityImpersonation)
if err != nil {
return 0, err
}
vendor/github.com/Microsoft/go-winio/reparse.go
Generated Vendored
+7 -4
View File
@@ -1,3 +1,6 @@
//go:build windows
// +build windows
package winio
import (
@@ -113,16 +116,16 @@ func EncodeReparsePoint(rp *ReparsePoint) []byte {
}
var b bytes.Buffer
binary.Write(&b, binary.LittleEndian, &data)
_ = binary.Write(&b, binary.LittleEndian, &data)
if !rp.IsMountPoint {
flags := uint32(0)
if relative {
flags |= 1
}
binary.Write(&b, binary.LittleEndian, flags)
_ = binary.Write(&b, binary.LittleEndian, flags)
}
binary.Write(&b, binary.LittleEndian, ntTarget16)
binary.Write(&b, binary.LittleEndian, target16)
_ = binary.Write(&b, binary.LittleEndian, ntTarget16)
_ = binary.Write(&b, binary.LittleEndian, target16)
return b.Bytes()
}
vendor/github.com/Microsoft/go-winio/sd.go
Generated Vendored
+55 -9
View File
@@ -1,23 +1,25 @@
//go:build windows
// +build windows
package winio
import (
"errors"
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
//sys lookupAccountName(systemName *uint16, accountName string, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) = advapi32.LookupAccountNameW
//sys lookupAccountSid(systemName *uint16, sid *byte, name *uint16, nameSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) = advapi32.LookupAccountSidW
//sys convertSidToStringSid(sid *byte, str **uint16) (err error) = advapi32.ConvertSidToStringSidW
//sys convertStringSidToSid(str *uint16, sid **byte) (err error) = advapi32.ConvertStringSidToSidW
//sys convertStringSecurityDescriptorToSecurityDescriptor(str string, revision uint32, sd *uintptr, size *uint32) (err error) = advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW
//sys convertSecurityDescriptorToStringSecurityDescriptor(sd *byte, revision uint32, secInfo uint32, sddl **uint16, sddlSize *uint32) (err error) = advapi32.ConvertSecurityDescriptorToStringSecurityDescriptorW
//sys localFree(mem uintptr) = LocalFree
//sys getSecurityDescriptorLength(sd uintptr) (len uint32) = advapi32.GetSecurityDescriptorLength
const (
cERROR_NONE_MAPPED = syscall.Errno(1332)
)
type AccountLookupError struct {
Name string
Err error
@@ -28,8 +30,10 @@ func (e *AccountLookupError) Error() string {
return "lookup account: empty account name specified"
}
var s string
switch e.Err {
case cERROR_NONE_MAPPED:
switch {
case errors.Is(e.Err, windows.ERROR_INVALID_SID):
s = "the security ID structure is invalid"
case errors.Is(e.Err, windows.ERROR_NONE_MAPPED):
s = "not found"
default:
s = e.Err.Error()
@@ -37,6 +41,8 @@ func (e *AccountLookupError) Error() string {
return "lookup account " + e.Name + ": " + s
}
func (e *AccountLookupError) Unwrap() error { return e.Err }
type SddlConversionError struct {
Sddl string
Err error
@@ -46,15 +52,19 @@ func (e *SddlConversionError) Error() string {
return "convert " + e.Sddl + ": " + e.Err.Error()
}
func (e *SddlConversionError) Unwrap() error { return e.Err }
// LookupSidByName looks up the SID of an account by name
//
//revive:disable-next-line:var-naming SID, not Sid
func LookupSidByName(name string) (sid string, err error) {
if name == "" {
return "", &AccountLookupError{name, cERROR_NONE_MAPPED}
return "", &AccountLookupError{name, windows.ERROR_NONE_MAPPED}
}
var sidSize, sidNameUse, refDomainSize uint32
err = lookupAccountName(nil, name, nil, &sidSize, nil, &refDomainSize, &sidNameUse)
if err != nil && err != syscall.ERROR_INSUFFICIENT_BUFFER {
if err != nil && err != syscall.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // err is Errno
return "", &AccountLookupError{name, err}
}
sidBuffer := make([]byte, sidSize)
@@ -73,6 +83,42 @@ func LookupSidByName(name string) (sid string, err error) {
return sid, nil
}
// LookupNameBySid looks up the name of an account by SID
//
//revive:disable-next-line:var-naming SID, not Sid
func LookupNameBySid(sid string) (name string, err error) {
if sid == "" {
return "", &AccountLookupError{sid, windows.ERROR_NONE_MAPPED}
}
sidBuffer, err := windows.UTF16PtrFromString(sid)
if err != nil {
return "", &AccountLookupError{sid, err}
}
var sidPtr *byte
if err = convertStringSidToSid(sidBuffer, &sidPtr); err != nil {
return "", &AccountLookupError{sid, err}
}
defer localFree(uintptr(unsafe.Pointer(sidPtr)))
var nameSize, refDomainSize, sidNameUse uint32
err = lookupAccountSid(nil, sidPtr, nil, &nameSize, nil, &refDomainSize, &sidNameUse)
if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // err is Errno
return "", &AccountLookupError{sid, err}
}
nameBuffer := make([]uint16, nameSize)
refDomainBuffer := make([]uint16, refDomainSize)
err = lookupAccountSid(nil, sidPtr, &nameBuffer[0], &nameSize, &refDomainBuffer[0], &refDomainSize, &sidNameUse)
if err != nil {
return "", &AccountLookupError{sid, err}
}
name = windows.UTF16ToString(nameBuffer)
return name, nil
}
func SddlToSecurityDescriptor(sddl string) ([]byte, error) {
var sdBuffer uintptr
err := convertStringSecurityDescriptorToSecurityDescriptor(sddl, 1, &sdBuffer, nil)
@@ -87,7 +133,7 @@ func SddlToSecurityDescriptor(sddl string) ([]byte, error) {
func SecurityDescriptorToSddl(sd []byte) (string, error) {
var sddl *uint16
// The returned string length seems to including an aribtrary number of terminating NULs.
// The returned string length seems to include an arbitrary number of terminating NULs.
// Don't use it.
err := convertSecurityDescriptorToStringSecurityDescriptor(&sd[0], 1, 0xff, &sddl, nil)
if err != nil {
vendor/github.com/Microsoft/go-winio/syscall.go
Generated Vendored
+3 -1
View File
@@ -1,3 +1,5 @@
//go:build windows
package winio
//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsyscall_windows.go file.go pipe.go sd.go fileinfo.go privilege.go backup.go hvsock.go
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go ./*.go
vendor/github.com/Microsoft/go-winio/tools.go
Generated Vendored
+5
View File
@@ -0,0 +1,5 @@
//go:build tools
package winio
import _ "golang.org/x/tools/cmd/stringer"
vendor/github.com/Microsoft/go-winio/vhd/vhd.go
Generated Vendored
+43 -16
View File
@@ -11,7 +11,7 @@ import (
"golang.org/x/sys/windows"
)
//go:generate go run mksyscall_windows.go -output zvhd_windows.go vhd.go
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zvhd_windows.go vhd.go
//sys createVirtualDisk(virtualStorageType *VirtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, createVirtualDiskFlags uint32, providerSpecificFlags uint32, parameters *CreateVirtualDiskParameters, overlapped *syscall.Overlapped, handle *syscall.Handle) (win32err error) = virtdisk.CreateVirtualDisk
//sys openVirtualDisk(virtualStorageType *VirtualStorageType, path string, virtualDiskAccessMask uint32, openVirtualDiskFlags uint32, parameters *openVirtualDiskParameters, handle *syscall.Handle) (win32err error) = virtdisk.OpenVirtualDisk
@@ -62,8 +62,8 @@ type OpenVirtualDiskParameters struct {
Version2 OpenVersion2
}
// The higher level `OpenVersion2` struct uses bools to refer to `GetInfoOnly` and `ReadOnly` for ease of use. However,
// the internal windows structure uses `BOOLS` aka int32s for these types. `openVersion2` is used for translating
// The higher level `OpenVersion2` struct uses `bool`s to refer to `GetInfoOnly` and `ReadOnly` for ease of use. However,
// the internal windows structure uses `BOOL`s aka int32s for these types. `openVersion2` is used for translating
// `OpenVersion2` fields to the correct windows internal field types on the `Open____` methods.
type openVersion2 struct {
getInfoOnly int32
@@ -87,9 +87,10 @@ type AttachVirtualDiskParameters struct {
}
const (
//revive:disable-next-line:var-naming ALL_CAPS
VIRTUAL_STORAGE_TYPE_DEVICE_VHDX = 0x3
// Access Mask for opening a VHD
// Access Mask for opening a VHD.
VirtualDiskAccessNone VirtualDiskAccessMask = 0x00000000
VirtualDiskAccessAttachRO VirtualDiskAccessMask = 0x00010000
VirtualDiskAccessAttachRW VirtualDiskAccessMask = 0x00020000
@@ -101,7 +102,7 @@ const (
VirtualDiskAccessAll VirtualDiskAccessMask = 0x003f0000
VirtualDiskAccessWritable VirtualDiskAccessMask = 0x00320000
// Flags for creating a VHD
// Flags for creating a VHD.
CreateVirtualDiskFlagNone CreateVirtualDiskFlag = 0x0
CreateVirtualDiskFlagFullPhysicalAllocation CreateVirtualDiskFlag = 0x1
CreateVirtualDiskFlagPreventWritesToSourceDisk CreateVirtualDiskFlag = 0x2
@@ -109,12 +110,12 @@ const (
CreateVirtualDiskFlagCreateBackingStorage CreateVirtualDiskFlag = 0x8
CreateVirtualDiskFlagUseChangeTrackingSourceLimit CreateVirtualDiskFlag = 0x10
CreateVirtualDiskFlagPreserveParentChangeTrackingState CreateVirtualDiskFlag = 0x20
CreateVirtualDiskFlagVhdSetUseOriginalBackingStorage CreateVirtualDiskFlag = 0x40
CreateVirtualDiskFlagVhdSetUseOriginalBackingStorage CreateVirtualDiskFlag = 0x40 //revive:disable-line:var-naming VHD, not Vhd
CreateVirtualDiskFlagSparseFile CreateVirtualDiskFlag = 0x80
CreateVirtualDiskFlagPmemCompatible CreateVirtualDiskFlag = 0x100
CreateVirtualDiskFlagPmemCompatible CreateVirtualDiskFlag = 0x100 //revive:disable-line:var-naming PMEM, not Pmem
CreateVirtualDiskFlagSupportCompressedVolumes CreateVirtualDiskFlag = 0x200
// Flags for opening a VHD
// Flags for opening a VHD.
OpenVirtualDiskFlagNone VirtualDiskFlag = 0x00000000
OpenVirtualDiskFlagNoParents VirtualDiskFlag = 0x00000001
OpenVirtualDiskFlagBlankFile VirtualDiskFlag = 0x00000002
@@ -127,7 +128,7 @@ const (
OpenVirtualDiskFlagNoWriteHardening VirtualDiskFlag = 0x00000100
OpenVirtualDiskFlagSupportCompressedVolumes VirtualDiskFlag = 0x00000200
// Flags for attaching a VHD
// Flags for attaching a VHD.
AttachVirtualDiskFlagNone AttachVirtualDiskFlag = 0x00000000
AttachVirtualDiskFlagReadOnly AttachVirtualDiskFlag = 0x00000001
AttachVirtualDiskFlagNoDriveLetter AttachVirtualDiskFlag = 0x00000002
@@ -140,12 +141,14 @@ const (
AttachVirtualDiskFlagSinglePartition AttachVirtualDiskFlag = 0x00000100
AttachVirtualDiskFlagRegisterVolume AttachVirtualDiskFlag = 0x00000200
// Flags for detaching a VHD
// Flags for detaching a VHD.
DetachVirtualDiskFlagNone DetachVirtualDiskFlag = 0x0
)
// CreateVhdx is a helper function to create a simple vhdx file at the given path using
// default values.
//
//revive:disable-next-line:var-naming VHDX, not Vhdx
func CreateVhdx(path string, maxSizeInGb, blockSizeInMb uint32) error {
params := CreateVirtualDiskParameters{
Version: 2,
@@ -172,6 +175,8 @@ func DetachVirtualDisk(handle syscall.Handle) (err error) {
}
// DetachVhd detaches a vhd found at `path`.
//
//revive:disable-next-line:var-naming VHD, not Vhd
func DetachVhd(path string) error {
handle, err := OpenVirtualDisk(
path,
@@ -181,12 +186,16 @@ func DetachVhd(path string) error {
if err != nil {
return err
}
defer syscall.CloseHandle(handle)
defer syscall.CloseHandle(handle) //nolint:errcheck
return DetachVirtualDisk(handle)
}
// AttachVirtualDisk attaches a virtual hard disk for use.
func AttachVirtualDisk(handle syscall.Handle, attachVirtualDiskFlag AttachVirtualDiskFlag, parameters *AttachVirtualDiskParameters) (err error) {
func AttachVirtualDisk(
handle syscall.Handle,
attachVirtualDiskFlag AttachVirtualDiskFlag,
parameters *AttachVirtualDiskParameters,
) (err error) {
// Supports both version 1 and 2 of the attach parameters as version 2 wasn't present in RS5.
if err := attachVirtualDisk(
handle,
@@ -203,6 +212,8 @@ func AttachVirtualDisk(handle syscall.Handle, attachVirtualDiskFlag AttachVirtua
// AttachVhd attaches a virtual hard disk at `path` for use. Attaches using version 2
// of the ATTACH_VIRTUAL_DISK_PARAMETERS.
//
//revive:disable-next-line:var-naming VHD, not Vhd
func AttachVhd(path string) (err error) {
handle, err := OpenVirtualDisk(
path,
@@ -213,7 +224,7 @@ func AttachVhd(path string) (err error) {
return err
}
defer syscall.CloseHandle(handle)
defer syscall.CloseHandle(handle) //nolint:errcheck
params := AttachVirtualDiskParameters{Version: 2}
if err := AttachVirtualDisk(
handle,
@@ -226,7 +237,11 @@ func AttachVhd(path string) (err error) {
}
// OpenVirtualDisk obtains a handle to a VHD opened with supplied access mask and flags.
func OpenVirtualDisk(vhdPath string, virtualDiskAccessMask VirtualDiskAccessMask, openVirtualDiskFlags VirtualDiskFlag) (syscall.Handle, error) {
func OpenVirtualDisk(
vhdPath string,
virtualDiskAccessMask VirtualDiskAccessMask,
openVirtualDiskFlags VirtualDiskFlag,
) (syscall.Handle, error) {
parameters := OpenVirtualDiskParameters{Version: 2}
handle, err := OpenVirtualDiskWithParameters(
vhdPath,
@@ -241,7 +256,12 @@ func OpenVirtualDisk(vhdPath string, virtualDiskAccessMask VirtualDiskAccessMask
}
// OpenVirtualDiskWithParameters obtains a handle to a VHD opened with supplied access mask, flags and parameters.
func OpenVirtualDiskWithParameters(vhdPath string, virtualDiskAccessMask VirtualDiskAccessMask, openVirtualDiskFlags VirtualDiskFlag, parameters *OpenVirtualDiskParameters) (syscall.Handle, error) {
func OpenVirtualDiskWithParameters(
vhdPath string,
virtualDiskAccessMask VirtualDiskAccessMask,
openVirtualDiskFlags VirtualDiskFlag,
parameters *OpenVirtualDiskParameters,
) (syscall.Handle, error) {
var (
handle syscall.Handle
defaultType VirtualStorageType
@@ -279,7 +299,12 @@ func OpenVirtualDiskWithParameters(vhdPath string, virtualDiskAccessMask Virtual
}
// CreateVirtualDisk creates a virtual harddisk and returns a handle to the disk.
func CreateVirtualDisk(path string, virtualDiskAccessMask VirtualDiskAccessMask, createVirtualDiskFlags CreateVirtualDiskFlag, parameters *CreateVirtualDiskParameters) (syscall.Handle, error) {
func CreateVirtualDisk(
path string,
virtualDiskAccessMask VirtualDiskAccessMask,
createVirtualDiskFlags CreateVirtualDiskFlag,
parameters *CreateVirtualDiskParameters,
) (syscall.Handle, error) {
var (
handle syscall.Handle
defaultType VirtualStorageType
@@ -323,6 +348,8 @@ func GetVirtualDiskPhysicalPath(handle syscall.Handle) (_ string, err error) {
}
// CreateDiffVhd is a helper function to create a differencing virtual disk.
//
//revive:disable-next-line:var-naming VHD, not Vhd
func CreateDiffVhd(diffVhdPath, baseVhdPath string, blockSizeInMB uint32) error {
// Setting `ParentPath` is how to signal to create a differencing disk.
createParams := &CreateVirtualDiskParameters{
vendor/github.com/Microsoft/go-winio/vhd/zvhd_windows.go
Generated Vendored
+3 -1
View File
@@ -1,4 +1,6 @@
// Code generated by 'go generate'; DO NOT EDIT.
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package vhd
vendor/github.com/Microsoft/go-winio/zsyscall_windows.go
Generated Vendored
+28 -17
View File
@@ -1,4 +1,6 @@
// Code generated by 'go generate'; DO NOT EDIT.
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package winio
@@ -47,9 +49,11 @@ var (
procConvertSecurityDescriptorToStringSecurityDescriptorW = modadvapi32.NewProc("ConvertSecurityDescriptorToStringSecurityDescriptorW")
procConvertSidToStringSidW = modadvapi32.NewProc("ConvertSidToStringSidW")
procConvertStringSecurityDescriptorToSecurityDescriptorW = modadvapi32.NewProc("ConvertStringSecurityDescriptorToSecurityDescriptorW")
procConvertStringSidToSidW = modadvapi32.NewProc("ConvertStringSidToSidW")
procGetSecurityDescriptorLength = modadvapi32.NewProc("GetSecurityDescriptorLength")
procImpersonateSelf = modadvapi32.NewProc("ImpersonateSelf")
procLookupAccountNameW = modadvapi32.NewProc("LookupAccountNameW")
procLookupAccountSidW = modadvapi32.NewProc("LookupAccountSidW")
procLookupPrivilegeDisplayNameW = modadvapi32.NewProc("LookupPrivilegeDisplayNameW")
procLookupPrivilegeNameW = modadvapi32.NewProc("LookupPrivilegeNameW")
procLookupPrivilegeValueW = modadvapi32.NewProc("LookupPrivilegeValueW")
@@ -74,7 +78,6 @@ var (
procRtlDosPathNameToNtPathName_U = modntdll.NewProc("RtlDosPathNameToNtPathName_U")
procRtlNtStatusToDosErrorNoTeb = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb")
procWSAGetOverlappedResult = modws2_32.NewProc("WSAGetOverlappedResult")
procbind = modws2_32.NewProc("bind")
)
func adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) {
@@ -123,6 +126,14 @@ func _convertStringSecurityDescriptorToSecurityDescriptor(str *uint16, revision
return
}
func convertStringSidToSid(str *uint16, sid **byte) (err error) {
r1, _, e1 := syscall.Syscall(procConvertStringSidToSidW.Addr(), 2, uintptr(unsafe.Pointer(str)), uintptr(unsafe.Pointer(sid)), 0)
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func getSecurityDescriptorLength(sd uintptr) (len uint32) {
r0, _, _ := syscall.Syscall(procGetSecurityDescriptorLength.Addr(), 1, uintptr(sd), 0, 0)
len = uint32(r0)
@@ -154,6 +165,14 @@ func _lookupAccountName(systemName *uint16, accountName *uint16, sid *byte, sidS
return
}
func lookupAccountSid(systemName *uint16, sid *byte, name *uint16, nameSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) {
r1, _, e1 := syscall.Syscall9(procLookupAccountSidW.Addr(), 7, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameSize)), uintptr(unsafe.Pointer(refDomain)), uintptr(unsafe.Pointer(refDomainSize)), uintptr(unsafe.Pointer(sidNameUse)), 0, 0)
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) {
var _p0 *uint16
_p0, err = syscall.UTF16PtrFromString(systemName)
@@ -380,25 +399,25 @@ func setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err erro
return
}
func ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntstatus) {
func ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) {
r0, _, _ := syscall.Syscall15(procNtCreateNamedPipeFile.Addr(), 14, uintptr(unsafe.Pointer(pipe)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(share), uintptr(disposition), uintptr(options), uintptr(typ), uintptr(readMode), uintptr(completionMode), uintptr(maxInstances), uintptr(inboundQuota), uintptr(outputQuota), uintptr(unsafe.Pointer(timeout)), 0)
status = ntstatus(r0)
status = ntStatus(r0)
return
}
func rtlDefaultNpAcl(dacl *uintptr) (status ntstatus) {
func rtlDefaultNpAcl(dacl *uintptr) (status ntStatus) {
r0, _, _ := syscall.Syscall(procRtlDefaultNpAcl.Addr(), 1, uintptr(unsafe.Pointer(dacl)), 0, 0)
status = ntstatus(r0)
status = ntStatus(r0)
return
}
func rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntstatus) {
func rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntStatus) {
r0, _, _ := syscall.Syscall6(procRtlDosPathNameToNtPathName_U.Addr(), 4, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(ntName)), uintptr(filePart), uintptr(reserved), 0, 0)
status = ntstatus(r0)
status = ntStatus(r0)
return
}
func rtlNtStatusToDosError(status ntstatus) (winerr error) {
func rtlNtStatusToDosError(status ntStatus) (winerr error) {
r0, _, _ := syscall.Syscall(procRtlNtStatusToDosErrorNoTeb.Addr(), 1, uintptr(status), 0, 0)
if r0 != 0 {
winerr = syscall.Errno(r0)
@@ -417,11 +436,3 @@ func wsaGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint
}
return
}
func bind(s syscall.Handle, name unsafe.Pointer, namelen int32) (err error) {
r1, _, e1 := syscall.Syscall(procbind.Addr(), 3, uintptr(s), uintptr(name), uintptr(namelen))
if r1 == socketError {
err = errnoErr(e1)
}
return
}
vendor/github.com/containerd/stargz-snapshotter/estargz/build.go
Generated Vendored
+39 -5
View File
@@ -26,10 +26,10 @@ import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"runtime"
@@ -48,6 +48,7 @@ type options struct {
prioritizedFiles []string
missedPrioritizedFiles *[]string
compression Compression
ctx context.Context
}
type Option func(o *options) error
@@ -104,6 +105,14 @@ func WithCompression(compression Compression) Option {
}
}
// WithContext specifies a context that can be used for clean canceleration.
func WithContext(ctx context.Context) Option {
return func(o *options) error {
o.ctx = ctx
return nil
}
}
// Blob is an eStargz blob.
type Blob struct {
io.ReadCloser
@@ -139,12 +148,29 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
opts.compression = newGzipCompressionWithLevel(opts.compressionLevel)
}
layerFiles := newTempFiles()
ctx := opts.ctx
if ctx == nil {
ctx = context.Background()
}
done := make(chan struct{})
defer close(done)
go func() {
select {
case <-done:
// nop
case <-ctx.Done():
layerFiles.CleanupAll()
}
}()
defer func() {
if rErr != nil {
if err := layerFiles.CleanupAll(); err != nil {
rErr = fmt.Errorf("failed to cleanup tmp files: %v: %w", err, rErr)
}
}
if cErr := ctx.Err(); cErr != nil {
rErr = fmt.Errorf("error from context %q: %w", cErr, rErr)
}
}()
tarBlob, err := decompressBlob(tarBlob, layerFiles)
if err != nil {
@@ -506,12 +532,13 @@ func newTempFiles() *tempFiles {
}
type tempFiles struct {
files []*os.File
filesMu sync.Mutex
files []*os.File
filesMu sync.Mutex
cleanupOnce sync.Once
}
func (tf *tempFiles) TempFile(dir, pattern string) (*os.File, error) {
f, err := ioutil.TempFile(dir, pattern)
f, err := os.CreateTemp(dir, pattern)
if err != nil {
return nil, err
}
@@ -521,7 +548,14 @@ func (tf *tempFiles) TempFile(dir, pattern string) (*os.File, error) {
return f, nil
}
func (tf *tempFiles) CleanupAll() error {
func (tf *tempFiles) CleanupAll() (err error) {
tf.cleanupOnce.Do(func() {
err = tf.cleanupAll()
})
return
}
func (tf *tempFiles) cleanupAll() error {
tf.filesMu.Lock()
defer tf.filesMu.Unlock()
var allErr []error
vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go
Generated Vendored
+2 -3
View File
@@ -31,7 +31,6 @@ import (
"fmt"
"hash"
"io"
"io/ioutil"
"os"
"path"
"sort"
@@ -579,7 +578,7 @@ func (fr *fileReader) ReadAt(p []byte, off int64) (n int, err error) {
return 0, fmt.Errorf("fileReader.ReadAt.decompressor.Reader: %v", err)
}
defer dr.Close()
if n, err := io.CopyN(ioutil.Discard, dr, off); n != off || err != nil {
if n, err := io.CopyN(io.Discard, dr, off); n != off || err != nil {
return 0, fmt.Errorf("discard of %d bytes = %v, %v", off, n, err)
}
return io.ReadFull(dr, p)
@@ -933,7 +932,7 @@ func (w *Writer) appendTar(r io.Reader, lossless bool) error {
}
}
}
remainDest := ioutil.Discard
remainDest := io.Discard
if lossless {
remainDest = dst // Preserve the remaining bytes in lossless mode
}
vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go
Generated Vendored
+69 -3
View File
@@ -31,8 +31,8 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
@@ -287,11 +287,11 @@ func isSameTarGz(t *testing.T, controller TestingController, a, b []byte) bool {
return false
}
aFile, err := ioutil.ReadAll(aTar)
aFile, err := io.ReadAll(aTar)
if err != nil {
t.Fatal("failed to read tar payload of A")
}
bFile, err := ioutil.ReadAll(bTar)
bFile, err := io.ReadAll(bTar)
if err != nil {
t.Fatal("failed to read tar payload of B")
}
@@ -1314,6 +1314,18 @@ func testWriteAndOpen(t *testing.T, controllers ...TestingController) {
),
wantFailOnLossLess: true,
},
{
name: "hardlink should be replaced to the destination entry",
in: tarOf(
dir("foo/"),
file("foo/foo1", "test"),
link("foolink", "foo/foo1"),
),
wantNumGz: 4, // dir, foo1 + link, TOC, footer
want: checks(
mustSameEntry("foo/foo1", "foolink"),
),
},
}
for _, tt := range tests {
@@ -1731,6 +1743,60 @@ func hasEntryOwner(entry string, owner owner) stargzCheck {
})
}
func mustSameEntry(files ...string) stargzCheck {
return stargzCheckFn(func(t *testing.T, r *Reader) {
var first *TOCEntry
for _, f := range files {
if first == nil {
var ok bool
first, ok = r.Lookup(f)
if !ok {
t.Errorf("unknown first file on Lookup: %q", f)
return
}
}
// Test Lookup
e, ok := r.Lookup(f)
if !ok {
t.Errorf("unknown file on Lookup: %q", f)
return
}
if e != first {
t.Errorf("Lookup: %+v(%p) != %+v(%p)", e, e, first, first)
return
}
// Test LookupChild
pe, ok := r.Lookup(filepath.Dir(filepath.Clean(f)))
if !ok {
t.Errorf("failed to get parent of %q", f)
return
}
e, ok = pe.LookupChild(filepath.Base(filepath.Clean(f)))
if !ok {
t.Errorf("failed to get %q as the child of %+v", f, pe)
return
}
if e != first {
t.Errorf("LookupChild: %+v(%p) != %+v(%p)", e, e, first, first)
return
}
// Test ForeachChild
pe.ForeachChild(func(baseName string, e *TOCEntry) bool {
if baseName == filepath.Base(filepath.Clean(f)) {
if e != first {
t.Errorf("ForeachChild: %+v(%p) != %+v(%p)", e, e, first, first)
return false
}
}
return true
})
}
})
}
func tarOf(s ...tarEntry) []tarEntry { return s }
type tarEntry interface {
vendor/github.com/containerd/stargz-snapshotter/estargz/types.go
Generated Vendored
+2 -1
View File
@@ -159,7 +159,8 @@ type TOCEntry struct {
// NumLink is the number of entry names pointing to this entry.
// Zero means one name references this entry.
NumLink int
// This field is calculated during runtime and not recorded in TOC JSON.
NumLink int `json:"-"`
// Xattrs are the extended attribute for the entry.
Xattrs map[string][]byte `json:"xattrs,omitempty"`
vendor/github.com/docker/cli/cli/config/config.go
Generated Vendored
+13 -9
View File
@@ -104,14 +104,18 @@ func LoadFromReader(configData io.Reader) (*configfile.ConfigFile, error) {
return &configFile, err
}
// TODO remove this temporary hack, which is used to warn about the deprecated ~/.dockercfg file
var printLegacyFileWarning bool
// Load reads the configuration files in the given directory, and sets up
// the auth config information and returns values.
// FIXME: use the internal golang config parser
func Load(configDir string) (*configfile.ConfigFile, error) {
printLegacyFileWarning = false
cfg, _, err := load(configDir)
return cfg, err
}
// TODO remove this temporary hack, which is used to warn about the deprecated ~/.dockercfg file
// so we can remove the bool return value and collapse this back into `Load`
func load(configDir string) (*configfile.ConfigFile, bool, error) {
printLegacyFileWarning := false
if configDir == "" {
configDir = Dir()
@@ -127,11 +131,11 @@ func Load(configDir string) (*configfile.ConfigFile, error) {
if err != nil {
err = errors.Wrap(err, filename)
}
return configFile, err
return configFile, printLegacyFileWarning, err
} else if !os.IsNotExist(err) {
// if file is there but we can't stat it for any reason other
// than it doesn't exist then stop
return configFile, errors.Wrap(err, filename)
return configFile, printLegacyFileWarning, errors.Wrap(err, filename)
}
// Can't find latest config file so check for the old one
@@ -140,16 +144,16 @@ func Load(configDir string) (*configfile.ConfigFile, error) {
printLegacyFileWarning = true
defer file.Close()
if err := configFile.LegacyLoadFromReader(file); err != nil {
return configFile, errors.Wrap(err, filename)
return configFile, printLegacyFileWarning, errors.Wrap(err, filename)
}
}
return configFile, nil
return configFile, printLegacyFileWarning, nil
}
// LoadDefaultConfigFile attempts to load the default config file and returns
// an initialized ConfigFile struct if none is found.
func LoadDefaultConfigFile(stderr io.Writer) *configfile.ConfigFile {
configFile, err := Load(Dir())
configFile, printLegacyFileWarning, err := load(Dir())
if err != nil {
fmt.Fprintf(stderr, "WARNING: Error loading config file: %v\n", err)
}
vendor/github.com/docker/cli/cli/config/configfile/file.go
Generated Vendored
+1 -1
View File
@@ -119,7 +119,7 @@ func (configFile *ConfigFile) LegacyLoadFromReader(configData io.Reader) error {
// LoadFromReader reads the configuration data given and sets up the auth config
// information with given directory and populates the receiver object
func (configFile *ConfigFile) LoadFromReader(configData io.Reader) error {
if err := json.NewDecoder(configData).Decode(&configFile); err != nil && !errors.Is(err, io.EOF) {
if err := json.NewDecoder(configData).Decode(configFile); err != nil && !errors.Is(err, io.EOF) {
return err
}
var err error
vendor/github.com/docker/cli/cli/config/configfile/file_unix.go
Generated Vendored
+1
View File
@@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package configfile
vendor/github.com/docker/cli/cli/config/credentials/default_store_unsupported.go
Generated Vendored
+1
View File
@@ -1,3 +1,4 @@
//go:build !windows && !darwin && !linux
// +build !windows,!darwin,!linux
package credentials
vendor/github.com/docker/cli/cli/config/credentials/native_store.go
Generated Vendored
+1 -1
View File
@@ -7,7 +7,7 @@ import (
)
const (
remoteCredentialsPrefix = "docker-credential-"
remoteCredentialsPrefix = "docker-credential-" //nolint:gosec // ignore G101: Potential hardcoded credentials
tokenUsername = "<token>"
)
vendor/github.com/docker/distribution/.mailmap
Generated Vendored
+3
View File
@@ -44,3 +44,6 @@ Thomas Berger <loki@lokis-chaos.de> Thomas Berger <tbe@users.noreply.github.com>
Samuel Karp <skarp@amazon.com> Samuel Karp <samuelkarp@users.noreply.github.com>
Justin Cormack <justin.cormack@docker.com>
sayboras <sayboras@yahoo.com>
CrazyMax <github@crazymax.dev>
CrazyMax <github@crazymax.dev> <1951866+crazy-max@users.noreply.github.com>
CrazyMax <github@crazymax.dev> <crazy-max@users.noreply.github.com>
vendor/github.com/docker/distribution/Dockerfile
Generated Vendored
+6 -2
View File
@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:1.3
ARG GO_VERSION=1.16
ARG GO_VERSION=1.16.15
ARG GORELEASER_XX_VERSION=1.2.5
FROM --platform=$BUILDPLATFORM crazymax/goreleaser-xx:${GORELEASER_XX_VERSION} AS goreleaser-xx
@@ -12,6 +12,10 @@ WORKDIR /go/src/github.com/docker/distribution
FROM base AS build
ENV GO111MODULE=auto
ENV CGO_ENABLED=0
# GIT_REF is used by goreleaser-xx to handle the proper git ref when available.
# It will fallback to the working tree info if empty and use "git tag --points-at"
# or "git describe" to define the version info.
ARG GIT_REF
ARG TARGETPLATFORM
ARG PKG="github.com/distribution/distribution"
ARG BUILDTAGS="include_oss include_gcs"
@@ -28,7 +32,7 @@ RUN --mount=type=bind,rw \
--files="LICENSE" \
--files="README.md"
FROM scratch AS artifacts
FROM scratch AS artifact
COPY --from=build /out/*.tar.gz /
COPY --from=build /out/*.zip /
COPY --from=build /out/*.sha256 /
vendor/github.com/docker/distribution/README.md
Generated Vendored
+1 -1
View File
@@ -2,7 +2,7 @@
The Docker toolset to pack, ship, store, and deliver content.
This repository's main product is the Docker Registry 2.0 implementation
This repository provides the Docker Registry 2.0 implementation
for storing and distributing Docker images. It supersedes the
[docker/docker-registry](https://github.com/docker/docker-registry)
project with a new API design, focused around security and performance.
vendor/github.com/docker/distribution/docker-bake.hcl
Generated Vendored
+16 -2
View File
@@ -1,3 +1,15 @@
// GITHUB_REF is the actual ref that triggers the workflow
// https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
variable "GITHUB_REF" {
default = ""
}
target "_common" {
args = {
GIT_REF = GITHUB_REF
}
}
group "default" {
targets = ["image-local"]
}
@@ -8,12 +20,14 @@ target "docker-metadata-action" {
}
target "binary" {
inherits = ["_common"]
target = "binary"
output = ["./bin"]
}
target "artifact" {
target = "artifacts"
inherits = ["_common"]
target = "artifact"
output = ["./bin"]
}
@@ -30,7 +44,7 @@ target "artifact-all" {
}
target "image" {
inherits = ["docker-metadata-action"]
inherits = ["_common", "docker-metadata-action"]
}
target "image-local" {
vendor/github.com/docker/distribution/registry/api/errcode/errors.go
Generated Vendored
-267
View File
@@ -1,267 +0,0 @@
package errcode
import (
"encoding/json"
"fmt"
"strings"
)
// ErrorCoder is the base interface for ErrorCode and Error allowing
// users of each to just call ErrorCode to get the real ID of each
type ErrorCoder interface {
ErrorCode() ErrorCode
}
// ErrorCode represents the error type. The errors are serialized via strings
// and the integer format may change and should *never* be exported.
type ErrorCode int
var _ error = ErrorCode(0)
// ErrorCode just returns itself
func (ec ErrorCode) ErrorCode() ErrorCode {
return ec
}
// Error returns the ID/Value
func (ec ErrorCode) Error() string {
// NOTE(stevvooe): Cannot use message here since it may have unpopulated args.
return strings.ToLower(strings.Replace(ec.String(), "_", " ", -1))
}
// Descriptor returns the descriptor for the error code.
func (ec ErrorCode) Descriptor() ErrorDescriptor {
d, ok := errorCodeToDescriptors[ec]
if !ok {
return ErrorCodeUnknown.Descriptor()
}
return d
}
// String returns the canonical identifier for this error code.
func (ec ErrorCode) String() string {
return ec.Descriptor().Value
}
// Message returned the human-readable error message for this error code.
func (ec ErrorCode) Message() string {
return ec.Descriptor().Message
}
// MarshalText encodes the receiver into UTF-8-encoded text and returns the
// result.
func (ec ErrorCode) MarshalText() (text []byte, err error) {
return []byte(ec.String()), nil
}
// UnmarshalText decodes the form generated by MarshalText.
func (ec *ErrorCode) UnmarshalText(text []byte) error {
desc, ok := idToDescriptors[string(text)]
if !ok {
desc = ErrorCodeUnknown.Descriptor()
}
*ec = desc.Code
return nil
}
// WithMessage creates a new Error struct based on the passed-in info and
// overrides the Message property.
func (ec ErrorCode) WithMessage(message string) Error {
return Error{
Code: ec,
Message: message,
}
}
// WithDetail creates a new Error struct based on the passed-in info and
// set the Detail property appropriately
func (ec ErrorCode) WithDetail(detail interface{}) Error {
return Error{
Code: ec,
Message: ec.Message(),
}.WithDetail(detail)
}
// WithArgs creates a new Error struct and sets the Args slice
func (ec ErrorCode) WithArgs(args ...interface{}) Error {
return Error{
Code: ec,
Message: ec.Message(),
}.WithArgs(args...)
}
// Error provides a wrapper around ErrorCode with extra Details provided.
type Error struct {
Code ErrorCode `json:"code"`
Message string `json:"message"`
Detail interface{} `json:"detail,omitempty"`
// TODO(duglin): See if we need an "args" property so we can do the
// variable substitution right before showing the message to the user
}
var _ error = Error{}
// ErrorCode returns the ID/Value of this Error
func (e Error) ErrorCode() ErrorCode {
return e.Code
}
// Error returns a human readable representation of the error.
func (e Error) Error() string {
return fmt.Sprintf("%s: %s", e.Code.Error(), e.Message)
}
// WithDetail will return a new Error, based on the current one, but with
// some Detail info added
func (e Error) WithDetail(detail interface{}) Error {
return Error{
Code: e.Code,
Message: e.Message,
Detail: detail,
}
}
// WithArgs uses the passed-in list of interface{} as the substitution
// variables in the Error's Message string, but returns a new Error
func (e Error) WithArgs(args ...interface{}) Error {
return Error{
Code: e.Code,
Message: fmt.Sprintf(e.Code.Message(), args...),
Detail: e.Detail,
}
}
// ErrorDescriptor provides relevant information about a given error code.
type ErrorDescriptor struct {
// Code is the error code that this descriptor describes.
Code ErrorCode
// Value provides a unique, string key, often captilized with
// underscores, to identify the error code. This value is used as the
// keyed value when serializing api errors.
Value string
// Message is a short, human readable decription of the error condition
// included in API responses.
Message string
// Description provides a complete account of the errors purpose, suitable
// for use in documentation.
Description string
// HTTPStatusCode provides the http status code that is associated with
// this error condition.
HTTPStatusCode int
}
// ParseErrorCode returns the value by the string error code.
// `ErrorCodeUnknown` will be returned if the error is not known.
func ParseErrorCode(value string) ErrorCode {
ed, ok := idToDescriptors[value]
if ok {
return ed.Code
}
return ErrorCodeUnknown
}
// Errors provides the envelope for multiple errors and a few sugar methods
// for use within the application.
type Errors []error
var _ error = Errors{}
func (errs Errors) Error() string {
switch len(errs) {
case 0:
return "<nil>"
case 1:
return errs[0].Error()
default:
msg := "errors:\n"
for _, err := range errs {
msg += err.Error() + "\n"
}
return msg
}
}
// Len returns the current number of errors.
func (errs Errors) Len() int {
return len(errs)
}
// MarshalJSON converts slice of error, ErrorCode or Error into a
// slice of Error - then serializes
func (errs Errors) MarshalJSON() ([]byte, error) {
var tmpErrs struct {
Errors []Error `json:"errors,omitempty"`
}
for _, daErr := range errs {
var err Error
switch daErr := daErr.(type) {
case ErrorCode:
err = daErr.WithDetail(nil)
case Error:
err = daErr
default:
err = ErrorCodeUnknown.WithDetail(daErr)
}
// If the Error struct was setup and they forgot to set the
// Message field (meaning its "") then grab it from the ErrCode
msg := err.Message
if msg == "" {
msg = err.Code.Message()
}
tmpErrs.Errors = append(tmpErrs.Errors, Error{
Code: err.Code,
Message: msg,
Detail: err.Detail,
})
}
return json.Marshal(tmpErrs)
}
// UnmarshalJSON deserializes []Error and then converts it into slice of
// Error or ErrorCode
func (errs *Errors) UnmarshalJSON(data []byte) error {
var tmpErrs struct {
Errors []Error
}
if err := json.Unmarshal(data, &tmpErrs); err != nil {
return err
}
var newErrs Errors
for _, daErr := range tmpErrs.Errors {
// If Message is empty or exactly matches the Code's message string
// then just use the Code, no need for a full Error struct
if daErr.Detail == nil && (daErr.Message == "" || daErr.Message == daErr.Code.Message()) {
// Error's w/o details get converted to ErrorCode
newErrs = append(newErrs, daErr.Code)
} else {
// Error's w/ details are untouched
newErrs = append(newErrs, Error{
Code: daErr.Code,
Message: daErr.Message,
Detail: daErr.Detail,
})
}
}
*errs = newErrs
return nil
}
vendor/github.com/docker/distribution/registry/api/errcode/handler.go
Generated Vendored
-40
View File
@@ -1,40 +0,0 @@
package errcode
import (
"encoding/json"
"net/http"
)
// ServeJSON attempts to serve the errcode in a JSON envelope. It marshals err
// and sets the content-type header to 'application/json'. It will handle
// ErrorCoder and Errors, and if necessary will create an envelope.
func ServeJSON(w http.ResponseWriter, err error) error {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
var sc int
switch errs := err.(type) {
case Errors:
if len(errs) < 1 {
break
}
if err, ok := errs[0].(ErrorCoder); ok {
sc = err.ErrorCode().Descriptor().HTTPStatusCode
}
case ErrorCoder:
sc = errs.ErrorCode().Descriptor().HTTPStatusCode
err = Errors{err} // create an envelope.
default:
// We just have an unhandled error type, so just place in an envelope
// and move along.
err = Errors{err}
}
if sc == 0 {
sc = http.StatusInternalServerError
}
w.WriteHeader(sc)
return json.NewEncoder(w).Encode(err)
}
vendor/github.com/docker/distribution/registry/api/errcode/register.go
Generated Vendored
-138
View File
@@ -1,138 +0,0 @@
package errcode
import (
"fmt"
"net/http"
"sort"
"sync"
)
var (
errorCodeToDescriptors = map[ErrorCode]ErrorDescriptor{}
idToDescriptors = map[string]ErrorDescriptor{}
groupToDescriptors = map[string][]ErrorDescriptor{}
)
var (
// ErrorCodeUnknown is a generic error that can be used as a last
// resort if there is no situation-specific error message that can be used
ErrorCodeUnknown = Register("errcode", ErrorDescriptor{
Value: "UNKNOWN",
Message: "unknown error",
Description: `Generic error returned when the error does not have an
API classification.`,
HTTPStatusCode: http.StatusInternalServerError,
})
// ErrorCodeUnsupported is returned when an operation is not supported.
ErrorCodeUnsupported = Register("errcode", ErrorDescriptor{
Value: "UNSUPPORTED",
Message: "The operation is unsupported.",
Description: `The operation was unsupported due to a missing
implementation or invalid set of parameters.`,
HTTPStatusCode: http.StatusMethodNotAllowed,
})
// ErrorCodeUnauthorized is returned if a request requires
// authentication.
ErrorCodeUnauthorized = Register("errcode", ErrorDescriptor{
Value: "UNAUTHORIZED",
Message: "authentication required",
Description: `The access controller was unable to authenticate
the client. Often this will be accompanied by a
Www-Authenticate HTTP response header indicating how to
authenticate.`,
HTTPStatusCode: http.StatusUnauthorized,
})
// ErrorCodeDenied is returned if a client does not have sufficient
// permission to perform an action.
ErrorCodeDenied = Register("errcode", ErrorDescriptor{
Value: "DENIED",
Message: "requested access to the resource is denied",
Description: `The access controller denied access for the
operation on a resource.`,
HTTPStatusCode: http.StatusForbidden,
})
// ErrorCodeUnavailable provides a common error to report unavailability
// of a service or endpoint.
ErrorCodeUnavailable = Register("errcode", ErrorDescriptor{
Value: "UNAVAILABLE",
Message: "service unavailable",
Description: "Returned when a service is not available",
HTTPStatusCode: http.StatusServiceUnavailable,
})
// ErrorCodeTooManyRequests is returned if a client attempts too many
// times to contact a service endpoint.
ErrorCodeTooManyRequests = Register("errcode", ErrorDescriptor{
Value: "TOOMANYREQUESTS",
Message: "too many requests",
Description: `Returned when a client attempts to contact a
service too many times`,
HTTPStatusCode: http.StatusTooManyRequests,
})
)
var nextCode = 1000
var registerLock sync.Mutex
// Register will make the passed-in error known to the environment and
// return a new ErrorCode
func Register(group string, descriptor ErrorDescriptor) ErrorCode {
registerLock.Lock()
defer registerLock.Unlock()
descriptor.Code = ErrorCode(nextCode)
if _, ok := idToDescriptors[descriptor.Value]; ok {
panic(fmt.Sprintf("ErrorValue %q is already registered", descriptor.Value))
}
if _, ok := errorCodeToDescriptors[descriptor.Code]; ok {
panic(fmt.Sprintf("ErrorCode %v is already registered", descriptor.Code))
}
groupToDescriptors[group] = append(groupToDescriptors[group], descriptor)
errorCodeToDescriptors[descriptor.Code] = descriptor
idToDescriptors[descriptor.Value] = descriptor
nextCode++
return descriptor.Code
}
type byValue []ErrorDescriptor
func (a byValue) Len() int { return len(a) }
func (a byValue) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byValue) Less(i, j int) bool { return a[i].Value < a[j].Value }
// GetGroupNames returns the list of Error group names that are registered
func GetGroupNames() []string {
keys := []string{}
for k := range groupToDescriptors {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// GetErrorCodeGroup returns the named group of error descriptors
func GetErrorCodeGroup(name string) []ErrorDescriptor {
desc := groupToDescriptors[name]
sort.Sort(byValue(desc))
return desc
}
// GetErrorAllDescriptors returns a slice of all ErrorDescriptors that are
// registered, irrespective of what group they're in
func GetErrorAllDescriptors() []ErrorDescriptor {
result := []ErrorDescriptor{}
for _, group := range GetGroupNames() {
result = append(result, GetErrorCodeGroup(group)...)
}
sort.Sort(byValue(result))
return result
}
vendor/github.com/docker/docker-credential-helpers/credentials/credentials.go
Generated Vendored
+4 -4
View File
@@ -169,8 +169,8 @@ func Erase(helper Helper, reader io.Reader) error {
return helper.Delete(serverURL)
}
//List returns all the serverURLs of keys in
//the OS store as a list of strings
// List returns all the serverURLs of keys in
// the OS store as a list of strings
func List(helper Helper, writer io.Writer) error {
accts, err := helper.List()
if err != nil {
@@ -179,8 +179,8 @@ func List(helper Helper, writer io.Writer) error {
return json.NewEncoder(writer).Encode(accts)
}
//PrintVersion outputs the current version.
// PrintVersion outputs the current version.
func PrintVersion(writer io.Writer) error {
fmt.Fprintln(writer, Version)
fmt.Fprintf(writer, "%s (%s) %s\n", Name, Package, Version)
return nil
}
vendor/github.com/docker/docker-credential-helpers/credentials/version.go
Generated Vendored
+14 -2
View File
@@ -1,4 +1,16 @@
package credentials
// Version holds a string describing the current version
const Version = "0.6.4"
var (
// Name is filled at linking time
Name = ""
// Package is filled at linking time
Package = "github.com/docker/docker-credential-helpers"
// Version holds the complete version number. Filled in at linking time.
Version = "v0.0.0+unknown"
// Revision is filled with the VCS (e.g. git) revision being used to build
// the program at linking time.
Revision = ""
)
vendor/github.com/docker/docker/api/common_unix.go
Generated Vendored
+1
View File
@@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package api // import "github.com/docker/docker/api"
vendor/github.com/docker/docker/api/swagger.yaml
Generated Vendored
+54 -2
View File
@@ -24,7 +24,7 @@ info:
title: "Docker Engine API"
version: "1.41"
x-logo:
url: "https://docs.docker.com/images/logo-docker-main.png"
url: "https://docs.docker.com/assets/images/logo-docker-main.png"
description: |
The Engine API is an HTTP API served by Docker Engine. It is the API the
Docker client uses to communicate with the Engine, so everything the Docker
@@ -1891,23 +1891,52 @@ definitions:
BuildCache:
type: "object"
description: |
BuildCache contains information about a build cache record.
properties:
ID:
type: "string"
description: |
Unique ID of the build cache record.
example: "ndlpt0hhvkqcdfkputsk4cq9c"
Parent:
description: |
ID of the parent build cache record.
type: "string"
example: "hw53o5aio51xtltp5xjp8v7fx"
Type:
type: "string"
description: |
Cache record type.
example: "regular"
# see https://github.com/moby/buildkit/blob/fce4a32258dc9d9664f71a4831d5de10f0670677/client/diskusage.go#L75-L84
enum:
- "internal"
- "frontend"
- "source.local"
- "source.git.checkout"
- "exec.cachemount"
- "regular"
Description:
type: "string"
description: |
Description of the build-step that produced the build cache.
example: "mount / from exec /bin/sh -c echo 'Binary::apt::APT::Keep-Downloaded-Packages \"true\";' > /etc/apt/apt.conf.d/keep-cache"
InUse:
type: "boolean"
description: |
Indicates if the build cache is in use.
example: false
Shared:
type: "boolean"
description: |
Indicates if the build cache is shared.
example: true
Size:
description: |
Amount of disk space used by the build cache (in bytes).
type: "integer"
example: 51
CreatedAt:
description: |
Date and time at which the build cache was created in
@@ -1925,6 +1954,7 @@ definitions:
example: "2017-08-09T07:09:37.632105588Z"
UsageCount:
type: "integer"
example: 26
ImageID:
type: "object"
@@ -3347,7 +3377,7 @@ definitions:
Limits:
description: "Define resources limits."
$ref: "#/definitions/Limit"
Reservation:
Reservations:
description: "Define resources reservation."
$ref: "#/definitions/ResourceObject"
RestartPolicy:
@@ -5415,6 +5445,28 @@ paths:
`/?[a-zA-Z0-9][a-zA-Z0-9_.-]+`.
type: "string"
pattern: "^/?[a-zA-Z0-9][a-zA-Z0-9_.-]+$"
- name: "platform"
in: "query"
description: |
Platform in the format `os[/arch[/variant]]` used for image lookup.
When specified, the daemon checks if the requested image is present
in the local image cache with the given OS and Architecture, and
otherwise returns a `404` status.
If the option is not set, the host's native OS and Architecture are
used to look up the image in the image cache. However, if no platform
is passed and the given image does exist in the local image cache,
but its OS or architecture does not match, the container is created
with the available image, and a warning is added to the `Warnings`
field in the response, for example;
WARNING: The requested image's platform (linux/arm64/v8) does not
match the detected host platform (linux/amd64) and no
specific platform was requested
type: "string"
default: ""
- name: "body"
in: "body"
description: "Container to create"
vendor/github.com/docker/docker/api/types/container/hostconfig_unix.go
Generated Vendored
+1
View File
@@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package container // import "github.com/docker/docker/api/types/container"

Some files were not shown because too many files have changed in this diff Show More