public_git
/
postgres-operator
mirror of https://github.com/zalando/postgres-operator.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

move tests to a Docker container

This commit is contained in:
Sergey Dudoladov 2019-05-10 17:00:16 +02:00
parent 084167ff35
commit 407b2cab41
10 changed files with 41 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -15,8 +15,9 @@ before_install:
- go get github.com/mattn/goveralls - go get github.com/mattn/goveralls
install: install:
- make deps - make deps e2e-tools e2e-build
script: script:
- hack/verify-codegen.sh - hack/verify-codegen.sh
- travis_wait 20 goveralls -service=travis-ci -package ./pkg/... -v - travis_wait 20 goveralls -service=travis-ci -package ./pkg/... -v
- make e2e-run

14
Makefile
View File

@ -1,4 +1,4 @@
.PHONY: clean local test linux macos docker push scm-source.json e2e e2e-tools .PHONY: clean local test linux macos docker push scm-source.json e2e-run e2e-tools e2e-build
BINARY ?= postgres-operator BINARY ?= postgres-operator
BUILD_FLAGS ?= -v BUILD_FLAGS ?= -v
@ -92,11 +92,11 @@ test:
hack/verify-codegen.sh hack/verify-codegen.sh
@go test ./... @go test ./...
e2e-tools: e2e-build:
@pip3 install -r e2e/requirements.txt docker build --tag="postgres-operator-e2e-tests" -f e2e/Dockerfile .
@go get -u sigs.k8s.io/kind
# assumes kubectl is already isntalled
e2e: docker e2e-tools:
@go get -u sigs.k8s.io/kind
e2e-run: docker
e2e/run.sh e2e/run.sh
flake8 --exit-zero

5
delivery.yaml
View File

@ -46,10 +46,7 @@ pipeline:
export PATH=$PATH:$HOME/go/bin export PATH=$PATH:$HOME/go/bin
cd $OPERATOR_TOP_DIR/postgres-operator cd $OPERATOR_TOP_DIR/postgres-operator
echo "INFO install kubectl to test 'kind' from client side" echo "INFO install kubectl to test 'kind' from client side"
(curl -LO --silent https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && make e2e-tools e2e-build e2e-run
chmod +x ./kubectl &&
mv ./kubectl /usr/local/bin/kubectl) || exit 1
make e2e-tools e2e
- desc: 'Push docker image' - desc: 'Push docker image'
cmd: | cmd: |
export PATH=$PATH:$HOME/go/bin export PATH=$PATH:$HOME/go/bin

17
docs/developer.md
View File

@ -317,20 +317,11 @@ kubectl logs acid-minimal-cluster-0
## End-to-end tests ## End-to-end tests
The operator provides e2e (end-to-end) tests to ensure various infra parts work smoothly together. The operator provides reference e2e (end-to-end) tests to ensure various infra parts work smoothly together.
Such tests employ [kind](https://kind.sigs.k8s.io/) to start a local k8s cluster. Each e2e execution tests a Postgres operator image built from the current git branch. The test runner starts a [kind](https://kind.sigs.k8s.io/) (local k8s) cluster and Docker container with tests. The k8s API client from within the container connects to the `kind` cluster using the standard Docker `bridge` network.
We intend to execute e2e tests during builds, but you can invoke them locally with `make e2e` from the project's top directory; run `make e2e-tools` to install `kind` and Python dependencies before the first run. Each e2e execution tests a Postgres operator image built from the current git branch. The tests utilize examples from `/manifests` (ConfigMap is used for the operator configuration) to avoid maintaining yet another set of configuration files. The kind cluster is deleted if tests complete successfully.
### Smoke tests
The first provided collection of [smoke tests](../e2e/tests/test_smoke.py) covers the most basic operator capabilities. These tests utilize examples from `/manifests` (ConfigMap is used for the operator configuration) to avoid maintaining yet another set of configuration files. The `kind-smoke-test-postgres-operator` cluster is deleted if tests complete successfully.
### Contributing tests suites
1. Consider using a separate `kind` cluster per a logically separate collection of tests. This approach enables simple clean up after test success by deleting the relevant cluster.
2. Make tests time out; for now we use the [timeout-decorator](https://github.com/pnpnpn/timeout-decorator) for that purpose.
3. Please briefly describe the use case for the new test collection here and in the comments.
End-to-end tests are executed automatically during builds; to invoke them locally use `make e2e-run` from the project's top directory. Run `make e2e-tools e2e-build` to install `kind` and build the tests' image locally before the first run.
## Introduce additional configuration parameters ## Introduce additional configuration parameters

18
e2e/Dockerfile
View File

@ -1,18 +1,16 @@
FROM ubuntu:18.04 FROM ubuntu:18.04
MAINTAINER Team ACID @ Zalando <team-acid@zalando.de> LABEL maintainer="Team ACID @ Zalando <team-acid@zalando.de>"
WORKDIR /e2e WORKDIR /e2e
COPY e2e/requirements.txt ./ COPY manifests ./manifests
COPY e2e/manifests ./manifests COPY e2e/requirements.txt e2e/tests ./
COPY e2e/tests ./tests
COPY e2e/kind-config-kind-smoke-test-postgres-operator /root/.kube/config
RUN apt-get update RUN apt-get update \
RUN apt-get install -y python3 python3-pip curl \ && apt-get install -y python3 python3-pip curl \
&& pip3 install --no-cache-dir -r requirements.txt && pip3 install --no-cache-dir -r requirements.txt \
RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \ && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \
&& chmod +x ./kubectl \ && chmod +x ./kubectl \
&& mv ./kubectl /usr/local/bin/kubectl && mv ./kubectl /usr/local/bin/kubectl
CMD ["python3", "-m", "unittest", "discover", "--start-directory", "tests", "-v"] CMD ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"]

0
e2e/kind-config-smoke-tests.yaml → e2e/kind-cluster-postgres-operator-e2e-tests.yaml
View File

19
e2e/kind-config-kind-smoke-test-postgres-operator
View File

@ -1,19 +0,0 @@
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1EVXdPVEUxTlRVeU1Gb1hEVEk1TURVd05qRTFOVFV5TUZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTWhZCjhTSVNNejlYb3VDMG9HQkRkNmVFdk5QQS9VZVJldHBucFRiUkNNU2tObUNkSFgzVWQ3SC9tTVdJcnVwdVhUeDQKWUV5dmJqWUt3YnJ5bUpTRU5xQ0h2RHE1SlV4TWE0dEZWRFBTNUZIaThiRUVHYW1UVUM3alo0Q2hma2hPN0ZSRQpKVW1xeDBLVDJ5SVVySDZTT203ajc3SDlOL0J5SGhhdnRJNXJsSlp3emdsbGw2eCtsV0RGbEExMndmYVVUaTdlCmpnd1Mzcjl4N0h3R1JQejVHNmhmaGlmSzZMVVhsNXAzYjB0RTZ1QVllNFBlWlNtQmJuS1UwdzAzQlJoYmpmTmIKSEdleUJIYm43RWhWZmJ4Y1dIV2RnaEMzV1dWRDdSa05NTXQvREtPbUdHR3Bab1RCeEpTSXR3UEFyMS9tUGVMRQp5VkxubjJmYzhDZ0dXT01SS0MwQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFEb2RRS1Nzd2NJa0NRMFVGUWNJRi9yNmtMQ2YKSjFHL3ZkQ1FkTW1taU9rcWRsMW1tQjAyL2d1UDZtL3AxbUltRmRmWjZJNGlPNGpwUCsvaGllMDB5aUMwMS84MgpLaUE2a292ckpMVzZnQ05rR2Nyb3hhdHRITklEYlF1TkVoZ3RJYWgxK2cxclo4eStTblU4b2pSRVFXYTZJUGJQClJKQjQyZjZ4dFFxZ1BJanRBT2EyeGpuMFdDNGpRTXNiSUx0M2Z0S2RLOTFmNG5BTFBmZGRSdyt6N3NxcWZNeFEKQU9UL3NycVo0eVJqT3d1Q2w4clBsTFVEUzBMTkRuM2FST0VDNVF6dm1nUElKY2k1ZWIrV1JGWEthblpjU2FXMwp6VFlVck9oTStSNEEyUGRBWWFSbm5qdnRGZ3JuY1cwRHVSb2tWUnFHL0Rvd2xHbzdZRUxQNEg4eDVHST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
server: https://172.17.0.3:6443
name: kind-smoke-test-postgres-operator
contexts:
- context:
cluster: kind-smoke-test-postgres-operator
user: kubernetes-admin
name: kubernetes-admin@kind-smoke-test-postgres-operator
current-context: kubernetes-admin@kind-smoke-test-postgres-operator
kind: Config
preferences: {}
users:
- name: kubernetes-admin
user:
client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJWTVVRkRiKzc1c1l3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T1RBMU1Ea3hOVFUxTWpCYUZ3MHlNREExTURneE5UVTFNakJhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQW5OVkEvMFcrQmxqOENQcjYKZ1EvZUU0aWhMSnh3dlNuSEQ1VE1DYllRSFZFMTN5NmRiclRzTUhyU2duMlZBMmtaSGFEbmpPRXFieG0zMmt5aQowZmJhMzNZVm5BdGwrVmtyVzc2VW1USUFhb1RxaFNkRlpFczlrT3N5WUgweGluSXRMMExrWE9IeWwwVlVSUXVHCm94TVFxU2duaGpSMWJJVjdKbzYvYnhSbU42ZWJwb1FSRHN0QWk5Z2QramlHSU93Q05SWHVhM1B4Nm1DK04rVFoKTnZzRjNieHdIWURwTGo3aU1mNGUvRlM4MjQzVEU4eUo0Ymp1ejNyOHBsR2t4RkU0ZjZJbEtVQVpDK1dKN01KZAp2MDdOMDZVVFg3WlZxekE3R1l6QkROb2UrSmQ0S3JwcDFHbVdIWFgxaU9yRnhHNE9FSlFvbXE0c205VFhha25PCmlSVUQyUUlEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDMHdMSDRjTnBNeHlTQlRiVmlXYzd0Yk8rT2dyRlYzMmltNQpCa3pyMDBrd05nYmw5VFFUNFpkZDlHRnFTRVFpUDNXT2lWNjErMHpNdEpUcEthL0hITnBhNXg5MmhLc2FKZHdpCnErRm5TRisyVytIK0pnTy9tM0l6Z1VQK25kUVNNUXdwWmQ3bWQ0TnE0Wnh6VThrOTYwWWRMYlhHRXdTQTF5Ty8KN3RVQUJWWHVWdFFFaTQvQ1NSTXFjNndLT3NFNUU3QUNiYkU2MXhFTThsUzg5ZVhyaSs0aDZGenBpbkZWa1ZiRwpXaTAyYUV6cVJuTGFqV1NOZThuYnh2ZG1sRGhUZjFxYjkvWGJ0cWxIQWRMc1ZpdU5GT2Jxa3Yzb3FwSW5kQVNBClVRUDMxWXZ4VERPaEpaY3k2dkVyK2tqT2RpZjgrUytJa05BRnhzeVFPK0Vmc05zWUdvdz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBbk5WQS8wVytCbGo4Q1ByNmdRL2VFNGloTEp4d3ZTbkhENVRNQ2JZUUhWRTEzeTZkCmJyVHNNSHJTZ24yVkEya1pIYURuak9FcWJ4bTMya3lpMGZiYTMzWVZuQXRsK1Zrclc3NlVtVElBYW9UcWhTZEYKWkVzOWtPc3lZSDB4aW5JdEwwTGtYT0h5bDBWVVJRdUdveE1RcVNnbmhqUjFiSVY3Sm82L2J4Um1ONmVicG9RUgpEc3RBaTlnZCtqaUdJT3dDTlJYdWEzUHg2bUMrTitUWk52c0YzYnh3SFlEcExqN2lNZjRlL0ZTODI0M1RFOHlKCjRianV6M3I4cGxHa3hGRTRmNklsS1VBWkMrV0o3TUpkdjA3TjA2VVRYN1pWcXpBN0dZekJETm9lK0pkNEtycHAKMUdtV0hYWDFpT3JGeEc0T0VKUW9tcTRzbTlUWGFrbk9pUlVEMlFJREFRQUJBb0lCQVFDTTM3R0dteXJab015agpkRzNYeUZ6K3h0ZWZydFpGMUdVT1JlWVJRd3l1aU9nUEZWd3N1UzcvVFJRU1NxT3pjSkF5NFBtY3ZoVFR2eEk2CmNHUkFuYkIwMFNrUUJkMFBZVjFsQjRlTEpETGplNGo5R2cxbXpYNzcwWWhxeTRuWWhqNjRHU252bExYSDAycWkKcW52QnQ3cGJkOG9vN3E0YlVMc1NJMThwYy9WdFB5U1lmd3h3OVpFTEdRb3ZNNklRTGNBb1RxdHdKYXZDZmlKQQpKYjZFaXdROUljOTB1MDJxaGE3RGw4K3lrWmRaSnJmNmYwditoZDUrdGFRK3RkNnZMd3RvUGZCa2licXhBNkx1CkxId2IvWnBSOGlKbXJFTjkwdFNMTFpMWVAzQkhkYUNNcXdJYllXUkRsS2p2cE1DUUp2NDgrdDhuMERNcUVzSFUKQi9XK2EwdEJBb0dCQU02b3VHTmhUMWpkTTlSenBITzg3enpiaHh2T0VWQ3BDN29zdGl1dmJnUnVGcFl2WloxTQpzTHY5dVBrTFlmanAxais3VzJvWncvQ25Zcjh0ZGVYTHVBKzNpcXk5bzZlblhjVEExM0wwTFNHY0tuZXAwenpxCmtqUmNGYmZEOXNpM2Jsc0FmT0hnSzY5MFByZmdINjF1bW5rYTVjWEhSc3dTcThBbVBHVU5EQWxsQW9HQkFNSkgKR1Z3WWhlN2hlbE5FQ1FNTWxENW90c01zWGJtWEFzbUFCZ2dxQXhRVkw0dmVUTk9XZHV2b2t1eDFVVWZhMVhqdgpUNEduYTZmeCtvMDUyRlNkU1FLTkEwNlhwVWhxZUNIcHU1ZmV3MHV0WVJhblhrS2MzRnZ5NEJlbEd3aHZUUVJRCnIxVmdlWWc0V3A2bTFuaHJwRDEydGh2WDE2Rm9uN0I4Skp2clVxTmxBb0dCQU1vcVk3ZFV5cnEwS3EvN01UWEgKN29JcWY5SERsVXpERXFYZWQ1Zmsxa3VmSnBsbFpKS3RJM2ZFamQrVU14TytMY25MRDNLTUloS2FyUTg0K2MwRApyZHd5UVljYlBhNFZITFlOc0xiVUNCS0pJMEpNOEVqM2NHK29aZGFQN2l3TXhmaGdVY3JsOGRhQ2NaaVB1RzJCCmRieGpnOFFuWGlybFdQOXdhRVN5cnNQQkFvR0FmRkxwYktFWTNHU1lWajZja2NIei8vZ2N0TXRvY3dLck91MWQKYnM0THlFZENkUHhlSjYwTER5NTNEekNJUWpaTkU2WDVPQncrYld3UmpWeXVEbi9Vbi9oRFhJRDR1VjNBNE5ybApQR3ZHaUdBOFdEWGt3VFlHWWlVTHVMWGtsY0k4QS8zcUpmV2w4RUUzNUgwWmxGZzE4MHRMZ0lmZ3FwNzhTZ0UzCm9EdTRWMjBDZ1lCNER0MDNoNG4zWGxPMW8rZlBKMGJETnZnb1JGZnZYYWk2YjBVa1VpcXNKclRTZVFrRVEzenUKNnFPRnBLNUgzRDlWc3lweWF0N3JJNmk4bFQzbHlsaUU1dGdzMEV5ektvSjJCYVFEOE5MRVJueHVyRHFkV2paaQp6NXNJSjU0c0N6NDJjTytQSkVTUjRCTlBZU1ZNUEhDZURWd1diL1QwNXdOU1NxemIrRWJQN1E9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=

6
e2e/requirements.txt
View File

@ -1,4 +1,2 @@
kubernetes kubernetes==9.0.0
flake8 timeout_decorator==0.4.1
timeout_decorator
docker

25
e2e/run.sh
View File

@ -6,7 +6,11 @@ set -o nounset
set -o pipefail set -o pipefail
IFS=$'\n\t' IFS=$'\n\t'
readonly cluster_name="kind-smoke-test-postgres-operator" readonly cluster_name="postgres-operator-e2e-tests"
readonly operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1)
readonly e2e_test_image=${cluster_name}
readonly kind_api_server_port=6443 # well-known in the 'kind' codebase
readonly kubeconfig_path="./e2e/kind-config-${cluster_name}"
# avoid interference with previous test runs # avoid interference with previous test runs
if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]] if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]]
@ -14,20 +18,19 @@ then
kind delete cluster --name ${cluster_name} kind delete cluster --name ${cluster_name}
fi fi
kind create cluster --name ${cluster_name} --config ./e2e/kind-config-smoke-tests.yaml kind create cluster --name ${cluster_name} --config ./e2e/kind-cluster-postgres-operator-e2e-tests.yaml
export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" export KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})"
kind load docker-image ${operator_image} --name ${cluster_name}
kubectl cluster-info kubectl cluster-info
image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) # use the actual kubeconfig to connect to the 'kind' API server
kind load docker-image ${image} --name ${cluster_name} # but update the IP address of the API server to the one from the Docker 'bridge' network
cp -r ./manifests ./e2e/manifests
cp $KUBECONFIG ./e2e cp $KUBECONFIG ./e2e
d=$(docker inspect -f "{{ .NetworkSettings.IPAddress }}:6443" ${cluster_name}-control-plane) kind_api_server=$(docker inspect --format "{{ .NetworkSettings.IPAddress }}:${kind_api_server_port}" ${cluster_name}-control-plane)
sed -i "s/server.*$/server: https:\/\/$d/g" e2e/kind-config-${cluster_name} sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" ${kubeconfig_path}
docker build --tag=postgres-operator-e2e-tests -f e2e/Dockerfile . && docker run postgres-operator-e2e-tests docker run --rm --mount type=bind,source="$(realpath ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE=${operator_image} ${e2e_test_image}
#python3 -m unittest discover --start-directory e2e/tests/ -v &&
kind delete cluster --name ${cluster_name} kind delete cluster --name ${cluster_name}
rm -rf ./e2e/manifests ./e2e/kind-smoke-test-postgres-operator.yaml rm -rf ${kubeconfig_path}

12
e2e/tests/test_smoke.py
View File

@ -3,8 +3,7 @@ import time
import timeout_decorator import timeout_decorator
import subprocess import subprocess
import warnings import warnings
import docker import os
#from halo import Halo
from kubernetes import client, config, utils from kubernetes import client, config, utils
@ -40,10 +39,7 @@ class SmokeTestCase(unittest.TestCase):
for filename in ["configmap.yaml", "postgres-operator.yaml"]: for filename in ["configmap.yaml", "postgres-operator.yaml"]:
utils.create_from_yaml(k8s_api.k8s_client, "manifests/" + filename) utils.create_from_yaml(k8s_api.k8s_client, "manifests/" + filename)
# submit the most recent operator image built locally # submit the most recent operator image built on the Docker host
# HACK assumes "images list" returns the most recent image at index 0
# docker_client = docker.from_env()
image = "registry.opensource.zalan.do/acid/postgres-operator:v1.1.0-67-g52e53f1-dirty" # docker_client.images.list(name="registry.opensource.zalan.do/acid/postgres-operator")[0].tags[0]
body = { body = {
"spec": { "spec": {
"template": { "template": {
@ -51,7 +47,7 @@ class SmokeTestCase(unittest.TestCase):
"containers": [ "containers": [
{ {
"name": "postgres-operator", "name": "postgres-operator",
"image": image "image": os.environ['OPERATOR_IMAGE']
} }
] ]
} }
@ -114,7 +110,6 @@ class Utils:
@staticmethod @staticmethod
def wait_for_pod_start(k8s_api, pod_labels, retry_timeout_sec): def wait_for_pod_start(k8s_api, pod_labels, retry_timeout_sec):
pod_phase = 'No pod running' pod_phase = 'No pod running'
#with Halo(text="Wait for the pod '{}' to start. Pod phase: {}".format(pod_labels, pod_phase), spinner='dots'):
while pod_phase != 'Running': while pod_phase != 'Running':
pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items pods = k8s_api.core_v1.list_namespaced_pod('default', label_selector=pod_labels).items
if pods: if pods:
@ -133,7 +128,6 @@ class Utils:
"v1", "default", "postgresqls", "acid-minimal-cluster", body) "v1", "default", "postgresqls", "acid-minimal-cluster", body)
labels = 'version=acid-minimal-cluster' labels = 'version=acid-minimal-cluster'
#with Halo(text="Waiting for the cluster to scale to {} pods.".format(number_of_instances), spinner='dots'):
while Utils.count_pods_with_label(k8s_api, labels) != number_of_instances: while Utils.count_pods_with_label(k8s_api, labels) != number_of_instances:
time.sleep(retry_timeout_sec) time.sleep(retry_timeout_sec)