diff --git a/.vib/spark/goss/goss.yaml b/.vib/spark/goss/goss.yaml new file mode 100644 index 000000000000..a1d2d7f4df21 --- /dev/null +++ b/.vib/spark/goss/goss.yaml @@ -0,0 +1,12 @@ +gossfile: + # Goss tests exclusive to the current container + ../../spark/goss/spark.yaml: {} + # Load scripts from .vib/common/goss/templates + ../../common/goss/templates/check-app-version.yaml: {} + ../../common/goss/templates/check-binaries.yaml: {} + ../../common/goss/templates/check-broken-symlinks.yaml: {} + ../../common/goss/templates/check-ca-certs.yaml: {} + ../../common/goss/templates/check-directories.yaml: {} + ../../common/goss/templates/check-linked-libraries.yaml: {} + ../../common/goss/templates/check-sed-in-place.yaml: {} + ../../common/goss/templates/check-spdx.yaml: {} diff --git a/.vib/spark/goss/spark.yaml b/.vib/spark/goss/spark.yaml new file mode 100644 index 000000000000..fbdacbbaa5d8 --- /dev/null +++ b/.vib/spark/goss/spark.yaml @@ -0,0 +1,6 @@ +command: + {{- range $module:= .Vars.python.modules }} + check-python-{{ $module }}: + exec: /opt/bitnami/spark/venv/bin/python -c 'import {{ $module }}' + exit-status: 0 + {{- end }} diff --git a/.vib/spark/goss/vars.yaml b/.vib/spark/goss/vars.yaml new file mode 100644 index 000000000000..5180621ed073 --- /dev/null +++ b/.vib/spark/goss/vars.yaml @@ -0,0 +1,32 @@ +binaries: + - python + - spark-shell + - spark-submit + - java +directories: + - mode: "0775" + group: "root" + paths: + - /opt/bitnami/spark/tmp + - /opt/bitnami/spark/logs + - /opt/bitnami/spark/conf + - /opt/bitnami/spark/work + - /opt/bitnami/spark/jars +root_dir: /opt/bitnami +version: + bin_name: spark-shell --version + flag: --version +linked_libraries: + exclude_paths: + # Exclude specific Pip modules from paths because they link to non-existent libraries that are not expected to exist + - .*\/lib\/python[0-9.]+\/site-packages\/numpy.libs\/.* + - .*\/lib\/python[0-9.]+\/site-packages\/psycopg2_binary.libs\/.* +python: + modules: + - pyspark + - awscli + - boto3 + - botocore + - psycopg2 + - pandas + - py4j diff --git a/.vib/spark/vib-publish.json b/.vib/spark/vib-publish.json index 8b6fe8eab4e6..6663d2c6ec5d 100644 --- a/.vib/spark/vib-publish.json +++ b/.vib/spark/vib-publish.json @@ -3,7 +3,8 @@ "resources": { "url": "{VIB_ENV_CONTAINER_URL}", "path": "{VIB_ENV_PATH}" - } + }, + "runtime_parameters": "Y29tbWFuZDogWyJ0YWlsIiwgIi1mIiwgIi9kZXYvbnVsbCJd" }, "phases": { "package": { @@ -33,6 +34,21 @@ }, "verify": { "actions": [ + { + "action_id": "goss", + "params": { + "resources": { + "path": "/.vib" + }, + "tests_file": "spark/goss/goss.yaml", + "vars_file": "spark/goss/vars.yaml", + "remote": { + "pod": { + "workload": "deploy-spark" + } + } + } + }, { "action_id": "trivy", "params": { @@ -77,9 +93,9 @@ "url": "{VIB_ENV_PACKAGES_JSON_URL}", "path": "/{VIB_ENV_PATH}", "authn": { - "header": "Authorization", - "token": "Bearer {VIB_ENV_GITHUB_TOKEN}" - } + "header": "Authorization", + "token": "Bearer {VIB_ENV_GITHUB_TOKEN}" + } } } } diff --git a/.vib/spark/vib-verify.json b/.vib/spark/vib-verify.json index a34e8aee9161..bf7a925ac432 100644 --- a/.vib/spark/vib-verify.json +++ b/.vib/spark/vib-verify.json @@ -3,7 +3,8 @@ "resources": { "url": "{SHA_ARCHIVE}", "path": "{VIB_ENV_PATH}" - } + }, + "runtime_parameters": "Y29tbWFuZDogWyJ0YWlsIiwgIi1mIiwgIi9kZXYvbnVsbCJd" }, "phases": { "package": { @@ -29,6 +30,21 @@ }, "verify": { "actions": [ + { + "action_id": "goss", + "params": { + "resources": { + "path": "/.vib" + }, + "tests_file": "spark/goss/goss.yaml", + "vars_file": "spark/goss/vars.yaml", + "remote": { + "pod": { + "workload": "deploy-spark" + } + } + } + }, { "action_id": "trivy", "params": {