From cfb1db599b55a01c34efae50737b7879309665fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Su=C5=A1nik?= Date: Sat, 29 Oct 2022 14:16:21 +0200 Subject: [PATCH 01/20] Don't lock for NodeGetVolumeStats RPC --- src/utils/general.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/general.js b/src/utils/general.js index 10d9b3c..8dca9e0 100644 --- a/src/utils/general.js +++ b/src/utils/general.js @@ -87,10 +87,10 @@ function lockKeysFromRequest(call, serviceMethodName) { case "NodeUnstageVolume": case "NodePublishVolume": case "NodeUnpublishVolume": - case "NodeGetVolumeStats": case "NodeExpandVolume": return ["volume_id_" + call.request.volume_id]; + case "NodeGetVolumeStats": default: return []; } From 9d2943b62dec97084d4c88dc768daee7d472fe6f Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Tue, 20 Dec 2022 22:48:36 -0700 Subject: [PATCH 02/20] initial support for nvmeof Signed-off-by: Travis Glenn Hansen --- .github/workflows/main.yml | 1 + ci/configs/zfs-generic/nvmeof.yaml | 30 + package-lock.json | 874 ++++++++---------- package.json | 6 +- src/driver/controller-zfs-generic/index.js | 561 ++++++++++- src/driver/controller-zfs-local/index.js | 3 +- src/driver/factory.js | 1 + src/driver/freenas/ssh.js | 2 +- src/driver/index.js | 256 ++++- src/driver/node-manual/index.js | 1 + .../zfs-local-ephemeral-inline/index.js | 2 +- src/utils/filesystem.js | 18 +- src/utils/general.js | 12 + src/utils/nvmeof.js | 321 +++++++ .../zfs_local_exec_client.js} | 0 src/utils/{ssh.js => zfs_ssh_exec_client.js} | 0 16 files changed, 1555 insertions(+), 533 deletions(-) create mode 100644 ci/configs/zfs-generic/nvmeof.yaml create mode 100644 src/utils/nvmeof.js rename src/{driver/controller-zfs-local/exec.js => utils/zfs_local_exec_client.js} (100%) rename src/utils/{ssh.js => zfs_ssh_exec_client.js} (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ea7c393..27b0e55 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -256,6 +256,7 @@ jobs: - zfs-generic/iscsi.yaml - zfs-generic/nfs.yaml - zfs-generic/smb.yaml + - zfs-generic/nvmeof.yaml runs-on: - self-hosted - Linux diff --git a/ci/configs/zfs-generic/nvmeof.yaml b/ci/configs/zfs-generic/nvmeof.yaml new file mode 100644 index 0000000..820ee79 --- /dev/null +++ b/ci/configs/zfs-generic/nvmeof.yaml @@ -0,0 +1,30 @@ +driver: zfs-generic-nvmeof + +sshConnection: + host: ${SERVER_HOST} + port: 22 + username: ${SERVER_USERNAME} + password: ${SERVER_PASSWORD} + +zfs: + datasetParentName: tank/ci/${CI_BUILD_KEY}/v + detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s + + zvolCompression: + zvolDedup: + zvolEnableReservation: false + zvolBlocksize: + +nvmeof: + transports: + - "tcp://${SERVER_HOST}:4420" + namePrefix: "csi-ci-${CI_BUILD_KEY}-" + nameSuffix: "" + shareStrategy: "nvmetCli" + shareStrategyNvmetCli: + basename: "nqn.2003-01.org.linux-nvmeof.ubuntu-19.x8664" + ports: + - "1" + subsystem: + attributes: + allow_any_host: 1 diff --git a/package-lock.json b/package-lock.json index a901bb3..6e95f6e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,21 +1,21 @@ { "name": "democratic-csi", - "version": "1.7.7", + "version": "1.8.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "democratic-csi", - "version": "1.7.7", + "version": "1.8.0", "license": "MIT", "dependencies": { "@grpc/grpc-js": "^1.5.7", "@grpc/proto-loader": "^0.7.0", - "@kubernetes/client-node": "^0.17.0", + "@kubernetes/client-node": "^0.18.0", "async-mutex": "^0.4.0", "axios": "^1.1.3", "bunyan": "^1.8.15", - "fs-extra": "^10.1.0", + "fs-extra": "^11.1.0", "handlebars": "^4.7.7", "js-yaml": "^4.0.0", "lodash": "^4.17.21", @@ -51,15 +51,15 @@ } }, "node_modules/@eslint/eslintrc": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.3.tgz", - "integrity": "sha512-uj3pT6Mg+3t39fvLrj8iuCIJ38zKO9FpGtJ4BBJebJhEwjoT+KLVNCcHT5QC9NGRIEi7fZ0ZR8YRb884auB4Lg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.4.0.tgz", + "integrity": "sha512-7yfvXy6MWLgWSFsLhz5yH3iQ52St8cdUY6FoGieKkRDVxuxmrNuUetIuu6cmjNWwniUHiWXjxCr5tTXDrbYS5A==", "dev": true, "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", "espree": "^9.4.0", - "globals": "^13.15.0", + "globals": "^13.19.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", "js-yaml": "^4.1.0", @@ -74,9 +74,9 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.7.2.tgz", - "integrity": "sha512-MqqbVynbe3VUSnApFW/dpkDaa9T1ASqRnMWeSPGFO/Ro98R7XUDLacfeBa7RaSI1iFu9GYk5gBKARf0zipFe4w==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.0.tgz", + "integrity": "sha512-ySMTXQuMvvswoobvN+0LsaPf7ITO2JVfJmHxQKI4cGehNrrUms+n81BlHEX7Hl/LExji6XE3fnI9U04GSkRruA==", "dependencies": { "@grpc/proto-loader": "^0.7.0", "@types/node": ">=12.12.47" @@ -86,9 +86,9 @@ } }, "node_modules/@grpc/proto-loader": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.3.tgz", - "integrity": "sha512-5dAvoZwna2Py3Ef96Ux9jIkp3iZ62TUsV00p3wVBPNX5K178UbNi8Q7gQVqwXT1Yq9RejIGG9G2IPEo93T6RcA==", + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.4.tgz", + "integrity": "sha512-MnWjkGwqQ3W8fx94/c1CwqLsNmHHv2t0CFn+9++6+cDphC1lolpg9M2OU0iebIjK//pBNX9e94ho+gjx6vz39w==", "dependencies": { "@types/long": "^4.0.1", "lodash.camelcase": "^4.3.0", @@ -139,14 +139,14 @@ } }, "node_modules/@humanwhocodes/config-array": { - "version": "0.10.7", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.10.7.tgz", - "integrity": "sha512-MDl6D6sBsaV452/QSdX+4CXIjZhIcI0PELsxUjk4U828yd58vk3bTIvk/6w5FY+4hIy9sLW0sfrV7K7Kc++j/w==", + "version": "0.11.8", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", + "integrity": "sha512-UybHIJzJnR5Qc/MsD9Kr+RpO2h+/P1GhOwdiLPXK5TWk5sgTdu88bTD9UP+CKbPPh5Rni1u0GjAdYQLemG8g+g==", "dev": true, "dependencies": { "@humanwhocodes/object-schema": "^1.2.1", "debug": "^4.1.1", - "minimatch": "^3.0.4" + "minimatch": "^3.0.5" }, "engines": { "node": ">=10.10.0" @@ -172,10 +172,14 @@ "dev": true }, "node_modules/@kubernetes/client-node": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-0.17.1.tgz", - "integrity": "sha512-qXANjukuTq/drb1hq1NCYZafpdRTvbyTzbliWO6RwW7eEb2b9qwINbw0DiVHpBQg3e9DeQd8+brI1sR1Fck5kQ==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-0.18.0.tgz", + "integrity": "sha512-Mp6q0OkZQBp+HslIgvHYpsPJk8z6mch231QWtIZQHvs+PaTE6mkUfusYE8fNw3jMjru5mVO/JDz6PTjB9YT2rQ==", "dependencies": { + "@types/js-yaml": "^4.0.1", + "@types/node": "^10.12.0", + "@types/request": "^2.47.1", + "@types/ws": "^6.0.1", "byline": "^5.0.0", "execa": "5.0.0", "isomorphic-ws": "^4.0.1", @@ -187,14 +191,19 @@ "stream-buffers": "^3.0.2", "tar": "^6.1.11", "tmp-promise": "^3.0.2", - "tslib": "^1.9.3", - "underscore": "^1.9.1", + "tslib": "^2.4.1", + "underscore": "^1.13.6", "ws": "^7.3.1" }, "optionalDependencies": { - "openid-client": "^5.1.6" + "openid-client": "^5.3.0" } }, + "node_modules/@kubernetes/client-node/node_modules/@types/node": { + "version": "10.17.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", + "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -284,20 +293,67 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, + "node_modules/@types/caseless": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", + "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" + }, + "node_modules/@types/js-yaml": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz", + "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==" + }, "node_modules/@types/long": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" }, "node_modules/@types/node": { - "version": "18.11.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.0.tgz", - "integrity": "sha512-IOXCvVRToe7e0ny7HpT/X9Rb2RYtElG1a+VshjwT00HxrM2dWBApHQoqsI6WiY7Q03vdf2bCrIGzVrkF/5t10w==" + "version": "18.11.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.17.tgz", + "integrity": "sha512-HJSUJmni4BeDHhfzn6nF0sVmd1SMezP7/4F0Lq+aXzmp2xm9O7WXrUtHW/CHlYVtZUbByEvWidHqRtcJXGF2Ng==" + }, + "node_modules/@types/request": { + "version": "2.48.8", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.8.tgz", + "integrity": "sha512-whjk1EDJPcAR2kYHRbFl/lKeeKYTi05A15K9bnLInCVroNDCtXce57xKdI0/rQaA3K+6q0eFyUBPmqfSndUZdQ==", + "dependencies": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.0" + } + }, + "node_modules/@types/request/node_modules/form-data": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, + "node_modules/@types/tough-cookie": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", + "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==" + }, + "node_modules/@types/ws": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-6.0.4.tgz", + "integrity": "sha512-PpPrX7SZW9re6+Ha8ojZG4Se8AZXgf0GK6zmfqEuCsY49LFDNXO3SByp44X3dFEqtB73lkCDAdUazhAjVPiNwg==", + "dependencies": { + "@types/node": "*" + } }, "node_modules/acorn": { - "version": "8.8.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", - "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==", + "version": "8.8.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", + "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", "dev": true, "bin": { "acorn": "bin/acorn" @@ -357,15 +413,6 @@ "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, - "node_modules/array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", @@ -395,11 +442,6 @@ "tslib": "^2.4.0" } }, - "node_modules/async-mutex/node_modules/tslib": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", - "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -419,9 +461,9 @@ "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, "node_modules/axios": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.1.3.tgz", - "integrity": "sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.2.1.tgz", + "integrity": "sha512-I88cFiGu9ryt/tfVEi4kX2SITsvDddTajXTOFmt2uK1ZVA8LytjtdeyefdQWEf5PU8w+4SSJDoYnggflB5tW4A==", "dependencies": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", @@ -450,18 +492,6 @@ "concat-map": "0.0.1" } }, - "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, - "dependencies": { - "fill-range": "^7.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/buildcheck": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.3.tgz", @@ -709,18 +739,6 @@ "node": ">=0.4.0" } }, - "node_modules/dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "dev": true, - "dependencies": { - "path-type": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -786,14 +804,15 @@ } }, "node_modules/eslint": { - "version": "8.25.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.25.0.tgz", - "integrity": "sha512-DVlJOZ4Pn50zcKW5bYH7GQK/9MsoQG2d5eDH0ebEkE8PbgzTTmtt/VTH9GGJ4BfeZCpBLqFfvsjX35UacUL83A==", + "version": "8.30.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.30.0.tgz", + "integrity": "sha512-MGADB39QqYuzEGov+F/qb18r4i7DohCDOfatHaxI2iGlPuC65bwG2gxgO+7DkyL38dRFaRH7RaRAgU6JKL9rMQ==", "dev": true, "dependencies": { - "@eslint/eslintrc": "^1.3.3", - "@humanwhocodes/config-array": "^0.10.5", + "@eslint/eslintrc": "^1.4.0", + "@humanwhocodes/config-array": "^0.11.8", "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -809,14 +828,14 @@ "fast-deep-equal": "^3.1.3", "file-entry-cache": "^6.0.1", "find-up": "^5.0.0", - "glob-parent": "^6.0.1", - "globals": "^13.15.0", - "globby": "^11.1.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", "grapheme-splitter": "^1.0.4", "ignore": "^5.2.0", "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", "js-sdsl": "^4.1.4", "js-yaml": "^4.1.0", "json-stable-stringify-without-jsonify": "^1.0.1", @@ -890,9 +909,9 @@ } }, "node_modules/espree": { - "version": "9.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.0.tgz", - "integrity": "sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw==", + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz", + "integrity": "sha512-XwctdmTO6SIvCzd9810yyNzIrOrqNYV9Koizx4C/mRhf9uq0o4yHoCEU/670pOxOL/MSraektvSAji79kX90Vg==", "dev": true, "dependencies": { "acorn": "^8.8.0", @@ -996,34 +1015,6 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, - "node_modules/fast-glob": { - "version": "3.2.12", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", - "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", - "dev": true, - "dependencies": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - }, - "engines": { - "node": ">=8.6.0" - } - }, - "node_modules/fast-glob/node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -1036,9 +1027,9 @@ "dev": true }, "node_modules/fastq": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", - "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.14.0.tgz", + "integrity": "sha512-eR2D+V9/ExcbF9ls441yIuN6TI2ED1Y2ZcA5BmMtJsOkWOFRJQ0Jt0g1UwqXJJVAb+V+umH5Dfr8oh4EVP7VVg==", "dev": true, "dependencies": { "reusify": "^1.0.4" @@ -1061,18 +1052,6 @@ "node": "^10.12.0 || >=12.0.0" } }, - "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -1154,16 +1133,16 @@ } }, "node_modules/fs-extra": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", - "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.0.tgz", + "integrity": "sha512-0rcTq621PD5jM/e0a3EJoGC/1TC5ZBCERW82LQuwfGnCa1V8w7dpYH1yNu+SLb6E5dkeCBzKEyLGlFrnr+dUyw==", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" }, "engines": { - "node": ">=12" + "node": ">=14.14" } }, "node_modules/fs-minipass": { @@ -1177,6 +1156,17 @@ "node": ">= 8" } }, + "node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1246,9 +1236,9 @@ } }, "node_modules/globals": { - "version": "13.17.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.17.0.tgz", - "integrity": "sha512-1C+6nQRb1GwGMKm2dH/E7enFAMxGTmGI7/dEdhy/DNelv85w9B72t3uc5frtMNXIbzrarJJ/lTCjcaZwbLJmyw==", + "version": "13.19.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.19.0.tgz", + "integrity": "sha512-dkQ957uSRWHw7CFXLUtUHQI3g3aWApYhfNR2O6jn/907riyTYKVBmxYVROkBcY614FSSeSJh7Xm7SrUWCxvJMQ==", "dev": true, "dependencies": { "type-fest": "^0.20.2" @@ -1260,26 +1250,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "dependencies": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -1375,9 +1345,9 @@ } }, "node_modules/ignore": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", - "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", + "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", "dev": true, "engines": { "node": ">= 4" @@ -1436,9 +1406,9 @@ "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" }, "node_modules/is-core-module": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.10.0.tgz", - "integrity": "sha512-Erxj2n/LDAZ7H8WNJXd9tw38GYM3dv8rk8Zcs+jJuxYTW7sozH+SS8NtrSjVL1/vpLvWi1hxy96IzjJ3EHTJJg==", + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz", + "integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==", "dependencies": { "has": "^1.0.3" }, @@ -1475,13 +1445,13 @@ "node": ">=0.10.0" } }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", "dev": true, "engines": { - "node": ">=0.12.0" + "node": ">=8" } }, "node_modules/is-stream": { @@ -1519,19 +1489,23 @@ "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, "node_modules/jose": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/jose/-/jose-4.10.0.tgz", - "integrity": "sha512-KEhB/eLGLomWGPTb+/RNbYsTjIyx03JmbqAyIyiXBuNSa7CmNrJd5ysFhblayzs/e/vbOPMUaLnjHUMhGp4yLw==", + "version": "4.11.1", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.11.1.tgz", + "integrity": "sha512-YRv4Tk/Wlug8qicwqFNFVEZSdbROCHRAC6qu/i0dyNKr5JQdoa2pIGoS04lLO/jXQX7Z9omoNewYIVIxqZBd9Q==", "optional": true, "funding": { "url": "https://github.com/sponsors/panva" } }, "node_modules/js-sdsl": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.1.5.tgz", - "integrity": "sha512-08bOAKweV2NUC1wqTtf3qZlnpOX/R2DU9ikpjOHs0H+ibQv3zpncVQg6um4uYtRtrwIX8M4Nh3ytK4HGlYAq7Q==", - "dev": true + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.2.0.tgz", + "integrity": "sha512-dyBIzQBDkCqCu+0upx25Y2jGdbTGxE9fshMsCdK0ViOongpV+n5tXRcZY9v7CaVQ79AGS9KA1KHtojxiM7aXSQ==", + "dev": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } }, "node_modules/js-yaml": { "version": "4.1.0", @@ -1670,9 +1644,9 @@ "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" }, "node_modules/lru-cache": { - "version": "7.14.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.14.0.tgz", - "integrity": "sha512-EIRtP1GrSJny0dqb50QXRUNBxHJhcpxHC++M5tD7RYbvLLn5KVWKsbyswSSqDuU15UFi3bgTQIY8nhDMeF6aDQ==", + "version": "7.14.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.14.1.tgz", + "integrity": "sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA==", "engines": { "node": ">=12" } @@ -1682,28 +1656,6 @@ "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" }, - "node_modules/merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, - "engines": { - "node": ">= 8" - } - }, - "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", - "dev": true, - "dependencies": { - "braces": "^3.0.2", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -1751,9 +1703,9 @@ } }, "node_modules/minipass": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.4.tgz", - "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.0.0.tgz", + "integrity": "sha512-g2Uuh2jEKoht+zvO6vJqXmYpflPqzRBT+Th2h01DKh5z7wbY/AZ2gCQ78cP70YoHPyFdY30YBV5WxgLOEwOykw==", "dependencies": { "yallist": "^4.0.0" }, @@ -1773,6 +1725,17 @@ "node": ">= 8" } }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/mkdirp": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", @@ -1940,12 +1903,12 @@ } }, "node_modules/openid-client": { - "version": "5.1.10", - "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.1.10.tgz", - "integrity": "sha512-KYAtkxTuUwTvjAmH0QMFFP3i9l0+XhP2/blct6Q9kn+DUJ/lu8/g/bI8ghSgxz9dJLm/9cpB/1uLVGTcGGY0hw==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.3.1.tgz", + "integrity": "sha512-RLfehQiHch9N6tRWNx68cicf3b1WR0x74bJWHRc25uYIbSRwjxYcTFaRnzbbpls5jroLAaB/bFIodTgA5LJMvw==", "optional": true, "dependencies": { - "jose": "^4.1.4", + "jose": "^4.10.0", "lru-cache": "^6.0.0", "object-hash": "^2.0.1", "oidc-token-hash": "^5.0.1" @@ -2055,32 +2018,11 @@ "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, - "node_modules/path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, - "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -2106,11 +2048,11 @@ } }, "node_modules/prompt/node_modules/winston": { - "version": "2.4.6", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.6.tgz", - "integrity": "sha512-J5Zu4p0tojLde8mIOyDSsmLmcP8I3Z6wtwpTDHx1+hGcdhxcJaAmG4CFtagkb+NiN1M9Ek4b42pzMWqfc9jm8w==", + "version": "2.4.7", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.7.tgz", + "integrity": "sha512-vLB4BqzCKDnnZH9PHGoS2ycawueX4HLqENXQitvFHczhgW2vFpSOn31LZtVr1KU8YTw7DS4tM+cqyovxo8taVg==", "dependencies": { - "async": "^3.2.3", + "async": "^2.6.4", "colors": "1.0.x", "cycle": "1.0.x", "eyes": "0.1.x", @@ -2121,6 +2063,14 @@ "node": ">= 0.10.0" } }, + "node_modules/prompt/node_modules/winston/node_modules/async": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", + "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", + "dependencies": { + "lodash": "^4.17.14" + } + }, "node_modules/protobufjs": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.1.2.tgz", @@ -2145,9 +2095,9 @@ } }, "node_modules/protobufjs/node_modules/long": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/long/-/long-5.2.0.tgz", - "integrity": "sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.1.tgz", + "integrity": "sha512-GKSNGeNAtw8IryjjkhZxuKB3JzlcLTwjtiQCHKvqQet81I93kXslhDQruGI/QsddO83mcDToBVy7GqGS/zYf/A==" }, "node_modules/proxy-from-env": { "version": "1.1.0", @@ -2414,9 +2364,9 @@ "optional": true }, "node_modules/safe-stable-stringify": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.0.tgz", - "integrity": "sha512-eehKHKpab6E741ud7ZIMcXhKcP6TSIezPkNZhy5U8xC6+VvrRdUA2tMgxGxaGl4cz7c2Ew5+mg5+wNB16KQqrA==", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.1.tgz", + "integrity": "sha512-dVHE6bMtS/bnL2mwualjc6IxEv1F+OCUpA46pKUj6F8uDbUM0jCCulPqRNPSnWwGNKx5etqMjZYdXtrm5KJZGA==", "engines": { "node": ">=10" } @@ -2499,15 +2449,6 @@ "is-arrayish": "^0.3.1" } }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -2649,19 +2590,19 @@ } }, "node_modules/tar": { - "version": "6.1.11", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz", - "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", + "version": "6.1.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.13.tgz", + "integrity": "sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==", "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", - "minipass": "^3.0.0", + "minipass": "^4.0.0", "minizlib": "^2.1.1", "mkdirp": "^1.0.3", "yallist": "^4.0.0" }, "engines": { - "node": ">= 10" + "node": ">=10" } }, "node_modules/tar/node_modules/mkdirp": { @@ -2705,18 +2646,6 @@ "tmp": "^0.2.0" } }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, "node_modules/tough-cookie": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", @@ -2735,9 +2664,9 @@ "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" }, "node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", + "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" }, "node_modules/tunnel-agent": { "version": "0.6.0", @@ -2780,9 +2709,9 @@ } }, "node_modules/uglify-js": { - "version": "3.17.3", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.17.3.tgz", - "integrity": "sha512-JmMFDME3iufZnBpyKL+uS78LRiC+mK55zWfM5f/pWBJfpOttXAqYfdDGRukYhJuyRinvPVAtUhvy7rlDybNtFg==", + "version": "3.17.4", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.17.4.tgz", + "integrity": "sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==", "optional": true, "bin": { "uglifyjs": "bin/uglifyjs" @@ -2955,9 +2884,9 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/yargs": { - "version": "17.6.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.0.tgz", - "integrity": "sha512-8H/wTDqlSwoSnScvV2N/JHfLWOKuh5MVla9hqLjK3nsfyy6Y4kDSYSvkU5YCUEPOSnRXfIyx3Sq+B/IWudTo4g==", + "version": "17.6.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", + "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -2965,7 +2894,7 @@ "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", - "yargs-parser": "^21.0.0" + "yargs-parser": "^21.1.1" }, "engines": { "node": ">=12" @@ -3009,15 +2938,15 @@ } }, "@eslint/eslintrc": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.3.tgz", - "integrity": "sha512-uj3pT6Mg+3t39fvLrj8iuCIJ38zKO9FpGtJ4BBJebJhEwjoT+KLVNCcHT5QC9NGRIEi7fZ0ZR8YRb884auB4Lg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.4.0.tgz", + "integrity": "sha512-7yfvXy6MWLgWSFsLhz5yH3iQ52St8cdUY6FoGieKkRDVxuxmrNuUetIuu6cmjNWwniUHiWXjxCr5tTXDrbYS5A==", "dev": true, "requires": { "ajv": "^6.12.4", "debug": "^4.3.2", "espree": "^9.4.0", - "globals": "^13.15.0", + "globals": "^13.19.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", "js-yaml": "^4.1.0", @@ -3026,18 +2955,18 @@ } }, "@grpc/grpc-js": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.7.2.tgz", - "integrity": "sha512-MqqbVynbe3VUSnApFW/dpkDaa9T1ASqRnMWeSPGFO/Ro98R7XUDLacfeBa7RaSI1iFu9GYk5gBKARf0zipFe4w==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.0.tgz", + "integrity": "sha512-ySMTXQuMvvswoobvN+0LsaPf7ITO2JVfJmHxQKI4cGehNrrUms+n81BlHEX7Hl/LExji6XE3fnI9U04GSkRruA==", "requires": { "@grpc/proto-loader": "^0.7.0", "@types/node": ">=12.12.47" } }, "@grpc/proto-loader": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.3.tgz", - "integrity": "sha512-5dAvoZwna2Py3Ef96Ux9jIkp3iZ62TUsV00p3wVBPNX5K178UbNi8Q7gQVqwXT1Yq9RejIGG9G2IPEo93T6RcA==", + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.4.tgz", + "integrity": "sha512-MnWjkGwqQ3W8fx94/c1CwqLsNmHHv2t0CFn+9++6+cDphC1lolpg9M2OU0iebIjK//pBNX9e94ho+gjx6vz39w==", "requires": { "@types/long": "^4.0.1", "lodash.camelcase": "^4.3.0", @@ -3078,14 +3007,14 @@ } }, "@humanwhocodes/config-array": { - "version": "0.10.7", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.10.7.tgz", - "integrity": "sha512-MDl6D6sBsaV452/QSdX+4CXIjZhIcI0PELsxUjk4U828yd58vk3bTIvk/6w5FY+4hIy9sLW0sfrV7K7Kc++j/w==", + "version": "0.11.8", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", + "integrity": "sha512-UybHIJzJnR5Qc/MsD9Kr+RpO2h+/P1GhOwdiLPXK5TWk5sgTdu88bTD9UP+CKbPPh5Rni1u0GjAdYQLemG8g+g==", "dev": true, "requires": { "@humanwhocodes/object-schema": "^1.2.1", "debug": "^4.1.1", - "minimatch": "^3.0.4" + "minimatch": "^3.0.5" } }, "@humanwhocodes/module-importer": { @@ -3101,25 +3030,36 @@ "dev": true }, "@kubernetes/client-node": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-0.17.1.tgz", - "integrity": "sha512-qXANjukuTq/drb1hq1NCYZafpdRTvbyTzbliWO6RwW7eEb2b9qwINbw0DiVHpBQg3e9DeQd8+brI1sR1Fck5kQ==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-0.18.0.tgz", + "integrity": "sha512-Mp6q0OkZQBp+HslIgvHYpsPJk8z6mch231QWtIZQHvs+PaTE6mkUfusYE8fNw3jMjru5mVO/JDz6PTjB9YT2rQ==", "requires": { + "@types/js-yaml": "^4.0.1", + "@types/node": "^10.12.0", + "@types/request": "^2.47.1", + "@types/ws": "^6.0.1", "byline": "^5.0.0", "execa": "5.0.0", "isomorphic-ws": "^4.0.1", "js-yaml": "^4.1.0", "jsonpath-plus": "^0.19.0", - "openid-client": "^5.1.6", + "openid-client": "^5.3.0", "request": "^2.88.0", "rfc4648": "^1.3.0", "shelljs": "^0.8.5", "stream-buffers": "^3.0.2", "tar": "^6.1.11", "tmp-promise": "^3.0.2", - "tslib": "^1.9.3", - "underscore": "^1.9.1", + "tslib": "^2.4.1", + "underscore": "^1.13.6", "ws": "^7.3.1" + }, + "dependencies": { + "@types/node": { + "version": "10.17.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", + "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" + } } }, "@nodelib/fs.scandir": { @@ -3202,20 +3142,66 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, + "@types/caseless": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", + "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" + }, + "@types/js-yaml": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz", + "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==" + }, "@types/long": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" }, "@types/node": { - "version": "18.11.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.0.tgz", - "integrity": "sha512-IOXCvVRToe7e0ny7HpT/X9Rb2RYtElG1a+VshjwT00HxrM2dWBApHQoqsI6WiY7Q03vdf2bCrIGzVrkF/5t10w==" + "version": "18.11.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.17.tgz", + "integrity": "sha512-HJSUJmni4BeDHhfzn6nF0sVmd1SMezP7/4F0Lq+aXzmp2xm9O7WXrUtHW/CHlYVtZUbByEvWidHqRtcJXGF2Ng==" + }, + "@types/request": { + "version": "2.48.8", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.8.tgz", + "integrity": "sha512-whjk1EDJPcAR2kYHRbFl/lKeeKYTi05A15K9bnLInCVroNDCtXce57xKdI0/rQaA3K+6q0eFyUBPmqfSndUZdQ==", + "requires": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.0" + }, + "dependencies": { + "form-data": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + } + } + }, + "@types/tough-cookie": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", + "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==" + }, + "@types/ws": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-6.0.4.tgz", + "integrity": "sha512-PpPrX7SZW9re6+Ha8ojZG4Se8AZXgf0GK6zmfqEuCsY49LFDNXO3SByp44X3dFEqtB73lkCDAdUazhAjVPiNwg==", + "requires": { + "@types/node": "*" + } }, "acorn": { - "version": "8.8.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", - "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==", + "version": "8.8.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", + "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", "dev": true }, "acorn-jsx": { @@ -3254,12 +3240,6 @@ "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, - "array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true - }, "asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", @@ -3284,13 +3264,6 @@ "integrity": "sha512-eJFZ1YhRR8UN8eBLoNzcDPcy/jqjsg6I1AP+KvWQX80BqOSW1oJPJXDylPUEeMr2ZQvHgnQ//Lp6f3RQ1zI7HA==", "requires": { "tslib": "^2.4.0" - }, - "dependencies": { - "tslib": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", - "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" - } } }, "asynckit": { @@ -3309,9 +3282,9 @@ "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, "axios": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.1.3.tgz", - "integrity": "sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.2.1.tgz", + "integrity": "sha512-I88cFiGu9ryt/tfVEi4kX2SITsvDddTajXTOFmt2uK1ZVA8LytjtdeyefdQWEf5PU8w+4SSJDoYnggflB5tW4A==", "requires": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", @@ -3340,15 +3313,6 @@ "concat-map": "0.0.1" } }, - "braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, - "requires": { - "fill-range": "^7.0.1" - } - }, "buildcheck": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.3.tgz", @@ -3538,15 +3502,6 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, - "dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "dev": true, - "requires": { - "path-type": "^4.0.0" - } - }, "doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -3596,14 +3551,15 @@ "dev": true }, "eslint": { - "version": "8.25.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.25.0.tgz", - "integrity": "sha512-DVlJOZ4Pn50zcKW5bYH7GQK/9MsoQG2d5eDH0ebEkE8PbgzTTmtt/VTH9GGJ4BfeZCpBLqFfvsjX35UacUL83A==", + "version": "8.30.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.30.0.tgz", + "integrity": "sha512-MGADB39QqYuzEGov+F/qb18r4i7DohCDOfatHaxI2iGlPuC65bwG2gxgO+7DkyL38dRFaRH7RaRAgU6JKL9rMQ==", "dev": true, "requires": { - "@eslint/eslintrc": "^1.3.3", - "@humanwhocodes/config-array": "^0.10.5", + "@eslint/eslintrc": "^1.4.0", + "@humanwhocodes/config-array": "^0.11.8", "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -3619,14 +3575,14 @@ "fast-deep-equal": "^3.1.3", "file-entry-cache": "^6.0.1", "find-up": "^5.0.0", - "glob-parent": "^6.0.1", - "globals": "^13.15.0", - "globby": "^11.1.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", "grapheme-splitter": "^1.0.4", "ignore": "^5.2.0", "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", "js-sdsl": "^4.1.4", "js-yaml": "^4.1.0", "json-stable-stringify-without-jsonify": "^1.0.1", @@ -3675,9 +3631,9 @@ "dev": true }, "espree": { - "version": "9.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.0.tgz", - "integrity": "sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw==", + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz", + "integrity": "sha512-XwctdmTO6SIvCzd9810yyNzIrOrqNYV9Koizx4C/mRhf9uq0o4yHoCEU/670pOxOL/MSraektvSAji79kX90Vg==", "dev": true, "requires": { "acorn": "^8.8.0", @@ -3751,30 +3707,6 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, - "fast-glob": { - "version": "3.2.12", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", - "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", - "dev": true, - "requires": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - }, - "dependencies": { - "glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "requires": { - "is-glob": "^4.0.1" - } - } - } - }, "fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -3787,9 +3719,9 @@ "dev": true }, "fastq": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", - "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.14.0.tgz", + "integrity": "sha512-eR2D+V9/ExcbF9ls441yIuN6TI2ED1Y2ZcA5BmMtJsOkWOFRJQ0Jt0g1UwqXJJVAb+V+umH5Dfr8oh4EVP7VVg==", "dev": true, "requires": { "reusify": "^1.0.4" @@ -3809,15 +3741,6 @@ "flat-cache": "^3.0.4" } }, - "fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, - "requires": { - "to-regex-range": "^5.0.1" - } - }, "find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -3870,9 +3793,9 @@ } }, "fs-extra": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", - "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.0.tgz", + "integrity": "sha512-0rcTq621PD5jM/e0a3EJoGC/1TC5ZBCERW82LQuwfGnCa1V8w7dpYH1yNu+SLb6E5dkeCBzKEyLGlFrnr+dUyw==", "requires": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -3885,6 +3808,16 @@ "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", "requires": { "minipass": "^3.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "requires": { + "yallist": "^4.0.0" + } + } } }, "fs.realpath": { @@ -3938,28 +3871,14 @@ } }, "globals": { - "version": "13.17.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.17.0.tgz", - "integrity": "sha512-1C+6nQRb1GwGMKm2dH/E7enFAMxGTmGI7/dEdhy/DNelv85w9B72t3uc5frtMNXIbzrarJJ/lTCjcaZwbLJmyw==", + "version": "13.19.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.19.0.tgz", + "integrity": "sha512-dkQ957uSRWHw7CFXLUtUHQI3g3aWApYhfNR2O6jn/907riyTYKVBmxYVROkBcY614FSSeSJh7Xm7SrUWCxvJMQ==", "dev": true, "requires": { "type-fest": "^0.20.2" } }, - "globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "requires": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - } - }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -4027,9 +3946,9 @@ "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==" }, "ignore": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", - "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", + "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", "dev": true }, "import-fresh": { @@ -4073,9 +3992,9 @@ "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" }, "is-core-module": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.10.0.tgz", - "integrity": "sha512-Erxj2n/LDAZ7H8WNJXd9tw38GYM3dv8rk8Zcs+jJuxYTW7sozH+SS8NtrSjVL1/vpLvWi1hxy96IzjJ3EHTJJg==", + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz", + "integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==", "requires": { "has": "^1.0.3" } @@ -4100,10 +4019,10 @@ "is-extglob": "^2.1.1" } }, - "is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", "dev": true }, "is-stream": { @@ -4133,15 +4052,15 @@ "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, "jose": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/jose/-/jose-4.10.0.tgz", - "integrity": "sha512-KEhB/eLGLomWGPTb+/RNbYsTjIyx03JmbqAyIyiXBuNSa7CmNrJd5ysFhblayzs/e/vbOPMUaLnjHUMhGp4yLw==", + "version": "4.11.1", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.11.1.tgz", + "integrity": "sha512-YRv4Tk/Wlug8qicwqFNFVEZSdbROCHRAC6qu/i0dyNKr5JQdoa2pIGoS04lLO/jXQX7Z9omoNewYIVIxqZBd9Q==", "optional": true }, "js-sdsl": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.1.5.tgz", - "integrity": "sha512-08bOAKweV2NUC1wqTtf3qZlnpOX/R2DU9ikpjOHs0H+ibQv3zpncVQg6um4uYtRtrwIX8M4Nh3ytK4HGlYAq7Q==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.2.0.tgz", + "integrity": "sha512-dyBIzQBDkCqCu+0upx25Y2jGdbTGxE9fshMsCdK0ViOongpV+n5tXRcZY9v7CaVQ79AGS9KA1KHtojxiM7aXSQ==", "dev": true }, "js-yaml": { @@ -4261,31 +4180,15 @@ "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" }, "lru-cache": { - "version": "7.14.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.14.0.tgz", - "integrity": "sha512-EIRtP1GrSJny0dqb50QXRUNBxHJhcpxHC++M5tD7RYbvLLn5KVWKsbyswSSqDuU15UFi3bgTQIY8nhDMeF6aDQ==" + "version": "7.14.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.14.1.tgz", + "integrity": "sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA==" }, "merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" }, - "merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true - }, - "micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", - "dev": true, - "requires": { - "braces": "^3.0.2", - "picomatch": "^2.3.1" - } - }, "mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -4318,9 +4221,9 @@ "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==" }, "minipass": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.4.tgz", - "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.0.0.tgz", + "integrity": "sha512-g2Uuh2jEKoht+zvO6vJqXmYpflPqzRBT+Th2h01DKh5z7wbY/AZ2gCQ78cP70YoHPyFdY30YBV5WxgLOEwOykw==", "requires": { "yallist": "^4.0.0" } @@ -4332,6 +4235,16 @@ "requires": { "minipass": "^3.0.0", "yallist": "^4.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "requires": { + "yallist": "^4.0.0" + } + } } }, "mkdirp": { @@ -4467,12 +4380,12 @@ } }, "openid-client": { - "version": "5.1.10", - "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.1.10.tgz", - "integrity": "sha512-KYAtkxTuUwTvjAmH0QMFFP3i9l0+XhP2/blct6Q9kn+DUJ/lu8/g/bI8ghSgxz9dJLm/9cpB/1uLVGTcGGY0hw==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.3.1.tgz", + "integrity": "sha512-RLfehQiHch9N6tRWNx68cicf3b1WR0x74bJWHRc25uYIbSRwjxYcTFaRnzbbpls5jroLAaB/bFIodTgA5LJMvw==", "optional": true, "requires": { - "jose": "^4.1.4", + "jose": "^4.10.0", "lru-cache": "^6.0.0", "object-hash": "^2.0.1", "oidc-token-hash": "^5.0.1" @@ -4551,23 +4464,11 @@ "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, - "path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true - }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, - "picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true - }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -4587,16 +4488,26 @@ }, "dependencies": { "winston": { - "version": "2.4.6", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.6.tgz", - "integrity": "sha512-J5Zu4p0tojLde8mIOyDSsmLmcP8I3Z6wtwpTDHx1+hGcdhxcJaAmG4CFtagkb+NiN1M9Ek4b42pzMWqfc9jm8w==", + "version": "2.4.7", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.7.tgz", + "integrity": "sha512-vLB4BqzCKDnnZH9PHGoS2ycawueX4HLqENXQitvFHczhgW2vFpSOn31LZtVr1KU8YTw7DS4tM+cqyovxo8taVg==", "requires": { - "async": "^3.2.3", + "async": "^2.6.4", "colors": "1.0.x", "cycle": "1.0.x", "eyes": "0.1.x", "isstream": "0.1.x", "stack-trace": "0.0.x" + }, + "dependencies": { + "async": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", + "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", + "requires": { + "lodash": "^4.17.14" + } + } } } } @@ -4621,9 +4532,9 @@ }, "dependencies": { "long": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/long/-/long-5.2.0.tgz", - "integrity": "sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.1.tgz", + "integrity": "sha512-GKSNGeNAtw8IryjjkhZxuKB3JzlcLTwjtiQCHKvqQet81I93kXslhDQruGI/QsddO83mcDToBVy7GqGS/zYf/A==" } } }, @@ -4795,9 +4706,9 @@ "optional": true }, "safe-stable-stringify": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.0.tgz", - "integrity": "sha512-eehKHKpab6E741ud7ZIMcXhKcP6TSIezPkNZhy5U8xC6+VvrRdUA2tMgxGxaGl4cz7c2Ew5+mg5+wNB16KQqrA==" + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.1.tgz", + "integrity": "sha512-dVHE6bMtS/bnL2mwualjc6IxEv1F+OCUpA46pKUj6F8uDbUM0jCCulPqRNPSnWwGNKx5etqMjZYdXtrm5KJZGA==" }, "safer-buffer": { "version": "2.1.2", @@ -4858,12 +4769,6 @@ "is-arrayish": "^0.3.1" } }, - "slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true - }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -4958,13 +4863,13 @@ "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==" }, "tar": { - "version": "6.1.11", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz", - "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", + "version": "6.1.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.13.tgz", + "integrity": "sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==", "requires": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", - "minipass": "^3.0.0", + "minipass": "^4.0.0", "minizlib": "^2.1.1", "mkdirp": "^1.0.3", "yallist": "^4.0.0" @@ -5004,15 +4909,6 @@ "tmp": "^0.2.0" } }, - "to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "requires": { - "is-number": "^7.0.0" - } - }, "tough-cookie": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", @@ -5028,9 +4924,9 @@ "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" }, "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", + "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" }, "tunnel-agent": { "version": "0.6.0", @@ -5061,9 +4957,9 @@ "dev": true }, "uglify-js": { - "version": "3.17.3", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.17.3.tgz", - "integrity": "sha512-JmMFDME3iufZnBpyKL+uS78LRiC+mK55zWfM5f/pWBJfpOttXAqYfdDGRukYhJuyRinvPVAtUhvy7rlDybNtFg==", + "version": "3.17.4", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.17.4.tgz", + "integrity": "sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==", "optional": true }, "underscore": { @@ -5183,9 +5079,9 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "yargs": { - "version": "17.6.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.0.tgz", - "integrity": "sha512-8H/wTDqlSwoSnScvV2N/JHfLWOKuh5MVla9hqLjK3nsfyy6Y4kDSYSvkU5YCUEPOSnRXfIyx3Sq+B/IWudTo4g==", + "version": "17.6.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", + "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", "requires": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -5193,7 +5089,7 @@ "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", - "yargs-parser": "^21.0.0" + "yargs-parser": "^21.1.1" } }, "yargs-parser": { diff --git a/package.json b/package.json index f3756f1..f6209f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "democratic-csi", - "version": "1.7.7", + "version": "1.8.0", "description": "kubernetes csi driver framework", "main": "bin/democratic-csi", "scripts": { @@ -20,11 +20,11 @@ "dependencies": { "@grpc/grpc-js": "^1.5.7", "@grpc/proto-loader": "^0.7.0", - "@kubernetes/client-node": "^0.17.0", + "@kubernetes/client-node": "^0.18.0", "async-mutex": "^0.4.0", "axios": "^1.1.3", "bunyan": "^1.8.15", - "fs-extra": "^10.1.0", + "fs-extra": "^11.1.0", "handlebars": "^4.7.7", "js-yaml": "^4.0.0", "lodash": "^4.17.21", diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js index 991a922..08ebdb4 100644 --- a/src/driver/controller-zfs-generic/index.js +++ b/src/driver/controller-zfs-generic/index.js @@ -3,20 +3,29 @@ const { ControllerZfsBaseDriver } = require("../controller-zfs"); const { GrpcError, grpc } = require("../../utils/grpc"); const GeneralUtils = require("../../utils/general"); const registry = require("../../utils/registry"); -const SshClient = require("../../utils/ssh").SshClient; +const LocalCliExecClient = + require("../../utils/zfs_local_exec_client").LocalCliClient; +const SshClient = require("../../utils/zfs_ssh_exec_client").SshClient; const { Zetabyte, ZfsSshProcessManager } = require("../../utils/zfs"); const Handlebars = require("handlebars"); const ISCSI_ASSETS_NAME_PROPERTY_NAME = "democratic-csi:iscsi_assets_name"; +const NVMEOF_ASSETS_NAME_PROPERTY_NAME = "democratic-csi:nvmeof_assets_name"; const __REGISTRY_NS__ = "ControllerZfsGenericDriver"; class ControllerZfsGenericDriver extends ControllerZfsBaseDriver { getExecClient() { return registry.get(`${__REGISTRY_NS__}:exec_client`, () => { - return new SshClient({ - logger: this.ctx.logger, - connection: this.options.sshConnection, - }); + if (this.options.sshConnection) { + return new SshClient({ + logger: this.ctx.logger, + connection: this.options.sshConnection, + }); + } else { + return new LocalCliExecClient({ + logger: this.ctx.logger, + }); + } }); } @@ -24,7 +33,11 @@ class ControllerZfsGenericDriver extends ControllerZfsBaseDriver { return registry.getAsync(`${__REGISTRY_NS__}:zb`, async () => { const execClient = this.getExecClient(); const options = {}; - options.executor = new ZfsSshProcessManager(execClient); + if (this.options.sshConnection) { + options.executor = new ZfsSshProcessManager(execClient); + } else { + options.executor = execClient; + } options.idempotent = true; if ( @@ -55,6 +68,7 @@ class ControllerZfsGenericDriver extends ControllerZfsBaseDriver { case "zfs-generic-smb": return "filesystem"; case "zfs-generic-iscsi": + case "zfs-generic-nvmeof": return "volume"; default: throw new Error("unknown driver: " + this.ctx.args.driver); @@ -164,28 +178,28 @@ class ControllerZfsGenericDriver extends ControllerZfsBaseDriver { }; return volume_context; - case "zfs-generic-iscsi": + case "zfs-generic-iscsi": { let basename; - let iscsiName; + let assetName; if (this.options.iscsi.nameTemplate) { - iscsiName = Handlebars.compile(this.options.iscsi.nameTemplate)({ + assetName = Handlebars.compile(this.options.iscsi.nameTemplate)({ name: call.request.name, parameters: call.request.parameters, }); } else { - iscsiName = zb.helpers.extractLeafName(datasetName); + assetName = zb.helpers.extractLeafName(datasetName); } if (this.options.iscsi.namePrefix) { - iscsiName = this.options.iscsi.namePrefix + iscsiName; + assetName = this.options.iscsi.namePrefix + assetName; } if (this.options.iscsi.nameSuffix) { - iscsiName += this.options.iscsi.nameSuffix; + assetName += this.options.iscsi.nameSuffix; } - iscsiName = iscsiName.toLowerCase(); + assetName = assetName.toLowerCase(); let extentDiskName = "zvol/" + datasetName; @@ -239,20 +253,20 @@ class ControllerZfsGenericDriver extends ControllerZfsBaseDriver { ` # create target cd /iscsi -create ${basename}:${iscsiName} +create ${basename}:${assetName} # setup tpg -cd /iscsi/${basename}:${iscsiName}/tpg1 +cd /iscsi/${basename}:${assetName}/tpg1 ${setAttributesText} ${setAuthText} # create extent cd /backstores/block -create ${iscsiName} /dev/${extentDiskName} +create ${assetName} /dev/${extentDiskName} # add extent to target/tpg -cd /iscsi/${basename}:${iscsiName}/tpg1/luns -create /backstores/block/${iscsiName} +cd /iscsi/${basename}:${assetName}/tpg1/luns +create /backstores/block/${assetName} ` ); }, @@ -271,12 +285,12 @@ create /backstores/block/${iscsiName} } // iqn = target - let iqn = basename + ":" + iscsiName; + let iqn = basename + ":" + assetName; this.ctx.logger.info("iqn: " + iqn); // store this off to make delete process more bullet proof await zb.zfs.set(datasetName, { - [ISCSI_ASSETS_NAME_PROPERTY_NAME]: iscsiName, + [ISCSI_ASSETS_NAME_PROPERTY_NAME]: assetName, }); volume_context = { @@ -290,6 +304,231 @@ create /backstores/block/${iscsiName} lun: 0, }; return volume_context; + } + + case "zfs-generic-nvmeof": { + let basename; + let assetName; + + if (this.options.nvmeof.nameTemplate) { + assetName = Handlebars.compile(this.options.nvmeof.nameTemplate)({ + name: call.request.name, + parameters: call.request.parameters, + }); + } else { + assetName = zb.helpers.extractLeafName(datasetName); + } + + if (this.options.nvmeof.namePrefix) { + assetName = this.options.nvmeof.namePrefix + assetName; + } + + if (this.options.nvmeof.nameSuffix) { + assetName += this.options.nvmeof.nameSuffix; + } + + assetName = assetName.toLowerCase(); + + let extentDiskName = "zvol/" + datasetName; + + /** + * limit is a FreeBSD limitation + * https://www.ixsystems.com/documentation/freenas/11.2-U5/storage.html#zfs-zvol-config-opts-tab + */ + //if (extentDiskName.length > 63) { + // throw new GrpcError( + // grpc.status.FAILED_PRECONDITION, + // `extent disk name cannot exceed 63 characters: ${extentDiskName}` + // ); + //} + + let namespace = 1; + + switch (this.options.nvmeof.shareStrategy) { + case "nvmetCli": + { + basename = this.options.nvmeof.shareStrategyNvmetCli.basename; + let savefile = _.get( + this.options, + "nvmeof.shareStrategyNvmetCli.configPath", + "" + ); + if (savefile) { + savefile = `savefile=${savefile}`; + } + let setSubsystemAttributesText = ""; + if (this.options.nvmeof.shareStrategyNvmetCli.subsystem) { + if ( + this.options.nvmeof.shareStrategyNvmetCli.subsystem.attributes + ) { + for (const attributeName in this.options.nvmeof + .shareStrategyNvmetCli.subsystem.attributes) { + const attributeValue = + this.options.nvmeof.shareStrategyNvmetCli.subsystem + .attributes[attributeName]; + setSubsystemAttributesText += "\n"; + setSubsystemAttributesText += `set attr ${attributeName}=${attributeValue}`; + } + } + } + + let portCommands = ""; + this.options.nvmeof.shareStrategyNvmetCli.ports.forEach( + (port) => { + portCommands += ` +cd /ports/${port}/subsystems +create ${basename}:${assetName} +`; + } + ); + + await GeneralUtils.retry( + 3, + 2000, + async () => { + await this.nvmetCliCommand( + ` +# create subsystem +cd /subsystems +create ${basename}:${assetName} +cd ${basename}:${assetName} +${setSubsystemAttributesText} + +# create subsystem namespace +cd namespaces +create ${namespace} +cd ${namespace} +set device path=/dev/${extentDiskName} +enable + +# associate subsystem/target to port(al) +${portCommands} + +saveconfig ${savefile} +` + ); + }, + { + retryCondition: (err) => { + if (err.stdout && err.stdout.includes("Ran out of input")) { + return true; + } + return false; + }, + } + ); + } + break; + + case "spdkCli": + { + basename = this.options.nvmeof.shareStrategySpdkCli.basename; + let bdevAttributesText = ""; + if (this.options.nvmeof.shareStrategySpdkCli.bdev) { + if (this.options.nvmeof.shareStrategySpdkCli.bdev.attributes) { + for (const attributeName in this.options.nvmeof + .shareStrategySpdkCli.bdev.attributes) { + const attributeValue = + this.options.nvmeof.shareStrategySpdkCli.bdev.attributes[ + attributeName + ]; + bdevAttributesText += `${attributeName}=${attributeValue}`; + } + } + } + + let subsystemAttributesText = ""; + if (this.options.nvmeof.shareStrategySpdkCli.subsystem) { + if ( + this.options.nvmeof.shareStrategySpdkCli.subsystem.attributes + ) { + for (const attributeName in this.options.nvmeof + .shareStrategySpdkCli.subsystem.attributes) { + const attributeValue = + this.options.nvmeof.shareStrategySpdkCli.subsystem + .attributes[attributeName]; + subsystemAttributesText += `${attributeName}=${attributeValue}`; + } + } + } + + let listenerCommands = `cd /nvmf/subsystem/${basename}:${assetName}/listen_addresses\n`; + this.options.nvmeof.shareStrategySpdkCli.listeners.forEach( + (listener) => { + let listenerAttributesText = ""; + for (const attributeName in listener) { + const attributeValue = listener[attributeName]; + listenerAttributesText += ` ${attributeName}=${attributeValue} `; + } + listenerCommands += ` +create ${listenerAttributesText} +`; + } + ); + + await GeneralUtils.retry( + 3, + 2000, + async () => { + await this.spdkCliCommand( + ` +# create bdev +cd /bdevs/${this.options.nvmeof.shareStrategySpdkCli.bdev.type} +create filename=/dev/${extentDiskName} name=${basename}:${assetName} ${bdevAttributesText} + +# create subsystem +cd /nvmf/subsystem +create nqn=${basename}:${assetName} ${subsystemAttributesText} +cd ${basename}:${assetName} + +# create namespace +cd /nvmf/subsystem/${basename}:${assetName}/namespaces +create bdev_name=${basename}:${assetName} nsid=${namespace} + +# add listener +${listenerCommands} + +cd / +save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath} +` + ); + }, + { + retryCondition: (err) => { + if (err.stdout && err.stdout.includes("Ran out of input")) { + return true; + } + return false; + }, + } + ); + } + break; + + default: + break; + } + + // iqn = target + let nqn = basename + ":" + assetName; + this.ctx.logger.info("nqn: " + nqn); + + // store this off to make delete process more bullet proof + await zb.zfs.set(datasetName, { + [NVMEOF_ASSETS_NAME_PROPERTY_NAME]: assetName, + }); + + volume_context = { + node_attach_driver: "nvmeof", + transport: this.options.nvmeof.transport || "", + transports: this.options.nvmeof.transports + ? this.options.nvmeof.transports.join(",") + : "", + nqn, + nsid: namespace, + }; + return volume_context; + } default: throw new GrpcError( @@ -367,9 +606,9 @@ create /backstores/block/${iscsiName} } break; - case "zfs-generic-iscsi": + case "zfs-generic-iscsi": { let basename; - let iscsiName; + let assetName; // Delete iscsi assets try { @@ -386,23 +625,23 @@ create /backstores/block/${iscsiName} properties = properties[datasetName]; this.ctx.logger.debug("zfs props data: %j", properties); - iscsiName = properties[ISCSI_ASSETS_NAME_PROPERTY_NAME].value; + assetName = properties[ISCSI_ASSETS_NAME_PROPERTY_NAME].value; - if (zb.helpers.isPropertyValueSet(iscsiName)) { + if (zb.helpers.isPropertyValueSet(assetName)) { //do nothing } else { - iscsiName = zb.helpers.extractLeafName(datasetName); + assetName = zb.helpers.extractLeafName(datasetName); if (this.options.iscsi.namePrefix) { - iscsiName = this.options.iscsi.namePrefix + iscsiName; + assetName = this.options.iscsi.namePrefix + assetName; } if (this.options.iscsi.nameSuffix) { - iscsiName += this.options.iscsi.nameSuffix; + assetName += this.options.iscsi.nameSuffix; } } - iscsiName = iscsiName.toLowerCase(); + assetName = assetName.toLowerCase(); switch (this.options.iscsi.shareStrategy) { case "targetCli": basename = this.options.iscsi.shareStrategyTargetCli.basename; @@ -414,11 +653,11 @@ create /backstores/block/${iscsiName} ` # delete target cd /iscsi -delete ${basename}:${iscsiName} +delete ${basename}:${assetName} # delete extent cd /backstores/block -delete ${iscsiName} +delete ${assetName} ` ); }, @@ -437,6 +676,132 @@ delete ${iscsiName} break; } break; + } + + case "zfs-generic-nvmeof": { + let basename; + let assetName; + + // Delete nvmeof assets + try { + properties = await zb.zfs.get(datasetName, [ + NVMEOF_ASSETS_NAME_PROPERTY_NAME, + ]); + } catch (err) { + if (err.toString().includes("dataset does not exist")) { + return; + } + throw err; + } + + properties = properties[datasetName]; + this.ctx.logger.debug("zfs props data: %j", properties); + + assetName = properties[NVMEOF_ASSETS_NAME_PROPERTY_NAME].value; + + if (zb.helpers.isPropertyValueSet(assetName)) { + //do nothing + } else { + assetName = zb.helpers.extractLeafName(datasetName); + + if (this.options.nvmeof.namePrefix) { + assetName = this.options.nvmeof.namePrefix + assetName; + } + + if (this.options.nvmeof.nameSuffix) { + assetName += this.options.nvmeof.nameSuffix; + } + } + + assetName = assetName.toLowerCase(); + switch (this.options.nvmeof.shareStrategy) { + case "nvmetCli": + { + basename = this.options.nvmeof.shareStrategyNvmetCli.basename; + let savefile = _.get( + this.options, + "nvmeof.shareStrategyNvmetCli.configPath", + "" + ); + if (savefile) { + savefile = `savefile=${savefile}`; + } + let portCommands = ""; + this.options.nvmeof.shareStrategyNvmetCli.ports.forEach( + (port) => { + portCommands += ` +cd /ports/${port}/subsystems +delete ${basename}:${assetName} +`; + } + ); + await GeneralUtils.retry( + 3, + 2000, + async () => { + await this.nvmetCliCommand( + ` +# delete subsystem from port +${portCommands} + +# delete subsystem +cd /subsystems +delete ${basename}:${assetName} + +saveconfig ${savefile} +` + ); + }, + { + retryCondition: (err) => { + if (err.stdout && err.stdout.includes("Ran out of input")) { + return true; + } + return false; + }, + } + ); + } + break; + case "spdkCli": + { + basename = this.options.nvmeof.shareStrategySpdkCli.basename; + await GeneralUtils.retry( + 3, + 2000, + async () => { + await this.spdkCliCommand( + ` +# delete subsystem +cd /nvmf/subsystem/ +delete subsystem_nqn=${basename}:${assetName} + +# delete bdev +cd /bdevs/${this.options.nvmeof.shareStrategySpdkCli.bdev.type} +delete name=${basename}:${assetName} + +cd / +save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath} +` + ); + }, + { + retryCondition: (err) => { + if (err.stdout && err.stdout.includes("Ran out of input")) { + return true; + } + return false; + }, + } + ); + } + break; + + default: + break; + } + break; + } default: throw new GrpcError( @@ -477,18 +842,18 @@ delete ${iscsiName} let command = "sh"; let args = ["-c"]; - let targetCliArgs = ["targetcli"]; + let cliArgs = ["targetcli"]; if ( _.get(this.options, "iscsi.shareStrategyTargetCli.sudoEnabled", false) ) { targetCliArgs.unshift("sudo"); } - let targetCliCommand = []; - targetCliCommand.push(`echo "${data}"`.trim()); - targetCliCommand.push("|"); - targetCliCommand.push(targetCliArgs.join(" ")); - args.push("'" + targetCliCommand.join(" ") + "'"); + let cliCommand = []; + cliCommand.push(`echo "${data}"`.trim()); + cliCommand.push("|"); + cliCommand.push(cliArgs.join(" ")); + args.push("'" + cliCommand.join(" ") + "'"); let logCommandTmp = command + " " + args.join(" "); let logCommand = ""; @@ -527,6 +892,130 @@ delete ${iscsiName} } return response; } + + async nvmetCliCommand(data) { + const execClient = this.getExecClient(); + const driver = this; + + data = data.trim(); + + let command = "sh"; + let args = ["-c"]; + + let cliArgs = [ + _.get( + this.options, + "nvmeof.shareStrategyNvmetCli.nvmetcliPath", + "nvmetcli" + ), + ]; + if ( + _.get(this.options, "nvmeof.shareStrategyNvmetCli.sudoEnabled", false) + ) { + cliArgs.unshift("sudo"); + } + + let cliCommand = []; + cliCommand.push(`echo "${data}"`.trim()); + cliCommand.push("|"); + cliCommand.push(cliArgs.join(" ")); + args.push("'" + cliCommand.join(" ") + "'"); + + let logCommandTmp = command + " " + args.join(" "); + let logCommand = ""; + + logCommandTmp.split("\n").forEach((line) => { + if (line.startsWith("set auth password=")) { + logCommand += "set auth password="; + } else if (line.startsWith("set auth mutual_password=")) { + logCommand += "set auth mutual_password="; + } else { + logCommand += line; + } + + logCommand += "\n"; + }); + + driver.ctx.logger.verbose("nvmetCLI command: " + logCommand); + //process.exit(0); + + // https://github.com/democratic-csi/democratic-csi/issues/127 + // https://bugs.launchpad.net/ubuntu/+source/python-configshell-fb/+bug/1776761 + // can apply the linked patch with some modifications to overcome the + // KeyErrors or we can simply start a fake tty which does not seem to have + // a detrimental effect, only affects Ubuntu 18.04 and older + let options = { + pty: true, + }; + let response = await execClient.exec( + execClient.buildCommand(command, args), + options + ); + driver.ctx.logger.verbose("nvmetCLI response: " + JSON.stringify(response)); + if (response.code != 0) { + throw response; + } + return response; + } + + async spdkCliCommand(data) { + const execClient = this.getExecClient(); + const driver = this; + + data = data.trim(); + + let command = "sh"; + let args = ["-c"]; + + let cliArgs = [ + _.get(this.options, "nvmeof.shareStrategySpdkCli.spdkcliPath", "spdkcli"), + ]; + if (_.get(this.options, "nvmeof.shareStrategySpdkCli.sudoEnabled", false)) { + cliArgs.unshift("sudo"); + } + + let cliCommand = []; + cliCommand.push(`echo "${data}"`.trim()); + cliCommand.push("|"); + cliCommand.push(cliArgs.join(" ")); + args.push("'" + cliCommand.join(" ") + "'"); + + let logCommandTmp = command + " " + args.join(" "); + let logCommand = ""; + + logCommandTmp.split("\n").forEach((line) => { + if (line.startsWith("set auth password=")) { + logCommand += "set auth password="; + } else if (line.startsWith("set auth mutual_password=")) { + logCommand += "set auth mutual_password="; + } else { + logCommand += line; + } + + logCommand += "\n"; + }); + + driver.ctx.logger.verbose("spdkCLI command: " + logCommand); + //process.exit(0); + + // https://github.com/democratic-csi/democratic-csi/issues/127 + // https://bugs.launchpad.net/ubuntu/+source/python-configshell-fb/+bug/1776761 + // can apply the linked patch with some modifications to overcome the + // KeyErrors or we can simply start a fake tty which does not seem to have + // a detrimental effect, only affects Ubuntu 18.04 and older + let options = { + pty: true, + }; + let response = await execClient.exec( + execClient.buildCommand(command, args), + options + ); + driver.ctx.logger.verbose("spdkCLI response: " + JSON.stringify(response)); + if (response.code != 0) { + throw response; + } + return response; + } } module.exports.ControllerZfsGenericDriver = ControllerZfsGenericDriver; diff --git a/src/driver/controller-zfs-local/index.js b/src/driver/controller-zfs-local/index.js index 360b2d5..e22b21a 100644 --- a/src/driver/controller-zfs-local/index.js +++ b/src/driver/controller-zfs-local/index.js @@ -2,7 +2,8 @@ const _ = require("lodash"); const { ControllerZfsBaseDriver } = require("../controller-zfs"); const { GrpcError, grpc } = require("../../utils/grpc"); const GeneralUtils = require("../../utils/general"); -const LocalCliExecClient = require("./exec").LocalCliClient; +const LocalCliExecClient = + require("../../utils/zfs_local_exec_client").LocalCliClient; const registry = require("../../utils/registry"); const { Zetabyte } = require("../../utils/zfs"); diff --git a/src/driver/factory.js b/src/driver/factory.js index 02dc2ae..ea62157 100644 --- a/src/driver/factory.js +++ b/src/driver/factory.js @@ -35,6 +35,7 @@ function factory(ctx, options) { case "zfs-generic-nfs": case "zfs-generic-smb": case "zfs-generic-iscsi": + case "zfs-generic-nvmeof": return new ControllerZfsGenericDriver(ctx, options); case "zfs-local-dataset": case "zfs-local-zvol": diff --git a/src/driver/freenas/ssh.js b/src/driver/freenas/ssh.js index 1920202..fa5e950 100644 --- a/src/driver/freenas/ssh.js +++ b/src/driver/freenas/ssh.js @@ -2,7 +2,7 @@ const _ = require("lodash"); const { ControllerZfsBaseDriver } = require("../controller-zfs"); const { GrpcError, grpc } = require("../../utils/grpc"); const registry = require("../../utils/registry"); -const SshClient = require("../../utils/ssh").SshClient; +const SshClient = require("../../utils/zfs_ssh_exec_client").SshClient; const HttpClient = require("./http").Client; const TrueNASApiClient = require("./http/api").Api; const { Zetabyte, ZfsSshProcessManager } = require("../../utils/zfs"); diff --git a/src/driver/index.js b/src/driver/index.js index e9f72ea..e72e043 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -9,6 +9,7 @@ const { Mount } = require("../utils/mount"); const { OneClient } = require("../utils/oneclient"); const { Filesystem } = require("../utils/filesystem"); const { ISCSI } = require("../utils/iscsi"); +const { NVMEoF } = require("../utils/nvmeof"); const registry = require("../utils/registry"); const semver = require("semver"); const GeneralUtils = require("../utils/general"); @@ -139,6 +140,17 @@ class CsiBaseDriver { }); } + /** + * Get an instance of the NVMEoF class + * + * @returns NVMEoF + */ + getDefaultNVMEoFInstance() { + return registry.get(`${__REGISTRY_NS__}:default_nvmeof_instance`, () => { + return new NVMEoF(); + }); + } + getDefaultZetabyteInstance() { return registry.get(`${__REGISTRY_NS__}:default_zb_instance`, () => { return new Zetabyte({ @@ -560,6 +572,7 @@ class CsiBaseDriver { const mount = driver.getDefaultMountInstance(); const filesystem = driver.getDefaultFilesystemInstance(); const iscsi = driver.getDefaultISCSIInstance(); + const nvmeof = driver.getDefaultNVMEoFInstance(); let result; let device; let block_device_info; @@ -792,7 +805,11 @@ class CsiBaseDriver { await iscsi.iscsiadm.rescanSession(session); // find device name - device = iscsi.devicePathByPortalIQNLUN(iscsiConnection.portal, iscsiConnection.iqn, iscsiConnection.lun) + device = iscsi.devicePathByPortalIQNLUN( + iscsiConnection.portal, + iscsiConnection.iqn, + iscsiConnection.lun + ); let deviceByPath = device; // can take some time for device to show up, loop for some period @@ -887,6 +904,233 @@ class CsiBaseDriver { } break; + + case "nvmeof": + { + let transports = []; + if (volume_context.transport) { + transports.push(volume_context.transport.trim()); + } + + if (volume_context.transports) { + volume_context.transports.split(",").forEach((transport) => { + transports.push(transport.trim()); + }); + } + + // ensure unique entries only + transports = [...new Set(transports)]; + + // stores actual device paths after nvmeof login + let nvmeofControllerDevices = []; + let nvmeofNamespaceDevices = []; + + // stores configuration of targets/iqn/luns to connect to + let nvmeofConnections = []; + for (let transport of transports) { + nvmeofConnections.push({ + transport, + nqn: volume_context.nqn, + nsid: volume_context.nsid, + }); + } + + for (let nvmeofConnection of nvmeofConnections) { + // connect + try { + await GeneralUtils.retry(15, 2000, async () => { + await nvmeof.connectByNQNTransport( + nvmeofConnection.nqn, + nvmeofConnection.transport + ); + }); + } catch (err) { + driver.ctx.logger.warn( + `error: ${JSON.stringify(err)} connecting to transport: ${ + nvmeofConnection.transport + }` + ); + continue; + } + + // find controller device + let controllerDevice; + try { + await GeneralUtils.retry(15, 2000, async () => { + controllerDevice = + await nvmeof.controllerDevicePathByTransportNQN( + nvmeofConnection.transport, + nvmeofConnection.nqn, + nvmeofConnection.nsid + ); + + if (!controllerDevice) { + throw new Error(`failed to find controller device`); + } + }); + } catch (err) { + driver.ctx.logger.warn( + `error finding nvme controller device: ${JSON.stringify( + err + )}` + ); + continue; + } + + // find namespace device + let namespaceDevice; + try { + await GeneralUtils.retry(15, 2000, async () => { + namespaceDevice = + await nvmeof.namespaceDevicePathByNQNNamespace( + nvmeofConnection.nqn, + nvmeofConnection.nsid + ); + if (!controllerDevice) { + throw new Error(`failed to find namespace device`); + } + }); + } catch (err) { + driver.ctx.logger.warn( + `error finding nvme namespace device: ${JSON.stringify( + err + )}` + ); + continue; + } + + // sanity check for device files + if (!namespaceDevice) { + continue; + } + + // sanity check for device files + if (!controllerDevice) { + continue; + } + + // rescan in scenarios when login previously occurred but volumes never appeared + // must be the NVMe char device, not the namespace device + await nvmeof.rescanNamespace(controllerDevice); + + // can take some time for device to show up, loop for some period + result = await filesystem.pathExists(namespaceDevice); + let timer_start = Math.round(new Date().getTime() / 1000); + let timer_max = 30; + let deviceCreated = result; + while (!result) { + await GeneralUtils.sleep(2000); + result = await filesystem.pathExists(namespaceDevice); + + if (result) { + deviceCreated = true; + break; + } + + let current_time = Math.round(new Date().getTime() / 1000); + if (!result && current_time - timer_start > timer_max) { + driver.ctx.logger.warn( + `hit timeout waiting for namespace device node to appear: ${namespaceDevice}` + ); + break; + } + } + + if (deviceCreated) { + device = await filesystem.realpath(namespaceDevice); + nvmeofControllerDevices.push(controllerDevice); + nvmeofNamespaceDevices.push(namespaceDevice); + + driver.ctx.logger.info( + `successfully logged into nvmeof transport ${nvmeofConnection.transport} and created controller device: ${controllerDevice}, namespace device: ${namespaceDevice}` + ); + } + } + + // let things settle + // this will help in dm scenarios + await GeneralUtils.sleep(2000); + + // filter duplicates + nvmeofNamespaceDevices = nvmeofNamespaceDevices.filter( + (value, index, self) => { + return self.indexOf(value) === index; + } + ); + + nvmeofControllerDevices = nvmeofControllerDevices.filter( + (value, index, self) => { + return self.indexOf(value) === index; + } + ); + + // only throw an error if we were not able to attach to *any* devices + if (nvmeofNamespaceDevices.length < 1) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unable to attach any nvme devices` + ); + } + + if (nvmeofControllerDevices.length != nvmeofConnections.length) { + driver.ctx.logger.warn( + `failed to attach all nvmeof devices/subsystems/transports` + ); + + // TODO: allow a parameter to control this behavior in some form + if (false) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unable to attach all iscsi devices` + ); + } + } + + /** + * NVMEoF has native multipath capabilities without using device mapper + * You can disable the built-in using kernel param nvme_core.multipath=N/Y + */ + let useNativeMultipath = await nvmeof.nativeMultipathEnabled(); + + if (useNativeMultipath) { + // only throw an error if we were not able to attach to *any* devices + if (nvmeofNamespaceDevices.length > 1) { + throw new GrpcError( + grpc.status.UNKNOWN, + `too many nvme namespace devices, native multipath enabled therefore should only have 1` + ); + } + } else { + // compare all device-mapper slaves with the newly created devices + // if any of the new devices are device-mapper slaves treat this as a + // multipath scenario + let allDeviceMapperSlaves = + await filesystem.getAllDeviceMapperSlaveDevices(); + let commonDevices = allDeviceMapperSlaves.filter((value) => + nvmeofNamespaceDevices.includes(value) + ); + + const useDMMultipath = + nvmeofConnections.length > 1 || commonDevices.length > 0; + + // discover multipath device to use + if (useDMMultipath) { + device = await filesystem.getDeviceMapperDeviceFromSlaves( + nvmeofNamespaceDevices, + false + ); + + if (!device) { + throw new GrpcError( + grpc.status.UNKNOWN, + `failed to discover multipath device` + ); + } + } + } + } + break; + case "hostpath": result = await mount.pathIsMounted(staging_target_path); // if not mounted, mount @@ -989,6 +1233,7 @@ class CsiBaseDriver { let is_block = false; switch (node_attach_driver) { case "iscsi": + case "nvmeof": is_block = true; break; case "zfs-local": @@ -1093,6 +1338,7 @@ class CsiBaseDriver { fs_type = "cifs"; break; case "iscsi": + case "nvmeof": fs_type = "ext4"; break; default: @@ -1988,6 +2234,7 @@ class CsiBaseDriver { const mount = driver.getDefaultMountInstance(); const filesystem = driver.getDefaultFilesystemInstance(); const iscsi = driver.getDefaultISCSIInstance(); + const nvmeof = driver.getDefaultNVMEoFInstance(); let result; let is_block = false; let is_device_mapper = false; @@ -2211,6 +2458,13 @@ class CsiBaseDriver { } } } + + if (await filesystem.deviceIsNVMEoF(block_device_info_i.path)) { + let nqn = await nvmeof.nqnByNamespaceDeviceName( + block_device_info_i.name + ); + await nvmeof.disconnectByNQN(nqn); + } } } diff --git a/src/driver/node-manual/index.js b/src/driver/node-manual/index.js index 0d1c364..a69f26c 100644 --- a/src/driver/node-manual/index.js +++ b/src/driver/node-manual/index.js @@ -129,6 +129,7 @@ class NodeManualDriver extends CsiBaseDriver { driverResourceType = "filesystem"; break; case "iscsi": + case "nvmeof": driverResourceType = "volume"; fs_types = ["btrfs", "ext3", "ext4", "ext4dev", "xfs"]; break; diff --git a/src/driver/zfs-local-ephemeral-inline/index.js b/src/driver/zfs-local-ephemeral-inline/index.js index d9dfebd..2fd4fc5 100644 --- a/src/driver/zfs-local-ephemeral-inline/index.js +++ b/src/driver/zfs-local-ephemeral-inline/index.js @@ -4,7 +4,7 @@ const { GrpcError, grpc } = require("../../utils/grpc"); const { Filesystem } = require("../../utils/filesystem"); const registry = require("../../utils/registry"); const semver = require("semver"); -const SshClient = require("../../utils/ssh").SshClient; +const SshClient = require("../../utils/zfs_ssh_exec_client").SshClient; const { Zetabyte, ZfsSshProcessManager } = require("../../utils/zfs"); // zfs common properties diff --git a/src/utils/filesystem.js b/src/utils/filesystem.js index 58e039e..5d86a79 100644 --- a/src/utils/filesystem.js +++ b/src/utils/filesystem.js @@ -504,7 +504,8 @@ class Filesystem { * lsblk * blkid */ - const strategy = process.env.FILESYSTEM_TYPE_DETECTION_STRATEGY || "lsblk"; + const strategy = + process.env.FILESYSTEM_TYPE_DETECTION_STRATEGY || "lsblk"; switch (strategy) { // requires udev data to be present otherwise fstype property is always null but otherwise succeeds @@ -547,6 +548,21 @@ class Filesystem { return result && result.tran == "iscsi"; } + async deviceIsNVMEoF(device) { + const filesystem = this; + let result; + + do { + if (result) { + device = `/dev/${result.pkname}`; + } + result = await filesystem.getBlockDevice(device); + } while (result.pkname); + + // TODO: add further logic here to ensure the device is not a local pcie/etc device + return result && result.tran == "nvme"; + } + async getBlockDeviceParent(device) { const filesystem = this; let result; diff --git a/src/utils/general.js b/src/utils/general.js index 10d9b3c..9e389ab 100644 --- a/src/utils/general.js +++ b/src/utils/general.js @@ -8,6 +8,17 @@ function sleep(ms) { }); } +function trimchar(str, ch) { + var start = 0, + end = str.length; + + while (start < end && str[start] === ch) ++start; + + while (end > start && str[end - 1] === ch) --end; + + return start > 0 || end < str.length ? str.substring(start, end) : str; +} + function md5(val) { return crypto.createHash("md5").update(val).digest("hex"); } @@ -265,3 +276,4 @@ module.exports.default_supported_block_filesystems = module.exports.default_supported_file_filesystems = default_supported_file_filesystems; module.exports.retry = retry; +module.exports.trimchar = trimchar; diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js new file mode 100644 index 0000000..bd5898d --- /dev/null +++ b/src/utils/nvmeof.js @@ -0,0 +1,321 @@ +const cp = require("child_process"); +const { trimchar } = require("./general"); +const URI = require("uri-js"); +const { deleteItems } = require("@kubernetes/client-node"); + +const DEFAULT_TIMEOUT = process.env.NVMEOF_DEFAULT_TIMEOUT || 30000; + +class NVMEoF { + constructor(options = {}) { + const nvmeof = this; + nvmeof.options = options; + + options.paths = options.paths || {}; + if (!options.paths.nvme) { + options.paths.nvme = "nvme"; + } + + if (!options.paths.sudo) { + options.paths.sudo = "/usr/bin/sudo"; + } + + if (!options.executor) { + options.executor = { + spawn: cp.spawn, + }; + } + } + + /** + * List all NVMe devices and namespaces on machine + * + * @param {*} args + */ + async list(args = []) { + const nvmeof = this; + args.unshift("list", "-o", "json"); + let result = await nvmeof.exec(nvmeof.options.paths.nvme, args); + return result.parsed; + } + + /** + * List nvme subsystems + * + * @param {*} args + */ + async listSubsys(args = []) { + const nvmeof = this; + args.unshift("list-subsys", "-o", "json"); + await nvmeof.exec(nvmeof.options.paths.nvme, args); + } + + /** + * Connect to NVMeoF subsystem + * + * @param {*} args + */ + async connectByNQNTransport(nqn, transport, args = []) { + const nvmeof = this; + transport = nvmeof.parseTransport(transport); + + let transport_args = []; + if (transport.type) { + transport_args.push("--transport", transport.type); + } + if (transport.address) { + transport_args.push("--traddr", transport.address); + } + if (transport.service) { + transport_args.push("--trsvcid", transport.service); + } + + args.unshift("connect", "-o", "json", "--nqn", nqn, ...transport_args); + + try { + await nvmeof.exec(nvmeof.options.paths.nvme, args); + } catch (err) { + if (err.stderr && err.stderr.includes("already connnected")) { + // idempotent + } else { + throw err; + } + } + } + + /** + * Disconnect from NVMeoF subsystem + * + * @param {*} args + */ + async disconnectByNQN(nqn, args = []) { + const nvmeof = this; + args.unshift("disconnect", "--nqn", nqn); + await nvmeof.exec(nvmeof.options.paths.nvme, args); + } + + /** + * Disconnect from NVMeoF subsystem + * + * @param {*} args + */ + async disconnectByDevice(device, args = []) { + const nvmeof = this; + args.unshift("disconnect", "--device", device); + await nvmeof.exec(nvmeof.options.paths.nvme, args); + } + + /** + * Rescans the NVME namespaces + * + * @param {*} device + * @param {*} args + */ + async rescanNamespace(device, args = []) { + const nvmeof = this; + args.unshift("ns-rescan", device); + await nvmeof.exec(nvmeof.options.paths.nvme, args); + } + + parseTransport(transport) { + if (typeof transport === "object") { + return transport; + } + + transport = transport.trim(); + const parsed = URI.parse(transport); + + let type = parsed.scheme; + let address = parsed.host; + let service; + switch (parsed.scheme) { + case "fc": + case "rdma": + case "tcp": + type = parsed.scheme; + break; + default: + throw new Error(`unknown nvme transport type: ${parsed.scheme}`); + } + + switch (type) { + case "fc": + address = trimchar(address, "["); + address = trimchar(address, "]"); + break; + } + + switch (type) { + case "rdma": + case "tcp": + service = parsed.port; + + if (!service) { + service = 4420; + } + break; + } + + return { + type, + address, + service, + }; + } + + async nativeMultipathEnabled() { + const nvmeof = this; + let result = await nvmeof.exec("cat", [ + "/sys/module/nvme_core/parameters/multipath", + ]); + return result.stdout.trim() == "Y"; + } + + async namespaceDevicePathByNQNNamespace(nqn, namespace) { + const nvmeof = this; + let result = await nvmeof.list(["-v"]); + for (let device of result.Devices) { + for (let subsytem of device.Subsystems) { + if (subsytem.SubsystemNQN != nqn) { + continue; + } else { + for (let i_namespace of subsytem.Namespaces) { + if (i_namespace.NSID != namespace) { + continue; + } else { + return `/dev/${i_namespace.NameSpace}`; + } + } + } + } + } + } + + async controllerDevicePathByTransportNQN(transport, nqn) { + const nvmeof = this; + transport = nvmeof.parseTransport(transport); + let result = await nvmeof.list(["-v"]); + for (let device of result.Devices) { + for (let subsytem of device.Subsystems) { + if (subsytem.SubsystemNQN != nqn) { + continue; + } else { + for (let controller of subsytem.Controllers) { + if (controller.Transport != transport.type) { + continue; + } + + let controllerAddress = controller.Address; + let parts = controllerAddress.split(","); + + let traddr; + let trsvcid; + for (let i_part of parts) { + let i_parts = i_part.split("="); + switch (i_parts[0]) { + case "traddr": + traddr = i_parts[1]; + break; + case "trsvcid": + trsvcid = i_parts[1]; + break; + } + } + + if (traddr != transport.address) { + continue; + } + + if (transport.service && trsvcid != transport.service) { + continue; + } + + return `/dev/${controller.Controller}`; + } + } + } + } + } + + async nqnByNamespaceDeviceName(name) { + const nvmeof = this; + name = name.replace("/dev/", ""); + let result = await nvmeof.list(["-v"]); + for (let device of result.Devices) { + for (let subsytem of device.Subsystems) { + for (let namespace of subsytem.Namespaces) { + if (namespace.NameSpace != name) { + continue; + } else { + return subsytem.SubsystemNQN; + } + } + } + } + } + + devicePathByModelNumberSerialNumber(modelNumber, serialNumber) { + modelNumber = modelNumber.replaceAll(" ", "_"); + serialNumber = serialNumber.replaceAll(" ", "_"); + return `/dev/disk/by-id/nvme-${modelNumber}_${serialNumber}`; + } + + devicePathByPortalIQNLUN(portal, iqn, lun) { + const parsedPortal = this.parsePortal(portal); + const portalHost = parsedPortal.host + .replaceAll("[", "") + .replaceAll("]", ""); + return `/dev/disk/by-path/ip-${portalHost}:${parsedPortal.port}-iscsi-${iqn}-lun-${lun}`; + } + + exec(command, args, options = {}) { + if (!options.hasOwnProperty("timeout")) { + options.timeout = DEFAULT_TIMEOUT; + } + + const nvmeof = this; + args = args || []; + + if (nvmeof.options.sudo) { + args.unshift(command); + command = nvmeof.options.paths.sudo; + } + + console.log("executing nvmeof command: %s %s", command, args.join(" ")); + + return new Promise((resolve, reject) => { + const child = nvmeof.options.executor.spawn(command, args, options); + + let stdout = ""; + let stderr = ""; + + child.stdout.on("data", function (data) { + stdout = stdout + data; + }); + + child.stderr.on("data", function (data) { + stderr = stderr + data; + }); + + child.on("close", function (code) { + const result = { code, stdout, stderr, timeout: false }; + try { + result.parsed = JSON.parse(result.stdout); + } catch (err) {} + + // timeout scenario + if (code === null) { + result.timeout = true; + reject(result); + } + + if (code) { + reject(result); + } else { + resolve(result); + } + }); + }); + } +} + +module.exports.NVMEoF = NVMEoF; diff --git a/src/driver/controller-zfs-local/exec.js b/src/utils/zfs_local_exec_client.js similarity index 100% rename from src/driver/controller-zfs-local/exec.js rename to src/utils/zfs_local_exec_client.js diff --git a/src/utils/ssh.js b/src/utils/zfs_ssh_exec_client.js similarity index 100% rename from src/utils/ssh.js rename to src/utils/zfs_ssh_exec_client.js From 7ed64c22cc11246194102d2aba256ecfca19372b Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Tue, 20 Dec 2022 23:18:24 -0700 Subject: [PATCH 03/20] downgrade grpc-js package to fix regression Signed-off-by: Travis Glenn Hansen --- package-lock.json | 14 +++++++------- package.json | 2 +- src/driver/index.js | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6e95f6e..4872134 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.8.0", "license": "MIT", "dependencies": { - "@grpc/grpc-js": "^1.5.7", + "@grpc/grpc-js": "^1.7.3", "@grpc/proto-loader": "^0.7.0", "@kubernetes/client-node": "^0.18.0", "async-mutex": "^0.4.0", @@ -74,9 +74,9 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.0.tgz", - "integrity": "sha512-ySMTXQuMvvswoobvN+0LsaPf7ITO2JVfJmHxQKI4cGehNrrUms+n81BlHEX7Hl/LExji6XE3fnI9U04GSkRruA==", + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.7.3.tgz", + "integrity": "sha512-H9l79u4kJ2PVSxUNA08HMYAnUBLj9v6KjYQ7SQ71hOZcEXhShE/y5iQCesP8+6/Ik/7i2O0a10bPquIcYfufog==", "dependencies": { "@grpc/proto-loader": "^0.7.0", "@types/node": ">=12.12.47" @@ -2955,9 +2955,9 @@ } }, "@grpc/grpc-js": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.0.tgz", - "integrity": "sha512-ySMTXQuMvvswoobvN+0LsaPf7ITO2JVfJmHxQKI4cGehNrrUms+n81BlHEX7Hl/LExji6XE3fnI9U04GSkRruA==", + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.7.3.tgz", + "integrity": "sha512-H9l79u4kJ2PVSxUNA08HMYAnUBLj9v6KjYQ7SQ71hOZcEXhShE/y5iQCesP8+6/Ik/7i2O0a10bPquIcYfufog==", "requires": { "@grpc/proto-loader": "^0.7.0", "@types/node": ">=12.12.47" diff --git a/package.json b/package.json index f6209f4..6e20bc9 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "url": "https://github.com/democratic-csi/democratic-csi.git" }, "dependencies": { - "@grpc/grpc-js": "^1.5.7", + "@grpc/grpc-js": "^1.7.3", "@grpc/proto-loader": "^0.7.0", "@kubernetes/client-node": "^0.18.0", "async-mutex": "^0.4.0", diff --git a/src/driver/index.js b/src/driver/index.js index e72e043..af8dc11 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -2793,6 +2793,7 @@ class CsiBaseDriver { case "oneclient": case "hostpath": case "iscsi": + case "nvmeof": case "zfs-local": // ensure appropriate directories/files switch (access_type) { From 524dd85c3dbc34ff8d48d66ef4866a41b35b3523 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Wed, 21 Dec 2022 00:41:08 -0700 Subject: [PATCH 04/20] proper flow control between iscsi and nvmeof Signed-off-by: Travis Glenn Hansen --- src/driver/index.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/driver/index.js b/src/driver/index.js index af8dc11..97fa934 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -2457,9 +2457,7 @@ class CsiBaseDriver { } } } - } - - if (await filesystem.deviceIsNVMEoF(block_device_info_i.path)) { + } else if (await filesystem.deviceIsNVMEoF(block_device_info_i.path)) { let nqn = await nvmeof.nqnByNamespaceDeviceName( block_device_info_i.name ); From a9cc6fb29216ca19a9adab252f15cd2c7b453412 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Wed, 21 Dec 2022 03:00:39 -0700 Subject: [PATCH 05/20] update ci versions, minor tweaks to improve performance Signed-off-by: Travis Glenn Hansen --- .github/workflows/main.yml | 66 ++++++------- Dockerfile | 2 +- src/driver/index.js | 189 ++++++++++++++++++++----------------- 3 files changed, 137 insertions(+), 120 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 27b0e55..b9df326 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,14 +13,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.6.0 + uses: styfle/cancel-workflow-action@0.11.0 with: access_token: ${{ github.token }} build-npm-linux-amd64: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: node-version: 16 @@ -29,7 +29,7 @@ jobs: run: | ci/bin/build.sh - name: upload build - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: node-modules-linux-amd64 path: node_modules-linux-amd64.tar.gz @@ -38,7 +38,7 @@ jobs: build-npm-windows-amd64: runs-on: windows-2022 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: node-version: 16 @@ -47,7 +47,7 @@ jobs: run: | ci\bin\build.ps1 - name: upload build - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: node-modules-windows-amd64 path: node_modules-windows-amd64.tar.gz @@ -67,8 +67,8 @@ jobs: - X64 - csi-sanity-synology steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: node-modules-linux-amd64 - name: csi-sanity @@ -97,8 +97,8 @@ jobs: - X64 - csi-sanity-synology steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: node-modules-linux-amd64 - name: csi-sanity @@ -132,8 +132,8 @@ jobs: - csi-sanity-truenas #- csi-sanity-zfs-generic steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: node-modules-linux-amd64 - name: csi-sanity @@ -164,8 +164,8 @@ jobs: #- csi-sanity-truenas - csi-sanity-zfs-generic steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: node-modules-linux-amd64 - name: csi-sanity @@ -198,8 +198,8 @@ jobs: #- csi-sanity-truenas - csi-sanity-zfs-generic steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: node-modules-linux-amd64 - name: csi-sanity @@ -231,8 +231,8 @@ jobs: #- csi-sanity-truenas - csi-sanity-zfs-generic steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: node-modules-linux-amd64 - name: csi-sanity @@ -263,8 +263,8 @@ jobs: - X64 - csi-sanity-zfs-generic steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: node-modules-linux-amd64 - name: csi-sanity @@ -293,8 +293,8 @@ jobs: - X64 - csi-sanity-client steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: node-modules-linux-amd64 - name: csi-sanity @@ -321,8 +321,8 @@ jobs: - X64 - csi-sanity-client steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: node-modules-windows-amd64 - name: csi-sanity @@ -351,8 +351,8 @@ jobs: - X64 - csi-sanity-zfs-local steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: node-modules-linux-amd64 - name: csi-sanity @@ -390,8 +390,8 @@ jobs: - X64 - csi-sanity-local-hostpath steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: ${{ matrix.npmartifact }} - name: csi-sanity @@ -414,8 +414,8 @@ jobs: - Windows - X64 steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: node-modules-windows-amd64 - name: csi-sanity @@ -469,7 +469,7 @@ jobs: - csi-sanity-windows-node runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: docker build run: | export ARCH=$([ $(uname -m) = "x86_64" ] && echo "amd64" || echo "arm64") @@ -520,7 +520,7 @@ jobs: nano_base_tag: ltsc2022 file: Dockerfile.Windows steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: docker build shell: bash run: | @@ -532,7 +532,7 @@ jobs: docker inspect democratic-csi-windows:${GITHUB_RUN_ID}-${{ matrix.core_base_tag }} docker save democratic-csi-windows:${GITHUB_RUN_ID}-${{ matrix.core_base_tag }} -o democratic-csi-windows-${{ matrix.core_base_tag }}.tar - name: upload image tar - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: democratic-csi-windows-${{ matrix.core_base_tag }}.tar path: democratic-csi-windows-${{ matrix.core_base_tag }}.tar @@ -547,7 +547,7 @@ jobs: - self-hosted - buildah steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: name: democratic-csi-windows-ltsc2019.tar diff --git a/Dockerfile b/Dockerfile index fb42b40..f6f2076 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,7 +75,7 @@ COPY --from=build /usr/local/lib/nodejs/bin/node /usr/local/bin/node # netbase is required by rpcbind/rpcinfo to work properly # /etc/{services,rpc} are required RUN apt-get update && \ - apt-get install -y netbase socat e2fsprogs exfatprogs xfsprogs btrfs-progs fatresize dosfstools ntfs-3g nfs-common cifs-utils fdisk gdisk cloud-guest-utils sudo rsync procps util-linux && \ + apt-get install -y netbase socat e2fsprogs exfatprogs xfsprogs btrfs-progs fatresize dosfstools ntfs-3g nfs-common cifs-utils fdisk gdisk cloud-guest-utils sudo rsync procps util-linux nvme-cli && \ rm -rf /var/lib/apt/lists/* # controller requirements diff --git a/src/driver/index.js b/src/driver/index.js index 97fa934..d3d1c37 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -2369,99 +2369,116 @@ class CsiBaseDriver { // TODO: this could be made async to detach all simultaneously for (const block_device_info_i of realBlockDeviceInfos) { - if (await filesystem.deviceIsIscsi(block_device_info_i.path)) { - let parent_block_device = await filesystem.getBlockDeviceParent( - block_device_info_i.path - ); + switch (block_device_info_i.tran) { + case "iscsi": + { + if ( + await filesystem.deviceIsIscsi(block_device_info_i.path) + ) { + let parent_block_device = + await filesystem.getBlockDeviceParent( + block_device_info_i.path + ); - // figure out which iscsi session this belongs to and logout - // scan /dev/disk/by-path/ip-*? - // device = `/dev/disk/by-path/ip-${volume_context.portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`; - // parse output from `iscsiadm -m session -P 3` - let sessions = await iscsi.iscsiadm.getSessionsDetails(); - for (let i = 0; i < sessions.length; i++) { - let session = sessions[i]; - let is_attached_to_session = false; + // figure out which iscsi session this belongs to and logout + // scan /dev/disk/by-path/ip-*? + // device = `/dev/disk/by-path/ip-${volume_context.portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`; + // parse output from `iscsiadm -m session -P 3` + let sessions = await iscsi.iscsiadm.getSessionsDetails(); + for (let i = 0; i < sessions.length; i++) { + let session = sessions[i]; + let is_attached_to_session = false; - if ( - session.attached_scsi_devices && - session.attached_scsi_devices.host && - session.attached_scsi_devices.host.devices - ) { - is_attached_to_session = - session.attached_scsi_devices.host.devices.some( - (device) => { - if ( - device.attached_scsi_disk == parent_block_device.name - ) { - return true; + if ( + session.attached_scsi_devices && + session.attached_scsi_devices.host && + session.attached_scsi_devices.host.devices + ) { + is_attached_to_session = + session.attached_scsi_devices.host.devices.some( + (device) => { + if ( + device.attached_scsi_disk == + parent_block_device.name + ) { + return true; + } + return false; + } + ); + } + + if (is_attached_to_session) { + let timer_start; + let timer_max; + + timer_start = Math.round(new Date().getTime() / 1000); + timer_max = 30; + let loggedOut = false; + while (!loggedOut) { + try { + await iscsi.iscsiadm.logout(session.target, [ + session.persistent_portal, + ]); + loggedOut = true; + } catch (err) { + await GeneralUtils.sleep(2000); + let current_time = Math.round( + new Date().getTime() / 1000 + ); + if (current_time - timer_start > timer_max) { + // not throwing error for now as future invocations would not enter code path anyhow + loggedOut = true; + //throw new GrpcError( + // grpc.status.UNKNOWN, + // `hit timeout trying to logout of iscsi target: ${session.persistent_portal}` + //); + } + } + } + + timer_start = Math.round(new Date().getTime() / 1000); + timer_max = 30; + let deletedEntry = false; + while (!deletedEntry) { + try { + await iscsi.iscsiadm.deleteNodeDBEntry( + session.target, + session.persistent_portal + ); + deletedEntry = true; + } catch (err) { + await GeneralUtils.sleep(2000); + let current_time = Math.round( + new Date().getTime() / 1000 + ); + if (current_time - timer_start > timer_max) { + // not throwing error for now as future invocations would not enter code path anyhow + deletedEntry = true; + //throw new GrpcError( + // grpc.status.UNKNOWN, + // `hit timeout trying to delete iscsi node DB entry: ${session.target}, ${session.persistent_portal}` + //); + } + } } - return false; } + } + } + } + break; + case "nvme": + { + if ( + await filesystem.deviceIsNVMEoF(block_device_info_i.path) + ) { + let nqn = await nvmeof.nqnByNamespaceDeviceName( + block_device_info_i.name ); - } - - if (is_attached_to_session) { - let timer_start; - let timer_max; - - timer_start = Math.round(new Date().getTime() / 1000); - timer_max = 30; - let loggedOut = false; - while (!loggedOut) { - try { - await iscsi.iscsiadm.logout(session.target, [ - session.persistent_portal, - ]); - loggedOut = true; - } catch (err) { - await GeneralUtils.sleep(2000); - let current_time = Math.round( - new Date().getTime() / 1000 - ); - if (current_time - timer_start > timer_max) { - // not throwing error for now as future invocations would not enter code path anyhow - loggedOut = true; - //throw new GrpcError( - // grpc.status.UNKNOWN, - // `hit timeout trying to logout of iscsi target: ${session.persistent_portal}` - //); - } - } - } - - timer_start = Math.round(new Date().getTime() / 1000); - timer_max = 30; - let deletedEntry = false; - while (!deletedEntry) { - try { - await iscsi.iscsiadm.deleteNodeDBEntry( - session.target, - session.persistent_portal - ); - deletedEntry = true; - } catch (err) { - await GeneralUtils.sleep(2000); - let current_time = Math.round( - new Date().getTime() / 1000 - ); - if (current_time - timer_start > timer_max) { - // not throwing error for now as future invocations would not enter code path anyhow - deletedEntry = true; - //throw new GrpcError( - // grpc.status.UNKNOWN, - // `hit timeout trying to delete iscsi node DB entry: ${session.target}, ${session.persistent_portal}` - //); - } - } + await nvmeof.disconnectByNQN(nqn); } } - } - } else if (await filesystem.deviceIsNVMEoF(block_device_info_i.path)) { - let nqn = await nvmeof.nqnByNamespaceDeviceName( - block_device_info_i.name - ); - await nvmeof.disconnectByNQN(nqn); + break; } } } From afba7d852745a7ad4735cf8c5de6e007fa9c35a9 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Thu, 22 Dec 2022 01:24:43 -0700 Subject: [PATCH 06/20] proper support for both native nvme multipath and DM multipath Signed-off-by: Travis Glenn Hansen --- contrib/scale-nvmet-start.sh | 58 ++++++++++++++ src/driver/index.js | 26 +++++- src/utils/nvmeof.js | 148 ++++++++++++++++++++++------------- 3 files changed, 175 insertions(+), 57 deletions(-) create mode 100755 contrib/scale-nvmet-start.sh diff --git a/contrib/scale-nvmet-start.sh b/contrib/scale-nvmet-start.sh new file mode 100755 index 0000000..42fbae8 --- /dev/null +++ b/contrib/scale-nvmet-start.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# simple script to 'start' nvmet on TrueNAS SCALE +# +# to reinstall nvmetcli simply rm /usr/sbin/nvmetcli + +# debug +#set -x + +# exit non-zero +set -e + +SCRIPTDIR="$( + cd -- "$(dirname "$0")" >/dev/null 2>&1 + pwd -P +)" +cd "${SCRIPTDIR}" + +: "${NVMETCONFIG:="${SCRIPTDIR}/nvmet-config.json"}" + +export PATH=${HOME}/.local/bin:${PATH} + +modules=() +modules+=("nvmet") +modules+=("nvmet-fc") +modules+=("nvmet-rdma") +modules+=("nvmet-tcp") + +for module in "${modules[@]}"; do + modprobe "${module}" +done + +which nvmetcli &>/dev/null || { + which pip &>/dev/null || { + wget -O get-pip.py https://bootstrap.pypa.io/get-pip.py + python get-pip.py --user + rm get-pip.py + } + + if [[ ! -d nvmetcli ]]; then + git clone git://git.infradead.org/users/hch/nvmetcli.git + fi + + cd nvmetcli + + # install to root home dir + python3 setup.py install --user + + # install to root home dir + pip install configshell_fb --user + + # remove source + cd "${SCRIPTDIR}" + rm -rf nvmetcli +} + +cd "${SCRIPTDIR}" +nvmetcli restore "${NVMETCONFIG}" diff --git a/src/driver/index.js b/src/driver/index.js index d3d1c37..b2201d3 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -14,6 +14,7 @@ const registry = require("../utils/registry"); const semver = require("semver"); const GeneralUtils = require("../utils/general"); const { Zetabyte } = require("../utils/zfs"); +const { transport } = require("winston"); const __REGISTRY_NS__ = "CsiBaseDriver"; @@ -982,7 +983,8 @@ class CsiBaseDriver { try { await GeneralUtils.retry(15, 2000, async () => { namespaceDevice = - await nvmeof.namespaceDevicePathByNQNNamespace( + await nvmeof.namespaceDevicePathByTransportNQNNamespace( + nvmeofConnection.transport, nvmeofConnection.nqn, nvmeofConnection.nsid ); @@ -1126,6 +1128,14 @@ class CsiBaseDriver { `failed to discover multipath device` ); } + } else { + // only throw an error if we were not able to attach to *any* devices + if (nvmeofNamespaceDevices.length > 1) { + throw new GrpcError( + grpc.status.UNKNOWN, + `too many nvme namespace devices, neither DM nor native multipath enabled` + ); + } } } } @@ -2348,6 +2358,7 @@ class CsiBaseDriver { } if (is_block) { + let breakdeviceloop = false; let realBlockDeviceInfos = []; // detect if is a multipath device is_device_mapper = await filesystem.isDeviceMapperDevice( @@ -2369,6 +2380,9 @@ class CsiBaseDriver { // TODO: this could be made async to detach all simultaneously for (const block_device_info_i of realBlockDeviceInfos) { + if (breakdeviceloop) { + break; + } switch (block_device_info_i.tran) { case "iscsi": { @@ -2475,7 +2489,15 @@ class CsiBaseDriver { let nqn = await nvmeof.nqnByNamespaceDeviceName( block_device_info_i.name ); - await nvmeof.disconnectByNQN(nqn); + if (nqn) { + await nvmeof.disconnectByNQN(nqn); + /** + * the above disconnects *all* devices with the nqn so we + * do NOT want to keep iterating all the 'real' devices + * in the case of DM multipath + */ + breakdeviceloop = true; + } } } break; diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index bd5898d..43b399f 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -106,9 +106,9 @@ class NVMEoF { /** * Rescans the NVME namespaces - * - * @param {*} device - * @param {*} args + * + * @param {*} device + * @param {*} args */ async rescanNamespace(device, args = []) { const nvmeof = this; @@ -170,20 +170,29 @@ class NVMEoF { return result.stdout.trim() == "Y"; } - async namespaceDevicePathByNQNNamespace(nqn, namespace) { + async namespaceDevicePathByTransportNQNNamespace(transport, nqn, namespace) { const nvmeof = this; - let result = await nvmeof.list(["-v"]); - for (let device of result.Devices) { - for (let subsytem of device.Subsystems) { - if (subsytem.SubsystemNQN != nqn) { - continue; - } else { - for (let i_namespace of subsytem.Namespaces) { - if (i_namespace.NSID != namespace) { - continue; - } else { - return `/dev/${i_namespace.NameSpace}`; - } + transport = nvmeof.parseTransport(transport); + let nativeMultipathEnabled = await nvmeof.nativeMultipathEnabled(); + if (nativeMultipathEnabled) { + let subsystem = await nvmeof.getSubsystemByNQN(nqn); + if (subsystem) { + for (let i_namespace of subsystem.Namespaces) { + if (i_namespace.NSID != namespace) { + continue; + } else { + return `/dev/${i_namespace.NameSpace}`; + } + } + } + } else { + let controller = await nvmeof.getControllerByTransportNQN(transport, nqn); + if (controller) { + for (let i_namespace of controller.Namespaces) { + if (i_namespace.NSID != namespace) { + continue; + } else { + return `/dev/${i_namespace.NameSpace}`; } } } @@ -193,45 +202,60 @@ class NVMEoF { async controllerDevicePathByTransportNQN(transport, nqn) { const nvmeof = this; transport = nvmeof.parseTransport(transport); + let controller = await nvmeof.getControllerByTransportNQN(transport, nqn); + if (controller) { + return `/dev/${controller.Controller}`; + } + } + + async getSubsystemByNQN(nqn) { + const nvmeof = this; let result = await nvmeof.list(["-v"]); for (let device of result.Devices) { - for (let subsytem of device.Subsystems) { - if (subsytem.SubsystemNQN != nqn) { + for (let subsystem of device.Subsystems) { + if (subsystem.SubsystemNQN == nqn) { + return subsystem; + } + } + } + } + + async getControllerByTransportNQN(transport, nqn) { + const nvmeof = this; + transport = nvmeof.parseTransport(transport); + let subsystem = await nvmeof.getSubsystemByNQN(nqn); + if (subsystem) { + for (let controller of subsystem.Controllers) { + if (controller.Transport != transport.type) { continue; - } else { - for (let controller of subsytem.Controllers) { - if (controller.Transport != transport.type) { - continue; - } + } - let controllerAddress = controller.Address; - let parts = controllerAddress.split(","); + let controllerAddress = controller.Address; + let parts = controllerAddress.split(","); - let traddr; - let trsvcid; - for (let i_part of parts) { - let i_parts = i_part.split("="); - switch (i_parts[0]) { - case "traddr": - traddr = i_parts[1]; - break; - case "trsvcid": - trsvcid = i_parts[1]; - break; - } - } - - if (traddr != transport.address) { - continue; - } - - if (transport.service && trsvcid != transport.service) { - continue; - } - - return `/dev/${controller.Controller}`; + let traddr; + let trsvcid; + for (let i_part of parts) { + let i_parts = i_part.split("="); + switch (i_parts[0]) { + case "traddr": + traddr = i_parts[1]; + break; + case "trsvcid": + trsvcid = i_parts[1]; + break; } } + + if (traddr != transport.address) { + continue; + } + + if (transport.service && trsvcid != transport.service) { + continue; + } + + return controller; } } } @@ -240,13 +264,27 @@ class NVMEoF { const nvmeof = this; name = name.replace("/dev/", ""); let result = await nvmeof.list(["-v"]); - for (let device of result.Devices) { - for (let subsytem of device.Subsystems) { - for (let namespace of subsytem.Namespaces) { - if (namespace.NameSpace != name) { - continue; - } else { - return subsytem.SubsystemNQN; + let nativeMultipathEnabled = await nvmeof.nativeMultipathEnabled(); + + if (nativeMultipathEnabled) { + for (let device of result.Devices) { + for (let subsystem of device.Subsystems) { + for (let namespace of subsystem.Namespaces) { + if (namespace.NameSpace == name) { + return subsystem.SubsystemNQN; + } + } + } + } + } else { + for (let device of result.Devices) { + for (let subsystem of device.Subsystems) { + for (let controller of subsystem.Controllers) { + for (let namespace of controller.Namespaces) { + if (namespace.NameSpace == name) { + return subsystem.SubsystemNQN; + } + } } } } From 263a72635a547d37dd4aedc121255aa722c234c4 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Thu, 22 Dec 2022 08:13:22 -0700 Subject: [PATCH 07/20] use nvme args compatible with current container image Signed-off-by: Travis Glenn Hansen --- src/utils/nvmeof.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index 43b399f..4a9d8fc 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -69,7 +69,7 @@ class NVMEoF { transport_args.push("--trsvcid", transport.service); } - args.unshift("connect", "-o", "json", "--nqn", nqn, ...transport_args); + args.unshift("connect", "--nqn", nqn, ...transport_args); try { await nvmeof.exec(nvmeof.options.paths.nvme, args); From 1d49644217c98ea542afc3ecb1584b4eac433916 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Wed, 28 Dec 2022 21:08:51 -0700 Subject: [PATCH 08/20] fix nvme address parsing with older versions of nvme-cli package Signed-off-by: Travis Glenn Hansen --- src/driver/index.js | 3 ++- src/utils/nvmeof.js | 56 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/src/driver/index.js b/src/driver/index.js index b2201d3..e1eff97 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -147,8 +147,9 @@ class CsiBaseDriver { * @returns NVMEoF */ getDefaultNVMEoFInstance() { + const driver = this; return registry.get(`${__REGISTRY_NS__}:default_nvmeof_instance`, () => { - return new NVMEoF(); + return new NVMEoF({ logger: driver.ctx.logger }); }); } diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index 4a9d8fc..f514208 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -24,6 +24,12 @@ class NVMEoF { spawn: cp.spawn, }; } + + if (nvmeof.options.logger) { + nvmeof.logger = nvmeof.options.logger; + } else { + nvmeof.logger = console; + } } /** @@ -46,7 +52,35 @@ class NVMEoF { async listSubsys(args = []) { const nvmeof = this; args.unshift("list-subsys", "-o", "json"); - await nvmeof.exec(nvmeof.options.paths.nvme, args); + let result = await nvmeof.exec(nvmeof.options.paths.nvme, args); + return result.parsed; + } + + /** + * Discover NVMeoF subsystems + * + * @param {*} transport + * @param {*} args + * @returns + */ + async discover(transport, args = []) { + const nvmeof = this; + transport = nvmeof.parseTransport(transport); + + let transport_args = []; + if (transport.type) { + transport_args.push("--transport", transport.type); + } + if (transport.address) { + transport_args.push("--traddr", transport.address); + } + if (transport.service) { + transport_args.push("--trsvcid", transport.service); + } + + args.unshift("discover", "-o", "json", ...transport_args); + let result = await nvmeof.exec(nvmeof.options.paths.nvme, args); + return result.parsed; } /** @@ -218,6 +252,8 @@ class NVMEoF { } } } + + nvmeof.logger.warn(`failed to find subsystem for nqn: ${nqn}`); } async getControllerByTransportNQN(transport, nqn) { @@ -231,6 +267,12 @@ class NVMEoF { } let controllerAddress = controller.Address; + /** + * For backwards compatibility with older nvme-cli versions (at least < 2.2.1) + * old: "Address":"traddr=127.0.0.1 trsvcid=4420" + * new: "Address":"traddr=127.0.0.1,trsvcid=4420" + */ + controllerAddress = controllerAddress.replaceAll(" ", ","); let parts = controllerAddress.split(","); let traddr; @@ -258,6 +300,12 @@ class NVMEoF { return controller; } } + + nvmeof.logger.warn( + `failed to find controller for transport: ${JSON.stringify( + transport + )}, nqn: ${nqn}` + ); } async nqnByNamespaceDeviceName(name) { @@ -318,7 +366,11 @@ class NVMEoF { command = nvmeof.options.paths.sudo; } - console.log("executing nvmeof command: %s %s", command, args.join(" ")); + nvmeof.logger.verbose( + "executing nvmeof command: %s %s", + command, + args.join(" ") + ); return new Promise((resolve, reject) => { const child = nvmeof.options.executor.spawn(command, args, options); From 467968aa65063a198eac1dc63f186ff7d4e384c0 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Wed, 28 Dec 2022 21:19:04 -0700 Subject: [PATCH 09/20] make string replace more stringent and better cleanse values Signed-off-by: Travis Glenn Hansen --- src/utils/nvmeof.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index f514208..f7cfc35 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -272,19 +272,22 @@ class NVMEoF { * old: "Address":"traddr=127.0.0.1 trsvcid=4420" * new: "Address":"traddr=127.0.0.1,trsvcid=4420" */ - controllerAddress = controllerAddress.replaceAll(" ", ","); + controllerAddress = controllerAddress.replaceAll( + " trsvcid=", + ",trsvcid=" + ); let parts = controllerAddress.split(","); let traddr; let trsvcid; for (let i_part of parts) { let i_parts = i_part.split("="); - switch (i_parts[0]) { + switch (i_parts[0].trim()) { case "traddr": - traddr = i_parts[1]; + traddr = i_parts[1].trim(); break; case "trsvcid": - trsvcid = i_parts[1]; + trsvcid = i_parts[1].trim(); break; } } From d05e29a148070cb7c8f6b84d6a2d602e8cb5ab48 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Wed, 28 Dec 2022 21:39:14 -0700 Subject: [PATCH 10/20] use regex for more robust replacement parsing Signed-off-by: Travis Glenn Hansen --- src/utils/nvmeof.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index f7cfc35..794a0e3 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -272,9 +272,9 @@ class NVMEoF { * old: "Address":"traddr=127.0.0.1 trsvcid=4420" * new: "Address":"traddr=127.0.0.1,trsvcid=4420" */ - controllerAddress = controllerAddress.replaceAll( - " trsvcid=", - ",trsvcid=" + controllerAddress = controllerAddress.replace( + new RegExp(/ ([a-z]*=)/, "g"), + ",$1" ); let parts = controllerAddress.split(","); From 6738ca4a7e01540c9320557bf6f480538f2473dd Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Thu, 29 Dec 2022 00:14:28 -0700 Subject: [PATCH 11/20] always use ip for nvme transport address Signed-off-by: Travis Glenn Hansen --- .github/workflows/main.yml | 2 ++ src/utils/general.js | 14 ++++++++++++++ src/utils/nvmeof.js | 27 +++++++++++++++++++-------- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b9df326..8b70796 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -459,6 +459,7 @@ jobs: - csi-sanity-synology-dsm6 - csi-sanity-synology-dsm7 - csi-sanity-truenas-scale-22_02 + - csi-sanity-truenas-scale-22_12 - csi-sanity-truenas-core-12_0 - csi-sanity-truenas-core-13_0 - csi-sanity-zfs-generic @@ -498,6 +499,7 @@ jobs: - csi-sanity-synology-dsm6 - csi-sanity-synology-dsm7 - csi-sanity-truenas-scale-22_02 + - csi-sanity-truenas-scale-22_12 - csi-sanity-truenas-core-12_0 - csi-sanity-truenas-core-13_0 - csi-sanity-zfs-generic diff --git a/src/utils/general.js b/src/utils/general.js index 9be60ab..e07b7b9 100644 --- a/src/utils/general.js +++ b/src/utils/general.js @@ -1,6 +1,7 @@ const _ = require("lodash"); const axios = require("axios"); const crypto = require("crypto"); +const dns = require("dns"); function sleep(ms) { return new Promise((resolve) => { @@ -261,6 +262,18 @@ async function retry(retries, retriesDelay, code, options = {}) { } while (true); } +async function hostname_lookup(hostname) { + return new Promise((resolve, reject) => { + dns.lookup(hostname, function (err, result) { + if (err) { + return reject(err); + } + + return resolve(result); + }); + }); +} + module.exports.sleep = sleep; module.exports.md5 = md5; module.exports.crc32 = crc32; @@ -277,3 +290,4 @@ module.exports.default_supported_file_filesystems = default_supported_file_filesystems; module.exports.retry = retry; module.exports.trimchar = trimchar; +module.exports.hostname_lookup = hostname_lookup; diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index 794a0e3..4c42636 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -1,7 +1,6 @@ const cp = require("child_process"); -const { trimchar } = require("./general"); +const { hostname_lookup, trimchar } = require("./general"); const URI = require("uri-js"); -const { deleteItems } = require("@kubernetes/client-node"); const DEFAULT_TIMEOUT = process.env.NVMEOF_DEFAULT_TIMEOUT || 30000; @@ -65,7 +64,7 @@ class NVMEoF { */ async discover(transport, args = []) { const nvmeof = this; - transport = nvmeof.parseTransport(transport); + transport = await nvmeof.parseTransport(transport); let transport_args = []; if (transport.type) { @@ -90,7 +89,7 @@ class NVMEoF { */ async connectByNQNTransport(nqn, transport, args = []) { const nvmeof = this; - transport = nvmeof.parseTransport(transport); + transport = await nvmeof.parseTransport(transport); let transport_args = []; if (transport.type) { @@ -150,7 +149,7 @@ class NVMEoF { await nvmeof.exec(nvmeof.options.paths.nvme, args); } - parseTransport(transport) { + async parseTransport(transport) { if (typeof transport === "object") { return transport; } @@ -176,6 +175,18 @@ class NVMEoF { address = trimchar(address, "["); address = trimchar(address, "]"); break; + case "tcp": + /** + * kernel stores value as ip, so if address passed as hostname then + * translate to ip address + * + * TODO: this could be brittle + */ + let lookup = await hostname_lookup(address); + if (lookup) { + address = lookup; + } + break; } switch (type) { @@ -206,7 +217,7 @@ class NVMEoF { async namespaceDevicePathByTransportNQNNamespace(transport, nqn, namespace) { const nvmeof = this; - transport = nvmeof.parseTransport(transport); + transport = await nvmeof.parseTransport(transport); let nativeMultipathEnabled = await nvmeof.nativeMultipathEnabled(); if (nativeMultipathEnabled) { let subsystem = await nvmeof.getSubsystemByNQN(nqn); @@ -235,7 +246,7 @@ class NVMEoF { async controllerDevicePathByTransportNQN(transport, nqn) { const nvmeof = this; - transport = nvmeof.parseTransport(transport); + transport = await nvmeof.parseTransport(transport); let controller = await nvmeof.getControllerByTransportNQN(transport, nqn); if (controller) { return `/dev/${controller.Controller}`; @@ -258,7 +269,7 @@ class NVMEoF { async getControllerByTransportNQN(transport, nqn) { const nvmeof = this; - transport = nvmeof.parseTransport(transport); + transport = await nvmeof.parseTransport(transport); let subsystem = await nvmeof.getSubsystemByNQN(nqn); if (subsystem) { for (let controller of subsystem.Controllers) { From bb5019bd7e730fd00ef5420efb5c82fdff6eb0ef Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Thu, 29 Dec 2022 10:32:13 -0700 Subject: [PATCH 12/20] better support for nvme-cli 1.x Signed-off-by: Travis Glenn Hansen --- src/driver/index.js | 9 ++++---- src/utils/nvmeof.js | 53 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/driver/index.js b/src/driver/index.js index e1eff97..303c9a4 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -980,6 +980,11 @@ class CsiBaseDriver { } // find namespace device + + // rescan in scenarios when login previously occurred but volumes never appeared + // must be the NVMe char device, not the namespace device + await nvmeof.rescanNamespace(controllerDevice); + let namespaceDevice; try { await GeneralUtils.retry(15, 2000, async () => { @@ -1012,10 +1017,6 @@ class CsiBaseDriver { continue; } - // rescan in scenarios when login previously occurred but volumes never appeared - // must be the NVMe char device, not the namespace device - await nvmeof.rescanNamespace(controllerDevice); - // can take some time for device to show up, loop for some period result = await filesystem.pathExists(namespaceDevice); let timer_start = Math.round(new Date().getTime() / 1000); diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index 4c42636..5147cd3 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -219,6 +219,7 @@ class NVMEoF { const nvmeof = this; transport = await nvmeof.parseTransport(transport); let nativeMultipathEnabled = await nvmeof.nativeMultipathEnabled(); + if (nativeMultipathEnabled) { let subsystem = await nvmeof.getSubsystemByNQN(nqn); if (subsystem) { @@ -253,14 +254,34 @@ class NVMEoF { } } - async getSubsystemByNQN(nqn) { + async getSubsystems() { const nvmeof = this; let result = await nvmeof.list(["-v"]); + + return nvmeof.getResultSubsystems(result); + } + + async getResultSubsystems(result) { + let subsystems = []; + for (let device of result.Devices) { - for (let subsystem of device.Subsystems) { - if (subsystem.SubsystemNQN == nqn) { - return subsystem; - } + if (Array.isArray(device.Subsystems)) { + subsystems = subsystems.concat(device.Subsystems); + } else if (device.Subsystem) { + // nvme-cli 1.x support + subsystems.push(device); + } + } + + return subsystems; + } + + async getSubsystemByNQN(nqn) { + const nvmeof = this; + const subsystems = await nvmeof.getSubsystems(); + for (let subsystem of subsystems) { + if (subsystem.SubsystemNQN == nqn) { + return subsystem; } } @@ -325,12 +346,13 @@ class NVMEoF { async nqnByNamespaceDeviceName(name) { const nvmeof = this; name = name.replace("/dev/", ""); - let result = await nvmeof.list(["-v"]); let nativeMultipathEnabled = await nvmeof.nativeMultipathEnabled(); + const subsystems = await nvmeof.getSubsystems(); if (nativeMultipathEnabled) { - for (let device of result.Devices) { - for (let subsystem of device.Subsystems) { + // using per-subsystem namespace + for (let subsystem of subsystems) { + if (subsystem.Namespaces) { for (let namespace of subsystem.Namespaces) { if (namespace.NameSpace == name) { return subsystem.SubsystemNQN; @@ -339,18 +361,23 @@ class NVMEoF { } } } else { - for (let device of result.Devices) { - for (let subsystem of device.Subsystems) { + // using per-controller namespace + for (let subsystem of subsystems) { + if (subsystem.Controllers) { for (let controller of subsystem.Controllers) { - for (let namespace of controller.Namespaces) { - if (namespace.NameSpace == name) { - return subsystem.SubsystemNQN; + if (controller.Namespaces) { + for (let namespace of controller.Namespaces) { + if (namespace.NameSpace == name) { + return subsystem.SubsystemNQN; + } } } } } } } + + nvmeof.logger.warn(`failed to find nqn for device: ${name}`); } devicePathByModelNumberSerialNumber(modelNumber, serialNumber) { From 4e99e81255d046e946111071ffc8bbe50e2f211c Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Thu, 29 Dec 2022 11:00:44 -0700 Subject: [PATCH 13/20] better nvme connect idempotency Signed-off-by: Travis Glenn Hansen --- src/utils/nvmeof.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index 5147cd3..5e65622 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -107,7 +107,11 @@ class NVMEoF { try { await nvmeof.exec(nvmeof.options.paths.nvme, args); } catch (err) { - if (err.stderr && err.stderr.includes("already connnected")) { + if ( + err.stderr && + (err.stderr.includes("already connnected") || + err.stderr.includes("Operation already in progress")) + ) { // idempotent } else { throw err; @@ -258,10 +262,16 @@ class NVMEoF { const nvmeof = this; let result = await nvmeof.list(["-v"]); - return nvmeof.getResultSubsystems(result); + return nvmeof.getNormalizedSubsystems(result); } - async getResultSubsystems(result) { + /** + * used to normalize subsystem list/response across different versions of nvme-cli + * + * @param {*} result + * @returns + */ + async getNormalizedSubsystems(result) { let subsystems = []; for (let device of result.Devices) { From 9bf6d46b66ade3252d684ed5dbe893b03bf1c40b Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Thu, 29 Dec 2022 21:58:47 -0700 Subject: [PATCH 14/20] implement nvmeof node expand Signed-off-by: Travis Glenn Hansen --- src/driver/index.js | 19 +++++++--- src/utils/nvmeof.js | 92 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 103 insertions(+), 8 deletions(-) diff --git a/src/driver/index.js b/src/driver/index.js index 303c9a4..1648c7f 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -980,14 +980,13 @@ class CsiBaseDriver { } // find namespace device - - // rescan in scenarios when login previously occurred but volumes never appeared - // must be the NVMe char device, not the namespace device - await nvmeof.rescanNamespace(controllerDevice); - let namespaceDevice; try { await GeneralUtils.retry(15, 2000, async () => { + // rescan in scenarios when login previously occurred but volumes never appeared + // must be the NVMe char device, not the namespace device + await nvmeof.rescanNamespace(controllerDevice); + namespaceDevice = await nvmeof.namespaceDevicePathByTransportNQNNamespace( nvmeofConnection.transport, @@ -3499,6 +3498,8 @@ class CsiBaseDriver { const driver = this; const mount = driver.getDefaultMountInstance(); const filesystem = driver.getDefaultFilesystemInstance(); + const nvmeof = driver.getDefaultNVMEoFInstance(); + let device; let fs_info; let device_path; @@ -3561,6 +3562,14 @@ class CsiBaseDriver { rescan_devices.push(device); for (let sdevice of rescan_devices) { + let is_nvmeof = await filesystem.deviceIsNVMEoF(sdevice); + if (is_nvmeof) { + let controllers = + await nvmeof.getControllersByNamespaceDeviceName(sdevice); + for (let controller of controllers) { + await nvmeof.rescanNamespace(`/dev/${controller.Controller}`); + } + } // TODO: technically rescan is only relevant/available for remote drives // such as iscsi etc, should probably limit this call as appropriate // for now crudely checking the scenario inside the method itself diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index 5e65622..f3aa621 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -153,6 +153,54 @@ class NVMEoF { await nvmeof.exec(nvmeof.options.paths.nvme, args); } + async deviceIsNamespaceDevice(device) { + const nvmeof = this; + device = device.replace("/dev/", ""); + const subsystems = await nvmeof.getSubsystems(); + for (let subsystem of subsystems) { + // check subsystem namespaces + if (subsystem.Namespaces) { + for (let namespace of subsystem.Namespaces) { + if (namespace.NameSpace == device) { + return true; + } + } + } + + // check controller namespaces + if (subsystem.Controllers) { + for (let controller of subsystem.Controllers) { + if (controller.Namespaces) { + for (let namespace of controller.Namespaces) { + if (namespace.NameSpace == device) { + return true; + } + } + } + } + } + } + + return false; + } + + async deviceIsControllerDevice(device) { + const nvmeof = this; + device = device.replace("/dev/", ""); + const subsystems = await nvmeof.getSubsystems(); + for (let subsystem of subsystems) { + if (subsystem.Controllers) { + for (let controller of subsystem.Controllers) { + if (controller.Controller == device) { + return true; + } + } + } + } + + return false; + } + async parseTransport(transport) { if (typeof transport === "object") { return transport; @@ -267,9 +315,9 @@ class NVMEoF { /** * used to normalize subsystem list/response across different versions of nvme-cli - * - * @param {*} result - * @returns + * + * @param {*} result + * @returns */ async getNormalizedSubsystems(result) { let subsystems = []; @@ -298,6 +346,44 @@ class NVMEoF { nvmeof.logger.warn(`failed to find subsystem for nqn: ${nqn}`); } + async getControllersByNamespaceDeviceName(name) { + const nvmeof = this; + name = name.replace("/dev/", ""); + let nativeMultipathEnabled = await nvmeof.nativeMultipathEnabled(); + const subsystems = await nvmeof.getSubsystems(); + + if (nativeMultipathEnabled) { + // using per-subsystem namespace + for (let subsystem of subsystems) { + if (subsystem.Namespaces) { + for (let namespace of subsystem.Namespaces) { + if (namespace.NameSpace == name) { + return subsystem.Controllers; + } + } + } + } + } else { + // using per-controller namespace + for (let subsystem of subsystems) { + if (subsystem.Controllers) { + for (let controller of subsystem.Controllers) { + if (controller.Namespaces) { + for (let namespace of controller.Namespaces) { + if (namespace.NameSpace == name) { + return subsystem.Controllers; + } + } + } + } + } + } + } + + nvmeof.logger.warn(`failed to find controllers for device: ${name}`); + return []; + } + async getControllerByTransportNQN(transport, nqn) { const nvmeof = this; transport = await nvmeof.parseTransport(transport); From fc875eb2f213b058a02362b00ea83feb8e2273ee Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Wed, 11 Jan 2023 05:16:38 -0700 Subject: [PATCH 15/20] dep bumps, fix address parsing Signed-off-by: Travis Glenn Hansen --- package-lock.json | 192 ++++++++++++++++++++++---------------------- src/utils/nvmeof.js | 2 +- 2 files changed, 97 insertions(+), 97 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4872134..feeedb4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,9 +51,9 @@ } }, "node_modules/@eslint/eslintrc": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.4.0.tgz", - "integrity": "sha512-7yfvXy6MWLgWSFsLhz5yH3iQ52St8cdUY6FoGieKkRDVxuxmrNuUetIuu6cmjNWwniUHiWXjxCr5tTXDrbYS5A==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.4.1.tgz", + "integrity": "sha512-XXrH9Uarn0stsyldqDYq8r++mROmWRI1xKMXa640Bb//SY1+ECYX6VzT6Lcx5frD0V30XieqJ0oX9I2Xj5aoMA==", "dev": true, "dependencies": { "ajv": "^6.12.4", @@ -74,9 +74,9 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.7.3.tgz", - "integrity": "sha512-H9l79u4kJ2PVSxUNA08HMYAnUBLj9v6KjYQ7SQ71hOZcEXhShE/y5iQCesP8+6/Ik/7i2O0a10bPquIcYfufog==", + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.2.tgz", + "integrity": "sha512-5cqCjUvDKJWHGeu1prlrFOUmjuML0NequZKJ38PsCkfwIqPnZq4Q9burPP3It7/+46wpl0KsqVN3s6Te3B9Qtw==", "dependencies": { "@grpc/proto-loader": "^0.7.0", "@types/node": ">=12.12.47" @@ -309,9 +309,9 @@ "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" }, "node_modules/@types/node": { - "version": "18.11.17", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.17.tgz", - "integrity": "sha512-HJSUJmni4BeDHhfzn6nF0sVmd1SMezP7/4F0Lq+aXzmp2xm9O7WXrUtHW/CHlYVtZUbByEvWidHqRtcJXGF2Ng==" + "version": "18.11.18", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.18.tgz", + "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==" }, "node_modules/@types/request": { "version": "2.48.8", @@ -324,19 +324,6 @@ "form-data": "^2.5.0" } }, - "node_modules/@types/request/node_modules/form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 0.12" - } - }, "node_modules/@types/tough-cookie": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", @@ -456,20 +443,33 @@ } }, "node_modules/aws4": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", - "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz", + "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==" }, "node_modules/axios": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.2.1.tgz", - "integrity": "sha512-I88cFiGu9ryt/tfVEi4kX2SITsvDddTajXTOFmt2uK1ZVA8LytjtdeyefdQWEf5PU8w+4SSJDoYnggflB5tW4A==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.2.2.tgz", + "integrity": "sha512-bz/J4gS2S3I7mpN/YZfGFTqhXTYzRho8Ay38w2otuuDR322KzFIWm/4W2K6gIwvWaws5n+mnb7D1lN9uD+QH6Q==", "dependencies": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -804,12 +804,12 @@ } }, "node_modules/eslint": { - "version": "8.30.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.30.0.tgz", - "integrity": "sha512-MGADB39QqYuzEGov+F/qb18r4i7DohCDOfatHaxI2iGlPuC65bwG2gxgO+7DkyL38dRFaRH7RaRAgU6JKL9rMQ==", + "version": "8.31.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.31.0.tgz", + "integrity": "sha512-0tQQEVdmPZ1UtUKXjX7EMm9BlgJ08G90IhWh0PKDCb3ZLsgAOHI8fYSIzYVZej92zsgq+ft0FGsxhJ3xo2tbuA==", "dev": true, "dependencies": { - "@eslint/eslintrc": "^1.4.0", + "@eslint/eslintrc": "^1.4.1", "@humanwhocodes/config-array": "^0.11.8", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", @@ -1027,9 +1027,9 @@ "dev": true }, "node_modules/fastq": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.14.0.tgz", - "integrity": "sha512-eR2D+V9/ExcbF9ls441yIuN6TI2ED1Y2ZcA5BmMtJsOkWOFRJQ0Jt0g1UwqXJJVAb+V+umH5Dfr8oh4EVP7VVg==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", + "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", "dev": true, "dependencies": { "reusify": "^1.0.4" @@ -1120,16 +1120,16 @@ } }, "node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", "dependencies": { "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", + "combined-stream": "^1.0.6", "mime-types": "^2.1.12" }, "engines": { - "node": ">= 6" + "node": ">= 0.12" } }, "node_modules/fs-extra": { @@ -1489,9 +1489,9 @@ "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, "node_modules/jose": { - "version": "4.11.1", - "resolved": "https://registry.npmjs.org/jose/-/jose-4.11.1.tgz", - "integrity": "sha512-YRv4Tk/Wlug8qicwqFNFVEZSdbROCHRAC6qu/i0dyNKr5JQdoa2pIGoS04lLO/jXQX7Z9omoNewYIVIxqZBd9Q==", + "version": "4.11.2", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.11.2.tgz", + "integrity": "sha512-njj0VL2TsIxCtgzhO+9RRobBvws4oYyCM8TpvoUQwl/MbIM3NFJRR9+e6x0sS5xXaP1t6OCBkaBME98OV9zU5A==", "optional": true, "funding": { "url": "https://github.com/sponsors/panva" @@ -2110,9 +2110,9 @@ "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" }, "node_modules/punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.2.0.tgz", + "integrity": "sha512-LN6QV1IJ9ZhxWTNdktaPClrNfp8xdSAYS0Zk2ddX7XsXZAxckMHPCBcHRo0cTcEIgYPRiGEkmji3Idkh2yFtYw==", "engines": { "node": ">=6" } @@ -2364,9 +2364,9 @@ "optional": true }, "node_modules/safe-stable-stringify": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.1.tgz", - "integrity": "sha512-dVHE6bMtS/bnL2mwualjc6IxEv1F+OCUpA46pKUj6F8uDbUM0jCCulPqRNPSnWwGNKx5etqMjZYdXtrm5KJZGA==", + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.2.tgz", + "integrity": "sha512-gMxvPJYhP0O9n2pvcfYfIuYgbledAOJFcqRThtPRmjscaipiwcwPPKLytpVzMkG2HAN87Qmo2d4PtGiri1dSLA==", "engines": { "node": ">=10" } @@ -2938,9 +2938,9 @@ } }, "@eslint/eslintrc": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.4.0.tgz", - "integrity": "sha512-7yfvXy6MWLgWSFsLhz5yH3iQ52St8cdUY6FoGieKkRDVxuxmrNuUetIuu6cmjNWwniUHiWXjxCr5tTXDrbYS5A==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.4.1.tgz", + "integrity": "sha512-XXrH9Uarn0stsyldqDYq8r++mROmWRI1xKMXa640Bb//SY1+ECYX6VzT6Lcx5frD0V30XieqJ0oX9I2Xj5aoMA==", "dev": true, "requires": { "ajv": "^6.12.4", @@ -2955,9 +2955,9 @@ } }, "@grpc/grpc-js": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.7.3.tgz", - "integrity": "sha512-H9l79u4kJ2PVSxUNA08HMYAnUBLj9v6KjYQ7SQ71hOZcEXhShE/y5iQCesP8+6/Ik/7i2O0a10bPquIcYfufog==", + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.2.tgz", + "integrity": "sha512-5cqCjUvDKJWHGeu1prlrFOUmjuML0NequZKJ38PsCkfwIqPnZq4Q9burPP3It7/+46wpl0KsqVN3s6Te3B9Qtw==", "requires": { "@grpc/proto-loader": "^0.7.0", "@types/node": ">=12.12.47" @@ -3158,9 +3158,9 @@ "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" }, "@types/node": { - "version": "18.11.17", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.17.tgz", - "integrity": "sha512-HJSUJmni4BeDHhfzn6nF0sVmd1SMezP7/4F0Lq+aXzmp2xm9O7WXrUtHW/CHlYVtZUbByEvWidHqRtcJXGF2Ng==" + "version": "18.11.18", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.18.tgz", + "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==" }, "@types/request": { "version": "2.48.8", @@ -3171,18 +3171,6 @@ "@types/node": "*", "@types/tough-cookie": "*", "form-data": "^2.5.0" - }, - "dependencies": { - "form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - } } }, "@types/tough-cookie": { @@ -3277,18 +3265,30 @@ "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==" }, "aws4": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", - "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz", + "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==" }, "axios": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.2.1.tgz", - "integrity": "sha512-I88cFiGu9ryt/tfVEi4kX2SITsvDddTajXTOFmt2uK1ZVA8LytjtdeyefdQWEf5PU8w+4SSJDoYnggflB5tW4A==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.2.2.tgz", + "integrity": "sha512-bz/J4gS2S3I7mpN/YZfGFTqhXTYzRho8Ay38w2otuuDR322KzFIWm/4W2K6gIwvWaws5n+mnb7D1lN9uD+QH6Q==", "requires": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" + }, + "dependencies": { + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + } } }, "balanced-match": { @@ -3551,12 +3551,12 @@ "dev": true }, "eslint": { - "version": "8.30.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.30.0.tgz", - "integrity": "sha512-MGADB39QqYuzEGov+F/qb18r4i7DohCDOfatHaxI2iGlPuC65bwG2gxgO+7DkyL38dRFaRH7RaRAgU6JKL9rMQ==", + "version": "8.31.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.31.0.tgz", + "integrity": "sha512-0tQQEVdmPZ1UtUKXjX7EMm9BlgJ08G90IhWh0PKDCb3ZLsgAOHI8fYSIzYVZej92zsgq+ft0FGsxhJ3xo2tbuA==", "dev": true, "requires": { - "@eslint/eslintrc": "^1.4.0", + "@eslint/eslintrc": "^1.4.1", "@humanwhocodes/config-array": "^0.11.8", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", @@ -3719,9 +3719,9 @@ "dev": true }, "fastq": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.14.0.tgz", - "integrity": "sha512-eR2D+V9/ExcbF9ls441yIuN6TI2ED1Y2ZcA5BmMtJsOkWOFRJQ0Jt0g1UwqXJJVAb+V+umH5Dfr8oh4EVP7VVg==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", + "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", "dev": true, "requires": { "reusify": "^1.0.4" @@ -3783,12 +3783,12 @@ "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==" }, "form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", "requires": { "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", + "combined-stream": "^1.0.6", "mime-types": "^2.1.12" } }, @@ -4052,9 +4052,9 @@ "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, "jose": { - "version": "4.11.1", - "resolved": "https://registry.npmjs.org/jose/-/jose-4.11.1.tgz", - "integrity": "sha512-YRv4Tk/Wlug8qicwqFNFVEZSdbROCHRAC6qu/i0dyNKr5JQdoa2pIGoS04lLO/jXQX7Z9omoNewYIVIxqZBd9Q==", + "version": "4.11.2", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.11.2.tgz", + "integrity": "sha512-njj0VL2TsIxCtgzhO+9RRobBvws4oYyCM8TpvoUQwl/MbIM3NFJRR9+e6x0sS5xXaP1t6OCBkaBME98OV9zU5A==", "optional": true }, "js-sdsl": { @@ -4549,9 +4549,9 @@ "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" }, "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.2.0.tgz", + "integrity": "sha512-LN6QV1IJ9ZhxWTNdktaPClrNfp8xdSAYS0Zk2ddX7XsXZAxckMHPCBcHRo0cTcEIgYPRiGEkmji3Idkh2yFtYw==" }, "qs": { "version": "6.5.3", @@ -4706,9 +4706,9 @@ "optional": true }, "safe-stable-stringify": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.1.tgz", - "integrity": "sha512-dVHE6bMtS/bnL2mwualjc6IxEv1F+OCUpA46pKUj6F8uDbUM0jCCulPqRNPSnWwGNKx5etqMjZYdXtrm5KJZGA==" + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.2.tgz", + "integrity": "sha512-gMxvPJYhP0O9n2pvcfYfIuYgbledAOJFcqRThtPRmjscaipiwcwPPKLytpVzMkG2HAN87Qmo2d4PtGiri1dSLA==" }, "safer-buffer": { "version": "2.1.2", diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index f3aa621..26de8f6 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -401,7 +401,7 @@ class NVMEoF { * new: "Address":"traddr=127.0.0.1,trsvcid=4420" */ controllerAddress = controllerAddress.replace( - new RegExp(/ ([a-z]*=)/, "g"), + new RegExp(/ ([a-z_]*=)/, "g"), ",$1" ); let parts = controllerAddress.split(","); From 43284bb5a2e9cf7d91c2a3e0fc6116e7c9c0b8f1 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Wed, 11 Jan 2023 17:53:42 -0700 Subject: [PATCH 16/20] implement custom connect args use transport query string(s) Signed-off-by: Travis Glenn Hansen --- package-lock.json | 12 ++++++------ src/utils/nvmeof.js | 22 ++++++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index feeedb4..9ad3d8e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -74,9 +74,9 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.2.tgz", - "integrity": "sha512-5cqCjUvDKJWHGeu1prlrFOUmjuML0NequZKJ38PsCkfwIqPnZq4Q9burPP3It7/+46wpl0KsqVN3s6Te3B9Qtw==", + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.7.3.tgz", + "integrity": "sha512-H9l79u4kJ2PVSxUNA08HMYAnUBLj9v6KjYQ7SQ71hOZcEXhShE/y5iQCesP8+6/Ik/7i2O0a10bPquIcYfufog==", "dependencies": { "@grpc/proto-loader": "^0.7.0", "@types/node": ">=12.12.47" @@ -2955,9 +2955,9 @@ } }, "@grpc/grpc-js": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.2.tgz", - "integrity": "sha512-5cqCjUvDKJWHGeu1prlrFOUmjuML0NequZKJ38PsCkfwIqPnZq4Q9burPP3It7/+46wpl0KsqVN3s6Te3B9Qtw==", + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.7.3.tgz", + "integrity": "sha512-H9l79u4kJ2PVSxUNA08HMYAnUBLj9v6KjYQ7SQ71hOZcEXhShE/y5iQCesP8+6/Ik/7i2O0a10bPquIcYfufog==", "requires": { "@grpc/proto-loader": "^0.7.0", "@types/node": ">=12.12.47" diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index 26de8f6..63ab2a5 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -1,6 +1,7 @@ const cp = require("child_process"); const { hostname_lookup, trimchar } = require("./general"); const URI = require("uri-js"); +const querystring = require("querystring"); const DEFAULT_TIMEOUT = process.env.NVMEOF_DEFAULT_TIMEOUT || 30000; @@ -102,6 +103,17 @@ class NVMEoF { transport_args.push("--trsvcid", transport.service); } + if (transport.args) { + for (let arg in transport.args) { + let value = transport.args[arg]; + if (!arg.startsWith("-")) { + arg = `--${arg}`; + } + + transport_args.push(arg, value); + } + } + args.unshift("connect", "--nqn", nqn, ...transport_args); try { @@ -208,6 +220,7 @@ class NVMEoF { transport = transport.trim(); const parsed = URI.parse(transport); + let args = querystring.parse(parsed.query); let type = parsed.scheme; let address = parsed.host; @@ -256,6 +269,7 @@ class NVMEoF { type, address, service, + args, }; } @@ -482,14 +496,6 @@ class NVMEoF { return `/dev/disk/by-id/nvme-${modelNumber}_${serialNumber}`; } - devicePathByPortalIQNLUN(portal, iqn, lun) { - const parsedPortal = this.parsePortal(portal); - const portalHost = parsedPortal.host - .replaceAll("[", "") - .replaceAll("]", ""); - return `/dev/disk/by-path/ip-${portalHost}:${parsedPortal.port}-iscsi-${iqn}-lun-${lun}`; - } - exec(command, args, options = {}) { if (!options.hasOwnProperty("timeout")) { options.timeout = DEFAULT_TIMEOUT; From 10af6c639b430af106be3ea769fcaddec1e5e9d3 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Fri, 13 Jan 2023 11:24:11 -0700 Subject: [PATCH 17/20] implement gate logic to prevent nvmet commands until we are sure the config has been loaded Signed-off-by: Travis Glenn Hansen --- contrib/scale-nvmet-start.sh | 3 +++ package-lock.json | 14 +++++++------- package.json | 2 +- src/driver/controller-zfs-generic/index.js | 21 +++++++++++++++++++++ 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/contrib/scale-nvmet-start.sh b/contrib/scale-nvmet-start.sh index 42fbae8..a2cfc22 100755 --- a/contrib/scale-nvmet-start.sh +++ b/contrib/scale-nvmet-start.sh @@ -56,3 +56,6 @@ which nvmetcli &>/dev/null || { cd "${SCRIPTDIR}" nvmetcli restore "${NVMETCONFIG}" + +touch /var/run/nvmet-config-loaded +chmod +r /var/run/nvmet-config-loaded diff --git a/package-lock.json b/package-lock.json index 9ad3d8e..d504f2c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.8.0", "license": "MIT", "dependencies": { - "@grpc/grpc-js": "^1.7.3", + "@grpc/grpc-js": "^1.8.4", "@grpc/proto-loader": "^0.7.0", "@kubernetes/client-node": "^0.18.0", "async-mutex": "^0.4.0", @@ -74,9 +74,9 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.7.3.tgz", - "integrity": "sha512-H9l79u4kJ2PVSxUNA08HMYAnUBLj9v6KjYQ7SQ71hOZcEXhShE/y5iQCesP8+6/Ik/7i2O0a10bPquIcYfufog==", + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.4.tgz", + "integrity": "sha512-oaETBotls7FTBpySg5dhyUCyXSxSeCMmkBBXHXG1iw57MiNoB6D7VRhkrXYbwyHM3Q3Afjp4KlsBX0Zb+ELZXw==", "dependencies": { "@grpc/proto-loader": "^0.7.0", "@types/node": ">=12.12.47" @@ -2955,9 +2955,9 @@ } }, "@grpc/grpc-js": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.7.3.tgz", - "integrity": "sha512-H9l79u4kJ2PVSxUNA08HMYAnUBLj9v6KjYQ7SQ71hOZcEXhShE/y5iQCesP8+6/Ik/7i2O0a10bPquIcYfufog==", + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.4.tgz", + "integrity": "sha512-oaETBotls7FTBpySg5dhyUCyXSxSeCMmkBBXHXG1iw57MiNoB6D7VRhkrXYbwyHM3Q3Afjp4KlsBX0Zb+ELZXw==", "requires": { "@grpc/proto-loader": "^0.7.0", "@types/node": ">=12.12.47" diff --git a/package.json b/package.json index 6e20bc9..119f8c4 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "url": "https://github.com/democratic-csi/democratic-csi.git" }, "dependencies": { - "@grpc/grpc-js": "^1.7.3", + "@grpc/grpc-js": "^1.8.4", "@grpc/proto-loader": "^0.7.0", "@kubernetes/client-node": "^0.18.0", "async-mutex": "^0.4.0", diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js index 08ebdb4..af30fdc 100644 --- a/src/driver/controller-zfs-generic/index.js +++ b/src/driver/controller-zfs-generic/index.js @@ -897,6 +897,27 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath} const execClient = this.getExecClient(); const driver = this; + if ( + _.get( + this.options, + "nvmeof.shareStrategyNvmetCli.configIsImportedFilePath" + ) + ) { + try { + let response = await execClient.exec( + execClient.buildCommand("test", [ + "-f", + _.get( + this.options, + "nvmeof.shareStrategyNvmetCli.configIsImportedFilePath" + ), + ]) + ); + } catch (err) { + throw new Error("nvmet has not been fully configured"); + } + } + data = data.trim(); let command = "sh"; From 9baea05ca26cf75821c91e3b3a57d1eb7d6667dc Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Mon, 16 Jan 2023 12:46:53 -0700 Subject: [PATCH 18/20] default ext volumes to use 0 reserved space Signed-off-by: Travis Glenn Hansen --- src/driver/index.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/driver/index.js b/src/driver/index.js index 1648c7f..7edf891 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -1309,6 +1309,16 @@ class CsiBaseDriver { if (!Array.isArray(formatOptions)) { formatOptions = []; } + + switch (fs_type) { + case "ext3": + case "ext4": + case "ext4dev": + // disable reserved blocks in this scenario + formatOptions.unshift("-m", "0"); + break; + } + await filesystem.formatDevice(device, fs_type, formatOptions); } From 67b5e06f45c058689ae5ac8e6143fa780b92acfa Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Thu, 23 Feb 2023 22:11:05 -0700 Subject: [PATCH 19/20] nvmeof documentation Signed-off-by: Travis Glenn Hansen --- CHANGELOG.md | 13 +++ README.md | 183 +++++++++++++++++++++++++------ examples/lustre-client.yaml | 1 + examples/zfs-generic-nvmeof.yaml | 102 +++++++++++++++++ src/utils/windows.js | 3 + 5 files changed, 266 insertions(+), 36 deletions(-) create mode 100644 examples/zfs-generic-nvmeof.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 16fa8d5..2961a6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +# v1.8.0 + +Released 2023-02-23 + +- `nvmeof` support + +# v1.7.7 + +Released 2022-10-17 + +- support `csi.access_modes` config value in all zfs-based drivers +- bump deps + # v1.7.6 Released 2022-08-06 diff --git a/README.md b/README.md index 52c21f1..974f899 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ ![Image](https://img.shields.io/docker/pulls/democraticcsi/democratic-csi.svg) -![Image](https://img.shields.io/github/workflow/status/democratic-csi/democratic-csi/CI?style=flat-square) +![Image](https://img.shields.io/github/actions/workflow/status/democratic-csi/democratic-csi/main.yml?branch=master&style=flat-square) [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/democratic-csi)](https://artifacthub.io/packages/search?repo=democratic-csi) # Introduction @@ -24,6 +24,7 @@ have access to resizing, snapshots, clones, etc functionality. - `freenas-api-smb` experimental use with SCALE only (manages zfs datasets to share over smb) - `zfs-generic-nfs` (works with any ZoL installation...ie: Ubuntu) - `zfs-generic-iscsi` (works with any ZoL installation...ie: Ubuntu) + - `zfs-generic-nvmeof` (works with any ZoL installation...ie: Ubuntu) - `zfs-local-ephemeral-inline` (provisions node-local zfs datasets) - `zfs-local-dataset` (provision node-local volume as dataset) - `zfs-local-zvol` (provision node-local volume as zvol) @@ -67,21 +68,21 @@ You should install/configure the requirements for both nfs and iscsi. ### cifs -``` -RHEL / CentOS +```bash +# RHEL / CentOS sudo yum install -y cifs-utils -Ubuntu / Debian +# Ubuntu / Debian sudo apt-get install -y cifs-utils ``` ### nfs -``` -RHEL / CentOS +```bash +# RHEL / CentOS sudo yum install -y nfs-utils -Ubuntu / Debian +# Ubuntu / Debian sudo apt-get install -y nfs-common ``` @@ -96,7 +97,7 @@ If you are running Kubernetes with rancher/rke please see the following: #### RHEL / CentOS -``` +```bash # Install the following system packages sudo yum install -y lsscsi iscsi-initiator-utils sg3_utils device-mapper-multipath @@ -135,32 +136,40 @@ sudo systemctl enable open-iscsi.service sudo service open-iscsi start sudo systemctl status open-iscsi ``` + #### [Talos](https://www.talos.dev/) + To use iscsi storage in kubernetes cluster in talos these steps are needed which are similar to the ones explained in https://www.talos.dev/v1.1/kubernetes-guides/configuration/replicated-local-storage-with-openebs-jiva/#patching-the-jiva-installation ##### Patch nodes + since talos does not have iscsi support by default, the iscsi extension is needed create a `patch.yaml` file with + ```yaml - op: add path: /machine/install/extensions value: - image: ghcr.io/siderolabs/iscsi-tools:v0.1.1 ``` + and apply the patch across all of your nodes + ```bash talosctl -e -n patch mc -p @patch.yaml ``` + the extension will not activate until you "upgrade" the nodes, even if there is no update, use the latest version of talos installer. VERIFY THE TALOS VERSION IN THIS COMMAND BEFORE RUNNING IT AND READ THE [OpenEBS Jiva](https://www.talos.dev/v1.1/kubernetes-guides/configuration/replicated-local-storage-with-openebs-jiva/#patching-the-jiva-installation). upgrade all of the nodes in the cluster to get the extension + ```bash talosctl -e -n upgrade --image=ghcr.io/siderolabs/installer:v1.1.1 ``` in your `values.yaml` file make sure to enable these settings -```yaml +```yaml node: hostPID: true driver: @@ -172,17 +181,33 @@ node: iscsiDirHostPath: /usr/local/etc/iscsi iscsiDirHostPathType: "" ``` + and continue your democratic installation as usuall with other iscsi drivers. +### nvmeof -### freenas-smb +```bash +# not required but likely helpful (tools are included in the democratic images +# so not needed on the host) +apt-get install -y nvme-cli -If using with Windows based machines you may need to enable guest access (even -if you are connecting with credentials) +# get the nvme fabric modules +apt-get install linux-generic -``` -Set-ItemProperty HKLM:\SYSTEM\CurrentControlSet\Services\LanmanWorkstation\Parameters AllowInsecureGuestAuth -Value 1 -Restart-Service LanmanWorkstation -Force +# ensure the nvmeof modules get loaded at boot +cat < /etc/modules-load.d/nvme.conf +nvme +nvme-tcp +nvme-fc +nvme-rdma +EOF + +# nvme has native multipath or can use DM multipath. +# RedHat recommends DM multipath (nvme_core.multipath=N) +cat /sys/module/nvme_core/parameters/multipath + +# kernel arg to enable/disable native multipath +nvme_core.multipath=Y ``` ### zfs-local-ephemeral-inline @@ -237,17 +262,43 @@ linux nodes as well (using the `ntfs3` driver) so volumes created can be utilized by nodes with either operating system (in the case of `cifs` by both simultaneously). +If using any `-iscsi` driver be sure your iqns are always fully lower-case by +default (https://github.com/PowerShell/PowerShell/issues/17306). + Due to current limits in the kubernetes tooling it is not possible to use the `local-hostpath` driver but support is implemented in this project and will work as soon as kubernetes support is available. -``` +```powershell # ensure all updates are installed # enable the container feature Enable-WindowsOptionalFeature -Online -FeatureName Containers –All # install a HostProcess compatible kubernetes + +# smb support +# If using with Windows based machines you may need to enable guest access +# (even if you are connecting with credentials) +Set-ItemProperty HKLM:\SYSTEM\CurrentControlSet\Services\LanmanWorkstation\Parameters AllowInsecureGuestAuth -Value 1 +Restart-Service LanmanWorkstation -Force + +# iscsi +# enable iscsi service and mpio as appropriate +Get-Service -Name MSiSCSI +Set-Service -Name MSiSCSI -StartupType Automatic +Start-Service -Name MSiSCSI +Get-Service -Name MSiSCSI + +# mpio +Get-WindowsFeature -Name 'Multipath-IO' +Add-WindowsFeature -Name 'Multipath-IO' + +Enable-MSDSMAutomaticClaim -BusType "iSCSI" +Disable-MSDSMAutomaticClaim -BusType "iSCSI" + +Get-MSDSMGlobalDefaultLoadBalancePolicy +Set-MSDSMGlobalLoadBalancePolicy -Policy RR ``` - https://kubernetes.io/blog/2021/08/16/windows-hostprocess-containers/ @@ -353,7 +404,7 @@ Issues to review: - https://jira.ixsystems.com/browse/NAS-108522 - https://jira.ixsystems.com/browse/NAS-107219 -### ZoL (zfs-generic-nfs, zfs-generic-iscsi, zfs-generic-smb) +### ZoL (zfs-generic-nfs, zfs-generic-iscsi, zfs-generic-smb, zfs-generic-nvmeof) Ensure ssh and zfs is installed on the nfs/iscsi server and that you have installed `targetcli`. @@ -367,7 +418,7 @@ unecessarily: - https://github.com/democratic-csi/democratic-csi/issues/151 (some notes on using delegated zfs permissions) -``` +```bash ####### nfs yum install -y nfs-utils systemctl enable --now nfs-server.service @@ -389,6 +440,71 @@ passwd smbroot (optional) # create smb user and set password smbpasswd -L -a smbroot + +####### nvmeof +# install nvmetcli and systemd services +git clone git://git.infradead.org/users/hch/nvmetcli.git +cd nvmetcli + +## install globally +python3 setup.py install --prefix=/usr +pip install configshell_fb + +## install to root home dir +python3 setup.py install --user +pip install configshell_fb --user + +# prevent log files from filling up disk +ln -sf /dev/null ~/.nvmetcli/log.txt +ln -sf /dev/null ~/.nvmetcli/history.txt + +# install systemd unit and enable/start +## optionally to ensure the config file is loaded before we start +## reading/writing to it add an ExecStartPost= to the unit file +## +## ExecStartPost=/usr/bin/touch /var/run/nvmet-config-loaded +## +## in your dirver config set nvmeof.shareStrategyNvmetCli.configIsImportedFilePath=/var/run/nvmet-config-loaded +## which will prevent the driver from making any changes until the configured +## file is present +vi nvmet.service + +cp nvmet.service /etc/systemd/system/ +mkdir -p /etc/nvmet +systemctl daemon-reload +systemctl enable --now nvmet.service +systemctl status nvmet.service + +# ensure nvmeof target modules are loaded at startup +cat < /etc/modules-load.d/nvmet.conf +nvmet +nvmet-tcp +nvmet-fc +nvmet-rdma +EOF + +# create the port(s) configuration manually +echo " +cd / +ls +" | nvmetcli + +# do this multiple times altering as appropriate if you have/want multipath +# change the port to 2, 3.. each additional path +# the below example creates a tcp port listening on all IPs on port 4420 +echo " +cd /ports +create 1 +cd 1 +set addr adrfam=ipv4 trtype=tcp traddr=0.0.0.0 trsvcid=4420 + +saveconfig /etc/nvmet/config.json +" | nvmetcli + +# if running TrueNAS SCALE you can skip the above and simply copy +# contrib/scale-nvmet-start.sh to your machine and add it as a startup script +# to launch POSTINIT type COMMAND +# and then create the port(s) as mentioned above ``` ### Synology (synology-iscsi) @@ -397,7 +513,7 @@ Ensure iscsi manager has been installed and is generally setup/configured. DSM 6 ## Helm Installation -``` +```bash helm repo add democratic-csi https://democratic-csi.github.io/charts/ helm repo update # helm v2 @@ -441,13 +557,14 @@ microk8s helm upgrade \ - microk8s - `/var/snap/microk8s/common/var/lib/kubelet` - pivotal - `/var/vcap/data/kubelet` +- k0s - `/var/lib/k0s/kubelet` ### openshift `democratic-csi` generally works fine with openshift. Some special parameters need to be set with helm (support added in chart version `0.6.1`): -``` +```bash # for sure required --set node.rbac.openshift.privileged=true --set node.driver.localtimeHostPath=false @@ -461,6 +578,11 @@ need to be set with helm (support added in chart version `0.6.1`): `democratic-csi` works with Nomad in a functioning but limted capacity. See the [Nomad docs](docs/nomad.md) for details. +### Docker Swarm + +- https://github.com/moby/moby/blob/master/docs/cluster_volumes.md +- https://github.com/olljanat/csi-plugins-for-docker-swarm + ## Multiple Deployments You may install multiple deployments of each/any driver. It requires the @@ -479,25 +601,14 @@ following: # Snapshot Support -Install beta (v1.17+) CRDs (once per cluster): - -- https://github.com/kubernetes-csi/external-snapshotter/tree/master/client/config/crd - -``` -kubectl apply -f snapshot.storage.k8s.io_volumesnapshotclasses.yaml -kubectl apply -f snapshot.storage.k8s.io_volumesnapshotcontents.yaml -kubectl apply -f snapshot.storage.k8s.io_volumesnapshots.yaml -``` - Install snapshot controller (once per cluster): -- https://github.com/kubernetes-csi/external-snapshotter/tree/master/deploy/kubernetes/snapshot-controller +- https://github.com/democratic-csi/charts/tree/master/stable/snapshot-controller -``` -# replace namespace references to your liking -kubectl apply -f rbac-snapshot-controller.yaml -kubectl apply -f setup-snapshot-controller.yaml -``` +OR + +- https://github.com/kubernetes-csi/external-snapshotter/tree/master/client/config/crd +- https://github.com/kubernetes-csi/external-snapshotter/tree/master/deploy/kubernetes/snapshot-controller Install `democratic-csi` as usual with `volumeSnapshotClasses` defined as appropriate. diff --git a/examples/lustre-client.yaml b/examples/lustre-client.yaml index 7a331b4..2f7ea7c 100644 --- a/examples/lustre-client.yaml +++ b/examples/lustre-client.yaml @@ -1,6 +1,7 @@ driver: lustre-client instance_id: lustre: + # [:] shareHost: server address shareBasePath: "/some/path" # shareHost:shareBasePath should be mounted at this location in the controller container diff --git a/examples/zfs-generic-nvmeof.yaml b/examples/zfs-generic-nvmeof.yaml new file mode 100644 index 0000000..76a6c6f --- /dev/null +++ b/examples/zfs-generic-nvmeof.yaml @@ -0,0 +1,102 @@ +driver: zfs-generic-nvmeof +sshConnection: + host: server address + port: 22 + username: root + # use either password or key + password: "" + privateKey: | + -----BEGIN RSA PRIVATE KEY----- + ... + -----END RSA PRIVATE KEY----- + +zfs: + # can be used to override defaults if necessary + # the example below is useful for TrueNAS 12 + #cli: + # sudoEnabled: true + # paths: + # zfs: /usr/local/sbin/zfs + # zpool: /usr/local/sbin/zpool + # sudo: /usr/local/bin/sudo + # chroot: /usr/sbin/chroot + + # can be used to set arbitrary values on the dataset/zvol + # can use handlebars templates with the parameters from the storage class/CO + #datasetProperties: + # "org.freenas:description": "{{ parameters.[csi.storage.k8s.io/pvc/namespace] }}/{{ parameters.[csi.storage.k8s.io/pvc/name] }}" + # "org.freenas:test": "{{ parameters.foo }}" + # "org.freenas:test2": "some value" + + datasetParentName: tank/k8s/test + # do NOT make datasetParentName and detachedSnapshotsDatasetParentName overlap + # they may be siblings, but neither should be nested in the other + detachedSnapshotsDatasetParentName: tanks/k8s/test-snapshots + + # "" (inherit), lz4, gzip-9, etc + zvolCompression: + # "" (inherit), on, off, verify + zvolDedup: + zvolEnableReservation: false + # 512, 1K, 2K, 4K, 8K, 16K, 64K, 128K default is 16K + zvolBlocksize: + +nvmeof: + # these are for the node/client aspect + transports: + - tcp://server:port + #- "tcp://127.0.0.1:4420?host-iface=eth0" + #- "tcp://[2001:123:456::1]:4420" + #- "rdma://127.0.0.1:4420" + #- "fc://[nn-0x203b00a098cbcac6:pn-0x203d00a098cbcac6]" + + # MUST ensure uniqueness + # full iqn limit is 223 bytes, plan accordingly + # default is "{{ name }}" + #nameTemplate: "{{ parameters.[csi.storage.k8s.io/pvc/namespace] }}-{{ parameters.[csi.storage.k8s.io/pvc/name] }}" + namePrefix: + nameSuffix: + + shareStrategy: "nvmetCli" + #shareStrategy: "spdkCli" + + # https://documentation.suse.com/es-es/sles/15-SP1/html/SLES-all/cha-nvmeof.html + # https://www.linuxjournal.com/content/data-flash-part-iii-nvme-over-fabrics-using-tcp + # http://git.infradead.org/users/hch/nvmetcli.git + shareStrategyNvmetCli: + #sudoEnabled: true + #nvmetcliPath: nvmetcli + # prevent startup race conditions by ensuring the config on disk has been imported + # before we start messing with things + #configIsImportedFilePath: /var/run/nvmet-config-loaded + #configPath: /etc/nvmet/config.json + basename: "nqn.2003-01.org.linux-nvme" + # add more ports here as appropriate if you have multipath + ports: + - "1" + subsystem: + attributes: + allow_any_host: 1 + # not supported yet in nvmetcli + #namespace: + # attributes: + # buffered_io: 1 + + shareStrategySpdkCli: + # spdkcli.py + #spdkcliPath: spdkcli + configPath: /etc/spdk/spdk.json + basename: "nqn.2003-01.org.linux-nvmeof" + bdev: + type: uring + #type: aio + attributes: + block_size: 512 + subsystem: + attributes: + allow_any_host: "true" + listeners: + - trtype: tcp + traddr: server + trsvcid: port + adrfam: ipv4 diff --git a/src/utils/windows.js b/src/utils/windows.js index c02ceb3..3993c29 100644 --- a/src/utils/windows.js +++ b/src/utils/windows.js @@ -118,6 +118,9 @@ class Windows { // -UseWriteThrough $true // cannot have trailing slash nor a path // must be \\\ + // + // https://github.com/kubernetes-csi/csi-driver-smb/issues/219#issuecomment-781952587 + // -Persistent $false remotePath = this.uncPathToShare(remotePath); command = "$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential -RequirePrivacy $true"; From 8d83c1d6580111a67c3ac3b48150e0cac8a7feaa Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Fri, 24 Feb 2023 00:24:46 -0700 Subject: [PATCH 20/20] proper ci labels for SCALE 22.02 runners Signed-off-by: Travis Glenn Hansen --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8b70796..1feec5b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -129,8 +129,8 @@ jobs: - self-hosted - Linux - X64 - - csi-sanity-truenas - #- csi-sanity-zfs-generic + #- csi-sanity-truenas + - csi-sanity-zfs-generic steps: - uses: actions/checkout@v3 - uses: actions/download-artifact@v3