diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 020cb61..f72131b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -113,7 +113,6 @@ jobs: SYNOLOGY_PASSWORD: ${{ secrets.SANITY_SYNOLOGY_PASSWORD }} SYNOLOGY_VOLUME: ${{ secrets.SANITY_SYNOLOGY_VOLUME }} - # api-based drivers csi-sanity-truenas-scale-22_02: needs: @@ -312,18 +311,18 @@ jobs: matrix: os: [Linux, Windows] include: - - os: Linux - npmartifact: node-modules-linux-amd64 - template: "./ci/configs/local-hostpath/basic.yaml" - run: | - # run tests - ci/bin/run.sh - - os: Windows - npmartifact: node-modules-windows-amd64 - template: ".\\ci\\configs\\local-hostpath/basic.yaml" - run: | - # run tests - ci\bin\run.ps1 + - os: Linux + npmartifact: node-modules-linux-amd64 + template: "./ci/configs/local-hostpath/basic.yaml" + run: | + # run tests + ci/bin/run.sh + - os: Windows + npmartifact: node-modules-windows-amd64 + template: ".\\ci\\configs\\local-hostpath/basic.yaml" + run: | + # run tests + ci\bin\run.ps1 runs-on: - self-hosted - ${{ matrix.os }} @@ -376,6 +375,7 @@ jobs: - csi-sanity-truenas-core-12_0 - csi-sanity-truenas-core-13_0 - csi-sanity-zfs-generic + - csi-sanity-client - csi-sanity-zfs-local - csi-sanity-local-hostpath - csi-sanity-windows-node diff --git a/src/driver/index.js b/src/driver/index.js index 2f82745..605d37d 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -2043,7 +2043,21 @@ class CsiBaseDriver { result = await mount.pathIsMounted(normalized_staging_path); if (result) { try { - result = await mount.umount(normalized_staging_path, umount_args); + result = await GeneralUtils.retry( + 10, + 0, + async () => { + return await mount.umount(normalized_staging_path, umount_args); + }, + { + minExecutionTime: 1000, + retryCondition: (err) => { + if (_.get(err, "stderr", "").includes("busy")) { + return true; + } + }, + } + ); } catch (err) { if (err.timeout) { driver.ctx.logger.warn( @@ -2188,13 +2202,41 @@ class CsiBaseDriver { // remove touched file result = await filesystem.pathExists(block_path); if (result) { - result = await filesystem.rm(block_path); + result = await GeneralUtils.retry( + 10, + 0, + async () => { + return await filesystem.rm(block_path); + }, + { + minExecutionTime: 1000, + retryCondition: (err) => { + if (_.get(err, "stderr", "").includes("busy")) { + return true; + } + }, + } + ); } } result = await filesystem.pathExists(staging_target_path); if (result) { - result = await filesystem.rmdir(staging_target_path); + result = await GeneralUtils.retry( + 10, + 0, + async () => { + return await filesystem.rmdir(staging_target_path); + }, + { + minExecutionTime: 1000, + retryCondition: (err) => { + if (_.get(err, "stderr", "").includes("busy")) { + return true; + } + }, + } + ); } break; case NODE_OS_DRIVER_WINDOWS: { @@ -2777,7 +2819,21 @@ class CsiBaseDriver { if (result) { try { - result = await mount.umount(target_path, umount_args); + result = await GeneralUtils.retry( + 10, + 0, + async () => { + return await mount.umount(target_path, umount_args); + }, + { + minExecutionTime: 1000, + retryCondition: (err) => { + if (_.get(err, "stderr", "").includes("busy")) { + return true; + } + }, + } + ); } catch (err) { if (err.timeout) { driver.ctx.logger.warn( @@ -2808,9 +2864,30 @@ class CsiBaseDriver { result = await filesystem.pathExists(target_path); if (result) { if (fs.lstatSync(target_path).isDirectory()) { - result = await filesystem.rmdir(target_path); + result = await GeneralUtils.retry( + 10, + 0, + async () => { + return await filesystem.rmdir(target_path); + }, + { + minExecutionTime: 1000, + retryCondition: (err) => { + if (_.get(err, "stderr", "").includes("busy")) { + return true; + } + }, + } + ); } else { - result = await filesystem.rm([target_path]); + result = await GeneralUtils.retry( + 10, + 0, + async () => { + return await filesystem.rm([target_path]); + }, + { minExecutionTime: 1000 } + ); } } diff --git a/src/utils/general.js b/src/utils/general.js index b08acae..746f2ef 100644 --- a/src/utils/general.js +++ b/src/utils/general.js @@ -145,9 +145,7 @@ function axios_request(options, callback = function () {}) { options.validateStatus && typeof options.validateStatus == "function" ) { - if (!options.validateStatus(res.statusCode)) { - senderr = true; - } + senderr = true; } callback(senderr ? err : null, res, res.body); } else if (err.request) { @@ -214,12 +212,14 @@ async function retry(retries, retriesDelay, code, options = {}) { console.log(`retry - err:`, err); } } + // handle minExecutionTime if (options.minExecutionTime > 0) { - let minDelayTime = - options.minExecutionTime - (Date.now() - executeStartTime); - if (minDelayTime > 0) { - await sleep(minDelayTime); + let executionElapsedTIme = Date.now() - executeStartTime; + let minExecutionDelayTime = + options.minExecutionTime - executionElapsedTIme; + if (minExecutionDelayTime > 0) { + await sleep(minExecutionDelayTime); } } @@ -234,8 +234,10 @@ async function retry(retries, retriesDelay, code, options = {}) { sleep_time = maxwait; } } - console.log(`retry - waiting ${sleep_time}ms before trying again`); - await sleep(sleep_time); + if (sleep_time > 0) { + console.log(`retry - waiting ${sleep_time}ms before trying again`); + await sleep(sleep_time); + } } while (true); }