From 466845cbd91ef8f72c5c9d7f09f3aaed338d1651 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Sat, 7 May 2022 18:10:08 -0600 Subject: [PATCH] more complete usage of retry logic Signed-off-by: Travis Glenn Hansen --- src/driver/freenas/api.js | 44 ++++++++++++++++++++++++++++++-- src/driver/freenas/http/index.js | 6 +++++ src/driver/freenas/ssh.js | 44 ++++++++++++++++++++++++++++++-- src/utils/general.js | 24 ++++++++++++++++- 4 files changed, 113 insertions(+), 5 deletions(-) diff --git a/src/driver/freenas/api.js b/src/driver/freenas/api.js index 8788d6a..e082250 100644 --- a/src/driver/freenas/api.js +++ b/src/driver/freenas/api.js @@ -1402,7 +1402,27 @@ class FreeNASApiDriver extends CsiBaseDriver { }); if (deleteAsset) { - response = await httpClient.delete(endpoint); + response = await GeneralUtils.retry( + 3, + 1000, + async () => { + return await httpClient.delete(endpoint); + }, + { + retryCondition: (err) => { + if (err.code == "ECONNRESET") { + return true; + } + if (err.code == "ECONNABORTED") { + return true; + } + if (err.response && err.response.statusCode == 504) { + return true; + } + return false; + }, + } + ); // returns a 500 if does not exist // v1 = 204 @@ -1483,7 +1503,27 @@ class FreeNASApiDriver extends CsiBaseDriver { }); if (deleteAsset) { - response = await httpClient.delete(endpoint); + response = await GeneralUtils.retry( + 3, + 1000, + async () => { + return await httpClient.delete(endpoint); + }, + { + retryCondition: (err) => { + if (err.code == "ECONNRESET") { + return true; + } + if (err.code == "ECONNABORTED") { + return true; + } + if (err.response && err.response.statusCode == 504) { + return true; + } + return false; + }, + } + ); // returns a 500 if does not exist // v1 = 204 diff --git a/src/driver/freenas/http/index.js b/src/driver/freenas/http/index.js index d800e34..4d2009c 100644 --- a/src/driver/freenas/http/index.js +++ b/src/driver/freenas/http/index.js @@ -86,6 +86,12 @@ class Client { httpAgent: this.getHttpAgent(), httpsAgent: this.getHttpsAgent(), timeout: 60 * 1000, + validateStatus: function (status) { + if (status >= 500) { + return false; + } + return true; + }, }; if (client.options.apiKey) { diff --git a/src/driver/freenas/ssh.js b/src/driver/freenas/ssh.js index 249f05d..3ca8069 100644 --- a/src/driver/freenas/ssh.js +++ b/src/driver/freenas/ssh.js @@ -1450,7 +1450,27 @@ class FreeNASSshDriver extends ControllerZfsBaseDriver { }); if (deleteAsset) { - response = await httpClient.delete(endpoint); + response = await GeneralUtils.retry( + 3, + 1000, + async () => { + return await httpClient.delete(endpoint); + }, + { + retryCondition: (err) => { + if (err.code == "ECONNRESET") { + return true; + } + if (err.code == "ECONNABORTED") { + return true; + } + if (err.response && err.response.statusCode == 504) { + return true; + } + return false; + }, + } + ); // returns a 500 if does not exist // v1 = 204 @@ -1532,7 +1552,27 @@ class FreeNASSshDriver extends ControllerZfsBaseDriver { }); if (deleteAsset) { - response = await httpClient.delete(endpoint); + response = await GeneralUtils.retry( + 3, + 1000, + async () => { + return await httpClient.delete(endpoint); + }, + { + retryCondition: (err) => { + if (err.code == "ECONNRESET") { + return true; + } + if (err.code == "ECONNABORTED") { + return true; + } + if (err.response && err.response.statusCode == 504) { + return true; + } + return false; + }, + } + ); // returns a 500 if does not exist // v1 = 204 diff --git a/src/utils/general.js b/src/utils/general.js index 928d613..b08acae 100644 --- a/src/utils/general.js +++ b/src/utils/general.js @@ -140,7 +140,16 @@ function axios_request(options, callback = function () {}) { // The request was made and the server responded with a status code // that falls out of the range of 2xx let res = prep_response(err.response); - callback(null, res, res.body); + let senderr = false; + if ( + options.validateStatus && + typeof options.validateStatus == "function" + ) { + if (!options.validateStatus(res.statusCode)) { + senderr = true; + } + } + callback(senderr ? err : null, res, res.body); } else if (err.request) { // The request was made but no response was received // `error.request` is an instance of XMLHttpRequest in the browser and an instance of @@ -183,9 +192,12 @@ async function retry(retries, retriesDelay, code, options = {}) { let maxwait = _.get(options, "maxwait"); let logerrors = _.get(options, "logerrors", false); let retryCondition = options.retryCondition; + let executeStartTime; + do { current_try++; try { + executeStartTime = Date.now(); return await code(); } catch (err) { if (current_try >= retries) { @@ -202,6 +214,16 @@ async function retry(retries, retriesDelay, code, options = {}) { console.log(`retry - err:`, err); } } + // handle minExecutionTime + if (options.minExecutionTime > 0) { + let minDelayTime = + options.minExecutionTime - (Date.now() - executeStartTime); + if (minDelayTime > 0) { + await sleep(minDelayTime); + } + } + + // handle delay let sleep_time = retriesDelay; if (_.get(options, "exponential", false) === true) { sleep_time = retriesDelay * current_try;