From abc3bbb96d0327feb16a8d3041994681bef40fc9 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Sun, 13 Mar 2022 12:07:48 -0600 Subject: [PATCH] handle target in use race condition Signed-off-by: Travis Glenn Hansen --- src/driver/freenas/ssh.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/driver/freenas/ssh.js b/src/driver/freenas/ssh.js index 11fa593..f9b0e98 100644 --- a/src/driver/freenas/ssh.js +++ b/src/driver/freenas/ssh.js @@ -4,6 +4,7 @@ const { GrpcError, grpc } = require("../../utils/grpc"); const SshClient = require("../../utils/ssh").SshClient; const HttpClient = require("./http").Client; const { Zetabyte, ZfsSshProcessManager } = require("../../utils/zfs"); +const sleep = require("../../utils/general").sleep; const Handlebars = require("handlebars"); @@ -1572,7 +1573,30 @@ class FreeNASSshDriver extends ControllerZfsBaseDriver { } if (deleteAsset) { + let retries = 0; + let maxRetries = 5; + let retryWait = 1000; response = await httpClient.delete(endpoint); + + // sometimes after an initiator has detached it takes a moment for TrueNAS to settle + // code: 422 body: {\"message\":\"Target csi-ci-55877e95sanity-node-expand-volume-e54f81fa-cd38e798 is in use.\",\"errno\":14} + while ( + response.statusCode == 422 && + retries < maxRetries && + _.get(response, "body.message").includes("Target") && + _.get(response, "body.message").includes("is in use") && + _.get(response, "body.errno") == 14 + ) { + retries++; + this.ctx.logger.debug( + "target: %s is in use, retry %s shortly", + targetId, + retries + ); + await sleep(retryWait); + response = await httpClient.delete(endpoint); + } + if (![200, 204].includes(response.statusCode)) { throw new GrpcError( grpc.status.UNKNOWN,