synology updates, dsm6 and dsm7 in ci

Signed-off-by: Travis Glenn Hansen <travisghansen@yahoo.com>
2022-04-20 14:08:51 -06:00 · 2022-04-20 14:08:51 -06:00 · 9026d5e0d6
parent c55f3957ac
commit 9026d5e0d6
7 changed files with 291 additions and 103 deletions
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -35,14 +35,14 @@ jobs:
          path: node_modules.tar.gz
          retention-days: 7

-  csi-sanity-synology:
+  csi-sanity-synology-dsm6:
    needs:
      - build-npm
    strategy:
      fail-fast: false
      matrix:
        config:
-          - synlogy/iscsi.yaml
+          - synlogy/dsm6/iscsi.yaml
    runs-on:
      - self-hosted
      - csi-sanity-synology
@ -57,12 +57,41 @@ jobs:
          ci/bin/run.sh
        env:
          TEMPLATE_CONFIG_FILE: "./ci/configs/${{ matrix.config }}"
-          SYNOLOGY_HOST: ${{ secrets.SANITY_SYNOLOGY_HOST }}
-          SYNOLOGY_PORT: ${{ secrets.SANITY_SYNOLOGY_PORT }}
+          SYNOLOGY_HOST: ${{ secrets.SANITY_SYNOLOGY_DSM6_HOST }}
+          SYNOLOGY_PORT: ${{ secrets.SANITY_SYNOLOGY_DSM6_PORT }}
          SYNOLOGY_USERNAME: ${{ secrets.SANITY_SYNOLOGY_USERNAME }}
          SYNOLOGY_PASSWORD: ${{ secrets.SANITY_SYNOLOGY_PASSWORD }}
          SYNOLOGY_VOLUME: ${{ secrets.SANITY_SYNOLOGY_VOLUME }}

+  csi-sanity-synology-dsm7:
+    needs:
+      - build-npm
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - synlogy/dsm7/iscsi.yaml
+    runs-on:
+      - self-hosted
+      - csi-sanity-synology
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+        with:
+          name: node-modules
+      - name: csi-sanity
+        run: |
+          # run tests
+          ci/bin/run.sh
+        env:
+          TEMPLATE_CONFIG_FILE: "./ci/configs/${{ matrix.config }}"
+          SYNOLOGY_HOST: ${{ secrets.SANITY_SYNOLOGY_DSM7_HOST }}
+          SYNOLOGY_PORT: ${{ secrets.SANITY_SYNOLOGY_DSM7_PORT }}
+          SYNOLOGY_USERNAME: ${{ secrets.SANITY_SYNOLOGY_USERNAME }}
+          SYNOLOGY_PASSWORD: ${{ secrets.SANITY_SYNOLOGY_PASSWORD }}
+          SYNOLOGY_VOLUME: ${{ secrets.SANITY_SYNOLOGY_VOLUME }}
+
+
  # api-based drivers
  csi-sanity-truenas-scale-22_02:
    needs:
@ -237,7 +266,8 @@ jobs:

  build-docker:
    needs:
-      - csi-sanity-synology
+      - csi-sanity-synology-dsm6
+      - csi-sanity-synology-dsm7
      - csi-sanity-truenas-scale-22_02
      - csi-sanity-truenas-core-12_0
      - csi-sanity-truenas-core-13_0
--- a/ci/configs/synlogy/dsm6/iscsi.yaml
+++ b/ci/configs/synlogy/dsm6/iscsi.yaml
--- a/ci/configs/synlogy/dsm7/iscsi.yaml
+++ b/ci/configs/synlogy/dsm7/iscsi.yaml
@ -0,0 +1,77 @@
+driver: synology-iscsi
+httpConnection:
+  protocol: http
+  host: ${SYNOLOGY_HOST}
+  port: ${SYNOLOGY_PORT}
+  username: ${SYNOLOGY_USERNAME}
+  password: ${SYNOLOGY_PASSWORD}
+  allowInsecure: true
+  session: "democratic-csi-${CI_BUILD_KEY}"
+  serialize: true
+
+synology:
+  volume: ${SYNOLOGY_VOLUME}
+
+iscsi:
+  targetPortal: ${SYNOLOGY_HOST}
+  targetPortals: []
+  baseiqn: "iqn.2000-01.com.synology:XpenoDsm62x."
+  namePrefix: "csi-${CI_BUILD_KEY}-"
+  nameSuffix: "-ci"
+
+  lunTemplate:
+    # btrfs thin provisioning
+    type: "BLUN"
+    # tpws = Hardware-assisted zeroing
+    # caw = Hardware-assisted locking
+    # 3pc = Hardware-assisted data transfer
+    # tpu = Space reclamation
+    # can_snapshot = Snapshot
+    #dev_attribs:
+    #- dev_attrib: emulate_tpws
+    #  enable: 1
+    #- dev_attrib: emulate_caw
+    #  enable: 1
+    #- dev_attrib: emulate_3pc
+    #  enable: 1
+    #- dev_attrib: emulate_tpu
+    #  enable: 0
+    #- dev_attrib: can_snapshot
+    #  enable: 1
+
+    # btfs thick provisioning
+    # only zeroing and locking supported
+    #type: "BLUN_THICK"
+    # tpws = Hardware-assisted zeroing
+    # caw = Hardware-assisted locking
+    #dev_attribs:
+    #- dev_attrib: emulate_tpws
+    #  enable: 1
+    #- dev_attrib: emulate_caw
+    #  enable: 1
+
+    # ext4 thinn provisioning UI sends everything with enabled=0
+    #type: "THIN"
+
+    # ext4 thin with advanced legacy features set
+    # can only alter tpu (all others are set as enabled=1)
+    #type: "ADV"
+    #dev_attribs:
+    #- dev_attrib: emulate_tpu
+    #  enable: 1
+
+    # ext4 thick
+    # can only alter caw
+    #type: "FILE"
+    #dev_attribs:
+    #- dev_attrib: emulate_caw
+    #  enable: 1
+
+  lunSnapshotTemplate:
+    is_locked: true
+    # https://kb.synology.com/en-me/DSM/tutorial/What_is_file_system_consistent_snapshot
+    is_app_consistent: true
+
+  targetTemplate:
+    auth_type: 0
+    max_sessions: 0
--- a/examples/synology-iscsi.yaml
+++ b/examples/synology-iscsi.yaml
@ -34,6 +34,9 @@ iscsi:
  # These options can also be configured per storage-class:
  # See https://github.com/democratic-csi/democratic-csi/blob/master/docs/storage-class-parameters.md
  lunTemplate:
+    # can be static value or handlebars template
+    #description: "{{ parameters.[csi.storage.k8s.io/pvc/namespace] }}-{{ parameters.[csi.storage.k8s.io/pvc/name] }}"
+    
    # btrfs thin provisioning
    type: "BLUN"
    # tpws = Hardware-assisted zeroing
--- a/src/driver/controller-synology/http/index.js
+++ b/src/driver/controller-synology/http/index.js
@ -10,17 +10,46 @@ const USER_AGENT = "democratic-csi";
 const __REGISTRY_NS__ = "SynologyHttpClient";

 SYNO_ERRORS = {
-  18990002: { status: grpc.status.RESOURCE_EXHAUSTED, message: "The synology volume is out of disk space." },
-  18990318: { status: grpc.status.INVALID_ARGUMENT, message: "The requested lun type is incompatible with the Synology filesystem." },
-  18990538: { status: grpc.status.ALREADY_EXISTS, message: "A LUN with this name already exists." },
-  18990541: { status: grpc.status.RESOURCE_EXHAUSTED, message: "The maximum number of LUNS has been reached." },
-  18990542: { status: grpc.status.RESOURCE_EXHAUSTED, message: "The maximum number if iSCSI target has been reached." },
-  18990744: { status: grpc.status.ALREADY_EXISTS, message: "An iSCSI target with this name already exists." },
+  400: {
+    status: grpc.status.UNAUTHENTICATED,
+    message: "Failed to authenticate to the Synology DSM",
+  },
+  18990002: {
+    status: grpc.status.RESOURCE_EXHAUSTED,
+    message: "The synology volume is out of disk space.",
+  },
+  18990318: {
+    status: grpc.status.INVALID_ARGUMENT,
+    message:
+      "The requested lun type is incompatible with the Synology filesystem.",
+  },
+  18990538: {
+    status: grpc.status.ALREADY_EXISTS,
+    message: "A LUN with this name already exists.",
+  },
+  18990541: {
+    status: grpc.status.RESOURCE_EXHAUSTED,
+    message: "The maximum number of LUNS has been reached.",
+  },
+  18990542: {
+    status: grpc.status.RESOURCE_EXHAUSTED,
+    message: "The maximum number if iSCSI target has been reached.",
+  },
+  18990744: {
+    status: grpc.status.ALREADY_EXISTS,
+    message: "An iSCSI target with this name already exists.",
+  },
  18990532: { status: grpc.status.NOT_FOUND, message: "No such snapshot." },
  18990500: { status: grpc.status.INVALID_ARGUMENT, message: "Bad LUN type" },
-  18990543: { status: grpc.status.RESOURCE_EXHAUSTED, message: "Maximum number of snapshots reached." },
-  18990635: { status: grpc.status.INVALID_ARGUMENT, message: "Invalid ioPolicy." }
-}
+  18990543: {
+    status: grpc.status.RESOURCE_EXHAUSTED,
+    message: "Maximum number of snapshots reached.",
+  },
+  18990635: {
+    status: grpc.status.INVALID_ARGUMENT,
+    message: "Invalid ioPolicy.",
+  },
+};

 class SynologyError extends GrpcError {
  constructor(code, httpCode = undefined) {
@ -28,9 +57,11 @@ class SynologyError extends GrpcError {
    this.synoCode = code;
    this.httpCode = httpCode;
    if (code > 0) {
-      const error = SYNO_ERRORS[code]
+      const error = SYNO_ERRORS[code];
      this.code = error?.status ?? grpc.status.UNKNOWN;
-      this.message = error?.message ?? `An unknown error occurred when executing a synology command (code = ${code}).`;
+      this.message =
+        error?.message ??
+        `An unknown error occurred when executing a synology command (code = ${code}).`;
    } else {
      this.code = grpc.status.UNKNOWN;
      this.message = `The synology webserver returned a status code ${httpCode}`;
@ -95,6 +126,15 @@ class SynologyHttpClient {
      _.set(options, prop, "redacted");
    }

+    prop = "params._sid";
+    val = _.get(options, prop, false);
+    if (val) {
+      _.set(options, prop, "redacted");
+    }
+
+    delete options.httpAgent;
+    delete options.httpsAgent;
+
    this.logger.debug("SYNOLOGY HTTP REQUEST: " + stringify(options));
    this.logger.debug("SYNOLOGY HTTP ERROR: " + error);
    this.logger.debug("SYNOLOGY HTTP STATUS: " + response.statusCode);
@ -179,7 +219,7 @@ class SynologyHttpClient {
          }

          if (response.statusCode > 299 || response.statusCode < 200) {
-            reject(new SynologyError(null, response.statusCode))
+            reject(new SynologyError(null, response.statusCode));
          }

          if (response.body.success === false) {
@ -187,7 +227,9 @@ class SynologyHttpClient {
            if (response.body.error.code == 119 && sid == client.sid) {
              client.sid = null;
            }
-            reject(new SynologyError(response.body.error.code, response.statusCode));
+            reject(
+              new SynologyError(response.body.error.code, response.statusCode)
+            );
          }

          resolve(response);
@ -602,7 +644,12 @@ class SynologyHttpClient {
    );
  }

-  async CreateClonedVolume(src_lun_uuid, dst_lun_name, dst_location, description) {
+  async CreateClonedVolume(
+    src_lun_uuid,
+    dst_lun_name,
+    dst_location,
+    description
+  ) {
    const create_cloned_volume = {
      api: "SYNO.Core.ISCSI.LUN",
      version: 1,
@ -619,7 +666,12 @@ class SynologyHttpClient {
    return await this.do_request("GET", "entry.cgi", create_cloned_volume);
  }

-  async CreateVolumeFromSnapshot(src_lun_uuid, snapshot_uuid, cloned_lun_name, description) {
+  async CreateVolumeFromSnapshot(
+    src_lun_uuid,
+    snapshot_uuid,
+    cloned_lun_name,
+    description
+  ) {
    const create_volume_from_snapshot = {
      api: "SYNO.Core.ISCSI.LUN",
      version: 1,
--- a/src/driver/controller-synology/index.js
+++ b/src/driver/controller-synology/index.js
@ -1,11 +1,12 @@
+const _ = require("lodash");
 const { CsiBaseDriver } = require("../index");
+const GeneralUtils = require("../../utils/general");
 const { GrpcError, grpc } = require("../../utils/grpc");
+const Handlebars = require("handlebars");
 const registry = require("../../utils/registry");
 const SynologyHttpClient = require("./http").SynologyHttpClient;
 const semver = require("semver");
-const sleep = require("../../utils/general").sleep;
 const yaml = require("js-yaml");
-const GeneralUtils = require("../../utils/general");

 const __REGISTRY_NS__ = "ControllerSynologyDriver";

@ -146,19 +147,33 @@ class ControllerSynologyDriver extends CsiBaseDriver {

  getObjectFromDevAttribs(list = []) {
    if (!list) {
-      return {}
+      return {};
    }
    return list.reduce(
-      (obj, item) => Object.assign(obj, {[item.dev_attrib]: item.enable}), {}
-    )
+      (obj, item) => Object.assign(obj, { [item.dev_attrib]: item.enable }),
+      {}
+    );
  }

  getDevAttribsFromObject(obj, keepNull = false) {
-    return Object.entries(obj).filter(
-      e => keepNull || (e[1] != null)
-    ).map(
-      e => ({dev_attrib: e[0], enable: e[1]})
-    );
+    return Object.entries(obj)
+      .filter((e) => keepNull || e[1] != null)
+      .map((e) => ({ dev_attrib: e[0], enable: e[1] }));
+  }
+
+  parseParameterYamlData(data, fieldHint = "") {
+    try {
+      return yaml.load(data);
+    } catch {
+      if (err instanceof yaml.YAMLException) {
+        throw new GrpcError(
+          grpc.status.INVALID_ARGUMENT,
+          `${fieldHint} not a valid YAML document.`.trim()
+        );
+      } else {
+        throw err;
+      }
+    }
  }

  buildIscsiName(name) {
@ -183,14 +198,14 @@ class ControllerSynologyDriver extends CsiBaseDriver {
   * @returns {String} The location of the volume.
   */
  getLocation() {
-    let location = this.options?.synology?.volume;
-    if (location === undefined) {
-      location = "volume1"
+    let location = _.get(this.options, "synology.volume");
+    if (!location) {
+      location = "volume1";
    }
-    if (!location.startsWith('/')) {
-      location = "/" + location
+    if (!location.startsWith("/")) {
+      location = "/" + location;
    }
-    return location
+    return location;
  }

  assertCapabilities(capabilities) {
@ -350,7 +365,9 @@ class ControllerSynologyDriver extends CsiBaseDriver {
    }

    let volume_context = {};
-    const normalizedParameters = driver.getNormalizedParameters(call.request.parameters);
+    const normalizedParameters = driver.getNormalizedParameters(
+      call.request.parameters
+    );
    switch (driver.getDriverShareType()) {
      case "nfs":
        // TODO: create volume here
@ -368,13 +385,53 @@ class ControllerSynologyDriver extends CsiBaseDriver {
        break;
      case "iscsi":
        let iscsiName = driver.buildIscsiName(name);
-        let storageClassTemplate;
+        let lunTemplate;
+        let targetTemplate;
        let data;
        let target;
        let lun_mapping;
        let lun_uuid;
        let existingLun;

+        lunTemplate = Object.assign(
+          {},
+          _.get(driver.options, "iscsi.lunTemplate", {}),
+          driver.parseParameterYamlData(
+            _.get(normalizedParameters, "lunTemplate", "{}"),
+            "parameters.lunTemplate"
+          ),
+          driver.parseParameterYamlData(
+            _.get(call.request, "secrets.lunTemplate", "{}"),
+            "secrets.lunTemplate"
+          )
+        );
+        targetTemplate = Object.assign(
+          {},
+          _.get(driver.options, "iscsi.targetTemplate", {}),
+          driver.parseParameterYamlData(
+            _.get(normalizedParameters, "targetTemplate", "{}"),
+            "parameters.targetTemplate"
+          ),
+          driver.parseParameterYamlData(
+            _.get(call.request, "secrets.targetTemplate", "{}"),
+            "secrets.targetTemplate"
+          )
+        );
+
+        // render the template for description
+        if (lunTemplate.description) {
+          lunTemplate.description = Handlebars.compile(lunTemplate.description)(
+            {
+              name: call.request.name,
+              parameters: call.request.parameters,
+              csi: {
+                name: this.ctx.args.csiName,
+                version: this.ctx.args.csiVersion,
+              },
+            }
+          );
+        }
+
        // ensure volumes with the same name being requested a 2nd time but with a different size fails
        try {
          let lun = await httpClient.GetLunByName(iscsiName);
@ -429,10 +486,11 @@ class ControllerSynologyDriver extends CsiBaseDriver {
                src_lun_uuid = await httpClient.GetLunByID(src_lun_uuid).uuid;
              }

-              let snapshot = await httpClient.GetSnapshotByLunUUIDAndSnapshotUUID(
-                src_lun_uuid,
-                snapshot_uuid
-              );
+              let snapshot =
+                await httpClient.GetSnapshotByLunUUIDAndSnapshotUUID(
+                  src_lun_uuid,
+                  snapshot_uuid
+                );
              if (!snapshot) {
                throw new GrpcError(
                  grpc.status.NOT_FOUND,
@ -446,7 +504,7 @@ class ControllerSynologyDriver extends CsiBaseDriver {
                  src_lun_uuid,
                  snapshot_uuid,
                  iscsiName,
-                  normalizedParameters.description
+                  lunTemplate.description
                );
              }
              break;
@ -474,7 +532,7 @@ class ControllerSynologyDriver extends CsiBaseDriver {
                  src_lun_uuid,
                  iscsiName,
                  driver.getLocation(),
-                  normalizedParameters.description
+                  lunTemplate.description
                );
              }
              break;
@ -494,62 +552,22 @@ class ControllerSynologyDriver extends CsiBaseDriver {
          }
        } else {
          // create lun
-          try {
-            storageClassTemplate = yaml.load(normalizedParameters.lunTemplate ?? "")
-            const devAttribs = driver.getDevAttribsFromObject(Object.assign(
-              {},
-              driver.getObjectFromDevAttribs(driver.options.iscsi.lunTemplate?.dev_attribs),
-              driver.getObjectFromDevAttribs(storageClassTemplate?.dev_attribs)
-            ))
-            data = Object.assign({}, driver.options.iscsi.lunTemplate, storageClassTemplate, {
-              name: iscsiName,
-              location: driver.getLocation(),
-              size: capacity_bytes,
-              dev_attribs: devAttribs
-            });
+          data = Object.assign({}, lunTemplate, {
+            name: iscsiName,
+            location: driver.getLocation(),
+            size: capacity_bytes,
+          });

-            lun_uuid = await httpClient.CreateLun(data);
-          } catch (err) {
-            if (err instanceof yaml.YAMLException) {
-              throw new GrpcError(
-                grpc.status.INVALID_ARGUMENT,
-                `The lunTemplate on StorageClass is not a valid YAML document.`
-              );
-            } else {
-              throw err
-            }
-          }
+          lun_uuid = await httpClient.CreateLun(data);
        }

        // create target
        let iqn = driver.options.iscsi.baseiqn + iscsiName;
-        try {
-          storageClassTemplate = yaml.load(normalizedParameters.targetTemplate ?? "")
-        } catch (err) {
-          throw new GrpcError(
-            grpc.status.INVALID_ARGUMENT,
-            `The targetTemplate on StorageClass is not a valid YAML document.`
-          );
-        }
-        data = Object.assign({}, driver.options.iscsi.targetTemplate, storageClassTemplate, {
+        data = Object.assign({}, targetTemplate, {
          name: iscsiName,
          iqn,
        });

-        if ('user' in call.request.secrets && 'password' in call.request.secrets) {
-          data.user = call.request.secrets.user;
-          data.password = call.request.secrets.password;
-          data.chap = true;
-          if ('mutualUser' in call.request.secrets && 'mutualPassword' in call.request.secrets) {
-            data.mutual_user = call.request.secrets.mutualUser;
-            data.mutual_password = call.request.secrets.mutualPassword;
-            data.auth_type = 2;
-            data.mutual_chap = true;
-          } else {
-            data.auth_type = 1;
-            data.mutual_chap = false;
-          }
-        }
        let target_id = await httpClient.CreateTarget(data);
        //target = await httpClient.GetTargetByTargetID(target_id);
        target = await httpClient.GetTargetByIQN(iqn);
@ -924,6 +942,24 @@ class ControllerSynologyDriver extends CsiBaseDriver {
      );
    }

+    const normalizedParameters = driver.getNormalizedParameters(
+      call.request.parameters
+    );
+    let lunSnapshotTemplate;
+
+    lunSnapshotTemplate = Object.assign(
+      {},
+      _.get(driver.options, "iscsi.lunSnapshotTemplate", {}),
+      driver.parseParameterYamlData(
+        _.get(normalizedParameters, "lunSnapshotTemplate", "{}"),
+        "parameters.lunSnapshotTemplate"
+      ),
+      driver.parseParameterYamlData(
+        _.get(call.request, "secrets.lunSnapshotTemplate", "{}"),
+        "secrets.lunSnapshotTemplate"
+      )
+    );
+
    // check for other snapshopts with the same name on other volumes and fail as appropriate
    // TODO: technically this should only be checking lun/snapshots relevant to this specific install of the driver
    // but alas an isolation/namespacing mechanism does not exist in synology
@ -939,19 +975,9 @@ class ControllerSynologyDriver extends CsiBaseDriver {

    // check for already exists
    let snapshot;
-    let snapshotClassTemplate;
    snapshot = await httpClient.GetSnapshotByLunUUIDAndName(lun.uuid, name);
    if (!snapshot) {
-      const normalizedParameters = driver.getNormalizedParameters(call.request.parameters);
-      try {
-        snapshotClassTemplate = yaml.load(normalizedParameters.lunSnapshotTemplate ?? "");
-      } catch (err) {
-        throw new GrpcError(
-          grpc.status.INVALID_ARGUMENT,
-          `The snapshotTemplate on VolumeSnapshotClass is not a valid YAML document.`
-        );
-      }
-      let data = Object.assign({}, driver.options.iscsi.lunSnapshotTemplate, snapshotClassTemplate, {
+      let data = Object.assign({}, lunSnapshotTemplate, {
        src_lun_uuid: lun.uuid,
        taken_by: "democratic-csi",
        description: name, //check
--- a/src/driver/controller-zfs-generic/index.js
+++ b/src/driver/controller-zfs-generic/index.js
@ -65,7 +65,7 @@ class ControllerZfsGenericDriver extends ControllerZfsBaseDriver {
    const driver = this;

    driver.ctx.logger.verbose(
-      `generating smb share name for dataset: ${typeof datasetName} ${datasetName}`
+      `generating smb share name for dataset: ${datasetName}`
    );

    let name = datasetName || "";