From 279c241fd14825d7ffbb354e533231ee47012fd1 Mon Sep 17 00:00:00 2001 From: Danil Uzlov <36223296+d-uzlov@users.noreply.github.com> Date: Wed, 26 Mar 2025 10:42:22 +0000 Subject: [PATCH] proxy: add NodeGetInfo support --- src/driver/controller-proxy/index.js | 132 ++++++++++++++++++++++-- src/driver/controller-proxy/nodeInfo.md | 40 +++++-- 2 files changed, 158 insertions(+), 14 deletions(-) diff --git a/src/driver/controller-proxy/index.js b/src/driver/controller-proxy/index.js index 01988d7..805212c 100644 --- a/src/driver/controller-proxy/index.js +++ b/src/driver/controller-proxy/index.js @@ -6,10 +6,11 @@ const fs = require('fs'); const { Registry } = require("../../utils/registry"); const { GrpcError, grpc } = require("../../utils/grpc"); const path = require('path'); +const os = require("os"); const volumeIdPrefix = 'v:'; const snapshotIdPrefix = 's:'; -const NODE_TOPOLOGY_KEY_NAME = "org.democratic-csi.topology/node"; +const TOPOLOGY_DEFAULT_PREFIX = 'org.democratic-csi.topology'; class CsiProxyDriver extends CsiBaseDriver { constructor(ctx, options) { @@ -21,6 +22,7 @@ class CsiProxyDriver extends CsiBaseDriver { if (configFolder.slice(-1) == '/') { configFolder = configFolder.slice(0, -1); } + this.nodeIdSerializer = new NodeIdSerializer(ctx, options.proxy.nodeId || {}); const timeoutMinutes = this.options.proxy.cacheTimeoutMinutes ?? 60; const defaultOptions = this.options; @@ -142,7 +144,7 @@ class CsiProxyDriver extends CsiBaseDriver { } async checkAndRun(driver, methodName, call, defaultValue) { - if(typeof driver[methodName] !== 'function') { + if (typeof driver[methodName] !== 'function') { if (defaultValue) return defaultValue; // UNIMPLEMENTED could possibly confuse CSI CO into thinking // that driver does not support methodName at all. @@ -225,7 +227,6 @@ class CsiProxyDriver extends CsiBaseDriver { ); } const result = await this.checkAndRun(driver, 'CreateVolume', call); - this.ctx.logger.debug("CreateVolume result " + result); result.volume.volume_id = this.decorateVolumeHandle(connectionName, result.volume.volume_id); return result; } @@ -292,9 +293,8 @@ class CsiProxyDriver extends CsiBaseDriver { } async NodeGetInfo(call) { - const nodeName = process.env.CSI_NODE_ID || os.hostname(); const result = { - node_id: nodeName, + node_id: this.nodeIdSerializer.serialize(), max_volumes_per_node: 0, }; const topologyType = this.options.proxy.nodeTopology?.type ?? 'cluster'; @@ -310,7 +310,7 @@ class CsiProxyDriver extends CsiBaseDriver { case 'node': result.accessible_topology = { segments: { - [prefix + '/node']: nodeName, + [prefix + '/node']: NodeIdSerializer.getLocalNodeName(), }, }; break @@ -507,4 +507,124 @@ class DriverCache { } } +const nodeIdCode_NodeName = 'n'; +const nodeIdCode_Hostname = 'h'; +const nodeIdCode_ISCSI = 'i'; +const nodeIdCode_NVMEOF = 'v'; +class NodeIdSerializer { + constructor(ctx, nodeIdConfig) { + this.ctx = ctx; + this.config = nodeIdConfig; + } + + static getLocalNodeName() { + if (!process.env.CSI_NODE_ID) { + throw 'CSI_NODE_ID is required for proxy driver'; + } + return process.env.CSI_NODE_ID; + } + static getLocalIqn() { + const iqnPath = '/etc/iscsi/initiatorname.iscsi'; + const lines = fs.readFileSync(iqnPath, "utf8").split('\n'); + for (let line of lines) { + line = line.replace(/#.*/, '').replace(/\s+$/, ''); + if (line == '') { + continue; + } + const linePrefix = 'InitiatorName='; + if (line.startsWith(linePrefix)) { + const iqn = line.slice(linePrefix.length); + return iqn; + } + } + throw 'iqn is not found'; + } + static getLocalNqn() { + const nqnPath = '/etc/nvme/hostnqn'; + return fs.readFileSync(nqnPath, "utf8").replace(/\s+$/, ''); + } + + // returns { prefixName, suffix } + findPrefix(value, prefixMap) { + for (const prefixInfo of prefixMap) { + if (value.startsWith(prefixInfo.prefix)) { + return { + prefixName: prefixInfo.shortName, + suffix: value.split(prefixInfo.prefix.length), + }; + } + } + } + + serializeByPrefix(code, value, prefixMap) { + const prefixInfo = prefixMap.find(prefixInfo => value.startsWith(prefixInfo.prefix)); + if (!prefixInfo) { + throw `node id: prefix is not found for value: ${value}`; + } + if (!prefixInfo.shortName.match(/^[0-9a-z]+$/)) { + throw `prefix short name must be alphanumeric, invalid name: '${prefixInfo.shortName}'`; + } + const suffix = value.substring(prefixInfo.prefix.length); + return code + prefixInfo.shortName + '=' + suffix; + } + + deserializeFromPrefix(value, prefixMap, humanName) { + const prefixName = value.substring(0, value.indexOf('=')); + const suffix = value.substring(value.indexOf('=') + 1); + const prefixInfo = prefixMap.find(prefixInfo => prefixInfo.shortName === prefixName); + if (!prefixInfo) { + throw new GrpcError( + grpc.status.INVALID_ARGUMENT, + `unknown node prefix short name for ${humanName}: ${value}` + ); + } + return prefixInfo.prefix + suffix; + } + + // returns a single string that incorporates node id components specified in config.parts + serialize() { + let result = ''; + if (this.config.parts?.nodeName ?? true) { + result += '/' + nodeIdCode_NodeName + '=' + NodeIdSerializer.getLocalNodeName(); + } + if (this.config.parts?.hostname ?? false) { + result += '/' + nodeIdCode_Hostname + '=' + os.hostname(); + } + if (this.config.parts?.iqn ?? false) { + result += '/' + this.serializeByPrefix(nodeIdCode_ISCSI, NodeIdSerializer.getLocalIqn(), this.config.iqnPrefix); + } + if (this.config.parts?.nqn ?? false) { + result += '/' + this.serializeByPrefix(nodeIdCode_NVMEOF, NodeIdSerializer.getLocalNqn(), this.config.nqnPrefix); + } + if (result === '') { + throw 'node id can not be empty'; + } + // remove starting / + return result.slice(1); + } + + // takes a string generated by NodeIdSerializer.serialize + // returns an { nodeName, iqn, nqn } if they exist in nodeId + deserialize(nodeId) { + const result = {}; + for (const v in nodeId.split("/")) { + switch (v[0]) { + case nodeIdCode_NodeName: + result.nodeName = v.substring(v.indexOf('=') + 1); + continue; + case nodeIdCode_Hostname: + result.hostname = v.substring(v.indexOf('=') + 1); + continue; + case nodeIdCode_ISCSI: + result.iqn = this.deserializeFromPrefix(v, this.config.iqnPrefix, 'iSCSI'); + continue; + case nodeIdCode_NVMEOF: + result.nqn = this.deserializeFromPrefix(v, this.config.nqnPrefix, 'NVMEoF'); + continue; + } + } + return result; + } +} + module.exports.CsiProxyDriver = CsiProxyDriver; diff --git a/src/driver/controller-proxy/nodeInfo.md b/src/driver/controller-proxy/nodeInfo.md index 5fb5ef8..ab65d1b 100644 --- a/src/driver/controller-proxy/nodeInfo.md +++ b/src/driver/controller-proxy/nodeInfo.md @@ -8,12 +8,11 @@ There are 2 important values: - topology - node ID -There are only 2 types of topology in democratic-csi: -topology without constraints and node-local volumes. -It's easy to account for with proxy settings. +# Node ID -Node ID is a bit harder to solve, but this page suggests a solution. -Also, currently no real driver actually needs `node_id` to work, +Node ID is a bit tricky to solve, because of limited field length. + +Currently no real driver actually needs `node_id` to work, so all of this is mostly a proof-of-concept. A proof that you can create a functional proxy driver even with current CSI spec. @@ -23,7 +22,7 @@ before calling the actual real driver method. Node ID docs are not a part of user documentation because currently this is very theoretical. Current implementation works fine but doesn't do anything useful for users. -# Node info: config example +## Node ID: config example ```yaml # configured in root proxy config @@ -62,7 +61,7 @@ proxy: nodeIdType: nodeName ``` -# Reasoning why such complex node_id is required +## Node ID: Reasoning why such complex node_id is required `node_name + iqn + nqn` can be very long. @@ -89,7 +88,7 @@ For example, if driver needs iqn, proxy will find field in node id starting with search `proxy.nodeId.iqnPrefix` for entry with `shortName = 1`, and then set `node_id` to `proxy.nodeId.iqnPrefix[name=1].prefix` + `qwerty` -## Alternatives to prefixes +## Node ID: Alternatives to prefixes Each driver can override `node_id` based on node name. @@ -118,3 +117,28 @@ Still, if this were to be useful for some reason, this is fully compatible with Theoretically, more info can be extracted from node to be used in `nodeIdTemplate`, provided the info is short enough to fit into `node_id` length limit. + +# Topology + +There are 3 cases of cluster topology: + +- Each node has unique topology domain (`local` drivers) +- All nodes are the same (usually the case for non-local drivers) +- Several availability zones that can contain several nodes + +Simple cases are currently supported by the proxy. +Custom availability zones are TBD. + +Example configuration: + +```yaml +proxy: + nodeTopology: + # allowed values: + # node - each node has its own storage + # cluster - the whole cluster has unified storage + type: node + # topology reported by CSI driver is reflected in k8s as node labels. + # you may want to set unique prefixes on different drivers to avoid collisions + prefix: org.democratic-csi.topology +```