freenas-proxmox/perl5/PVE/Storage/Custom/TrueNASMultipath.pm

package PVE::Storage::Custom::TrueNASMultipath;

# TrueNAS Multipath Storage Plugin for Proxmox VE
#
# Extends TrueNAS.pm with kernel iSCSI multipath support (iscsiadm + dm-multipath).
# Registers as PVE storage type 'truenas-multipath'.
#
# Requires: truenas-proxmox >= 3.1.0, open-iscsi, multipath-tools
# Auth:     Same Bearer token model as TrueNAS.pm
# Path:     /dev/mapper/<wwid> instead of iscsi:// — QEMU uses the block device directly

use strict;
use warnings;

use PVE::Storage::Custom::TrueNAS;
use base qw(PVE::Storage::Custom::TrueNAS);

# ── Plugin identity ───────────────────────────────────────────────────────────

sub type { return 'truenas-multipath'; }

sub plugindata {
    return {
        content => [ { images => 1 }, { images => 1 } ],
        format  => [ { raw    => 1 }, 'raw' ],
    };
}

# ── Config properties (extends parent) ───────────────────────────────────────

# Only declare NEW properties here. Inherited TrueNAS.pm properties (truenas_host,
# truenas_pool, etc.) are already registered globally by TrueNAS.pm — re-declaring
# them would cause a "duplicate property" error in PVE::SectionConfig.
sub properties {
    return {
        truenas_portals => {
            description => "Comma-separated iSCSI portal IP addresses for multipath "
                         . "(e.g. '172.31.69.91,192.168.69.91'). Each portal is logged "
                         . "into separately; dm-multipath aggregates the sessions.",
            type        => 'string',
        },
    };
}

sub options {
    my $opts = PVE::Storage::Custom::TrueNAS::options();
    $opts->{truenas_portals} = { optional => 0 };
    return $opts;
}

# ── Private helpers ───────────────────────────────────────────────────────────

# Parse truenas_portals into a list of IP strings.
sub _portals {
    my ($scfg) = @_;
    return grep { length } split /\s*,\s*/, ($scfg->{truenas_portals} // '');
}

# Convert TrueNAS NAA string (e.g. "0x6589cfc...") to the dm-multipath WWID
# used in /dev/mapper/ (e.g. "36589cfc..."). Returns undef for a missing/empty
# NAA — a bare "3" would be truthy and send callers to /dev/mapper/3.
sub _naa_to_wwid {
    my ($naa) = @_;
    return unless defined $naa && length $naa;
    $naa =~ s/^0x//i;
    return "3$naa";
}

# Run an iscsiadm command, returning (exit_code, output).
# Suppresses errors that indicate a session already exists.
sub _iscsiadm {
    my (@args) = @_;
    my $out = '';
    open(my $fh, '-|', 'iscsiadm', @args, '2>&1')
        or return (1, "failed to exec iscsiadm: $!");
    $out = do { local $/; <$fh> };
    close $fh;
    my $rc = $? >> 8;
    return ($rc, $out);
}

# Wait up to $timeout seconds for a block device path to appear.
sub _wait_for_device {
    my ($path, $timeout) = @_;
    $timeout //= 30;
    while ($timeout-- > 0) {
        return 1 if -b $path;
        sleep 1;
    }
    return 0;
}

# ── Path — block device via dm-multipath ─────────────────────────────────────

sub path {
    my ($class, $scfg, $volname, $storeid, $snapname) = @_;

    PVE::Storage::Custom::TrueNAS::_resolve_token($storeid, $scfg);

    # PVE's contract is ($path, $owner_vmid, $vtype). qm destroy skips any
    # disk whose owner != vmid, so returning the LUN id here silently
    # orphaned every multipath disk on destroy (#291).
    my ($vtype, undef, $ds_vmid) = $class->parse_volname($volname);

    # Orphaned zvol (no extent / no NAA): return a placeholder with the real
    # owner and vtype so pvesm free / GUI Remove can still delete it (#292).
    # activate_volume dies on the same condition, so it is never opened.
    my $ext  = PVE::Storage::Custom::TrueNAS::_find_extent($scfg, $volname);
    my $wwid = $ext ? _naa_to_wwid($ext->{naa}) : undef;
    unless ($wwid) {
        PVE::Storage::Custom::TrueNAS::_log('warning',
            "multipath path: $volname has no iSCSI extent/WWID — returning placeholder path");
        return ("/dev/mapper/truenas-unmapped-$volname", $ds_vmid, $vtype);
    }

    return ("/dev/mapper/$wwid", $ds_vmid, $vtype);
}

# ── volume_resize — grow zvol, then refresh kernel view of the LUN ────────────

# The base class grows the zvol. If the volume is active on this node, the
# kernel iSCSI sessions and the dm map still report the old size, and
# qemu-server's QMP block_resize on the host_device would fail. Rescan every
# session to the target, then have multipathd resize the map.
sub volume_resize {
    my ($class, $scfg, $storeid, $volname, $size, $running, $snapname) = @_;

    my $new_kb = $class->SUPER::volume_resize($scfg, $storeid, $volname, $size, $running, $snapname);

    my $ext  = PVE::Storage::Custom::TrueNAS::_find_extent($scfg, $volname);
    my $wwid = $ext ? _naa_to_wwid($ext->{naa}) : undef;
    return $new_kb unless $wwid && -b "/dev/mapper/$wwid" && defined $ext->{target_id};

    my $t   = PVE::Storage::Custom::TrueNAS::_api($scfg, 'GET',
                  "/iscsi/target/id/$ext->{target_id}") // {};
    my $iqn = PVE::Storage::Custom::TrueNAS::_basename($scfg) . ":$t->{name}";

    my ($rc, $out) = _iscsiadm('-m', 'node', '-T', $iqn, '-R');
    die "iscsiadm rescan of $iqn failed (rc=$rc): $out\n" if $rc;

    my $multipathd = -x '/usr/sbin/multipathd' ? '/usr/sbin/multipathd' : '/sbin/multipathd';
    system($multipathd, 'resize', 'map', $wwid) == 0
        or die "multipathd resize map $wwid failed (rc=" . ($? >> 8) . ")\n";

    PVE::Storage::Custom::TrueNAS::_log('info', "multipath resize: $wwid rescanned and resized");
    return $new_kb;
}

# ── qemu_blockdev_options — delegate to Plugin.pm (host_device for /dev/mapper) ─

# TrueNAS.pm overrides qemu_blockdev_options to return an iscsi:// blockdev —
# that's wrong for multipath where path() returns /dev/mapper/<wwid>.
# We skip the parent and call the grandparent (PVE::Storage::Plugin) directly.
# Plugin.pm calls path(), sees /dev/mapper/<wwid> starts with '/', stats it as a
# block device (S_ISBLK), and returns { driver => 'host_device', filename => ... }.
# QEMU then accesses the dm-multipath block device directly.
sub qemu_blockdev_options {
    my ($class, $scfg, $storeid, $volname, $machine_version, $options) = @_;
    return PVE::Storage::Plugin::qemu_blockdev_options(
        $class, $scfg, $storeid, $volname, $machine_version, $options // {});
}

# ── activate_volume — login via all portals, wait for mapper device ───────────

sub activate_volume {
    my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;

    PVE::Storage::Custom::TrueNAS::_resolve_token($storeid, $scfg);

    my $ext = PVE::Storage::Custom::TrueNAS::_find_extent($scfg, $volname);
    die "Volume '$volname' has no iSCSI extent on $scfg->{truenas_host}.\n" unless $ext;
    die "Volume '$volname' is not mapped to any iSCSI target.\n"
        unless defined $ext->{target_id};

    my $t   = PVE::Storage::Custom::TrueNAS::_api($scfg, 'GET',
                  "/iscsi/target/id/$ext->{target_id}") // {};
    my $iqn = PVE::Storage::Custom::TrueNAS::_basename($scfg) . ":$t->{name}";

    my $wwid = _naa_to_wwid($ext->{naa})
        or die "Volume '$volname' has no NAA/WWID.\n";
    my $mapper = "/dev/mapper/$wwid";

    return if -b $mapper;    # already active

    my @portals = _portals($scfg);
    die "No portals configured — set truenas_portals in storage config.\n" unless @portals;

    PVE::Storage::Custom::TrueNAS::_log('info',
        "multipath activate: $volname  iqn=$iqn  wwid=$wwid  portals=" . join(',', @portals));

    for my $ip (@portals) {
        my $portal = "$ip:3260";

        # Ensure node entry exists
        _iscsiadm('-m', 'node', '-T', $iqn, '-p', $portal, '-o', 'new');

        # Login — ISCSI_ERR_SESS_EXISTS (15) is fine
        my ($rc, $out) = _iscsiadm('-m', 'node', '-T', $iqn, '-p', $portal, '--login');
        if ($rc && $rc != 15) {
            die "iscsiadm login to $portal failed (rc=$rc): $out\n";
        }
    }

    # Signal multipathd to pick up the new sessions. Don't call multipath(8)
    # directly — that conflicts with a running multipathd daemon.
    if (-x '/usr/sbin/multipathd') {
        system('multipathd', 'reconfigure');
    } elsif (-x '/sbin/multipathd') {
        system('/sbin/multipathd', 'reconfigure');
    }

    # multipathd may take a moment to create the dm device after sessions appear
    unless (_wait_for_device($mapper, 30)) {
        # Last resort: force a manual scan
        system('multipath', '-v0') if !-b $mapper;
        unless (_wait_for_device($mapper, 10)) {
            my $found = `multipath -l 2>/dev/null` // '';
            die "Multipath device $mapper did not appear after 40s. "
              . "Check that multipathd is running and multipath.conf includes TrueNAS devices.\n"
              . "Current multipath map:\n$found\n";
        }
    }

    PVE::Storage::Custom::TrueNAS::_log('info', "multipath activate: $mapper ready");
    return;
}

# ── deactivate_volume — logout from all portals, flush mapper device ──────────

sub deactivate_volume {
    my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;

    PVE::Storage::Custom::TrueNAS::_resolve_token($storeid, $scfg);

    my $ext = PVE::Storage::Custom::TrueNAS::_find_extent($scfg, $volname);
    return unless $ext && defined $ext->{target_id};

    my $t   = PVE::Storage::Custom::TrueNAS::_api($scfg, 'GET',
                  "/iscsi/target/id/$ext->{target_id}") // {};
    my $iqn = PVE::Storage::Custom::TrueNAS::_basename($scfg) . ":$t->{name}";

    my $wwid   = _naa_to_wwid($ext->{naa} // '');
    my $mapper = $wwid ? "/dev/mapper/$wwid" : undef;

    # Flush multipath device first
    if ($mapper && -b $mapper) {
        system('multipath', '-f', $wwid);
    }

    for my $ip (_portals($scfg)) {
        my $portal = "$ip:3260";
        _iscsiadm('-m', 'node', '-T', $iqn, '-p', $portal, '--logout');
        _iscsiadm('-m', 'node', '-T', $iqn, '-p', $portal, '-o', 'delete');
    }

    PVE::Storage::Custom::TrueNAS::_log('info', "multipath deactivate: $volname done");
    return;
}

1;