From d2d20c339ae2315f839e0bb3ef8f24699127e8d7 Mon Sep 17 00:00:00 2001 From: Kevin Adams Date: Tue, 1 Sep 2026 21:58:05 -0400 Subject: [PATCH] fix: replace AnyEvent WebSocket transport with Protocol::WebSocket (#290) AnyEvent::WebSocket::Client's blocking ->recv() cannot run inside pveproxy/pvedaemon (themselves built on AnyEvent as their core reactor) without nesting event loops, which AnyEvent refuses by design -- confirmed by a real user on the first real WebUI use of v4.0.0, one day after release. Not a race condition: this fails on every real "Add Storage" attempt against a WebSocket-transport host, unconditionally. v4.0.0's testing never caught it because every verification script ran standalone, never inside the actual daemon process -- pvesm (a fresh CLI process) doesn't trigger it either, only the real REST API path through pveproxy does. Replaced _ws_connect/_ws_call's internals with Protocol::WebSocket::Client + IO::Socket::SSL -- a plain synchronous socket client with zero event-loop dependency, so there's no shared reactor state to conflict with pveproxy's. _api_ws's method-mapping logic is completely unchanged; only the connection/call internals were rewritten. Reproduced the exact bug via a real POST to /api2/json/storage through pveproxy on a lab node, then confirmed the fix resolves it the same way. Re-ran every check from v4.0.0's release (read path, write path, snapshots, multipath) against the new transport -- zero regressions. Full story: ADR-014, superseding ADR-012's library-choice section only (everything else in ADR-012 stands). $VERSION bumped to 4.0.1. Co-Authored-By: Claude Sonnet 5 --- .../adrs/ADR-012-websocket-transport-v4.md | 15 +- ...-014-websocket-transport-library-change.md | 129 +++++++++++++++ .claude/cos/adrs/README.md | 1 + .github/workflows/build.yml | 2 +- CHANGELOG.md | 36 ++++ packaging/DEBIAN/control.j2 | 2 +- perl5/PVE/Storage/Custom/TrueNAS.pm | 155 ++++++++++++------ 7 files changed, 289 insertions(+), 51 deletions(-) create mode 100644 .claude/cos/adrs/ADR-014-websocket-transport-library-change.md diff --git a/.claude/cos/adrs/ADR-012-websocket-transport-v4.md b/.claude/cos/adrs/ADR-012-websocket-transport-v4.md index 0377e03..f19eae9 100644 --- a/.claude/cos/adrs/ADR-012-websocket-transport-v4.md +++ b/.claude/cos/adrs/ADR-012-websocket-transport-v4.md @@ -1,7 +1,7 @@ # ADR-012: WebSocket JSON-RPC 2.0 Transport for v4.0.0 (Rivendell) **Date**: 2026-08-29 -**Status**: Accepted (2026-08-31) — design decided and fully live-verified: every read and write API call the WebSocket transport makes has been confirmed against real TrueNAS SCALE hosts (`.91`, `.92` across 25.10.x and 25.04.2.6) +**Status**: Accepted (2026-08-31) — design decided and fully live-verified: every read and write API call the WebSocket transport makes has been confirmed against real TrueNAS SCALE hosts (`.91`, `.92` across 25.10.x and 25.04.2.6). **Question 1 (library choice) superseded by [ADR-014](ADR-014-websocket-transport-library-change.md) 2026-09-01** — `AnyEvent::WebSocket::Client` broke in production (#290); everything else in this ADR stands. **Deciders**: Kevin Adams ## Context @@ -147,7 +147,18 @@ Question 1 below with real-world evidence rather than a cold guess. ## Decision -### Question 1: WebSocket client library — DECIDED +### Question 1: WebSocket client library — DECIDED, then SUPERSEDED + +> **Superseded 2026-09-01 — see [ADR-014](ADR-014-websocket-transport-library-change.md).** +> `AnyEvent::WebSocket::Client`'s blocking `->recv()` cannot run inside +> `pveproxy`/`pvedaemon` (themselves built on `AnyEvent`) without nesting +> event loops, which `AnyEvent` refuses by design — confirmed by a real user +> (#290) on the very first real WebUI use, one day after v4.0.0 shipped. The +> library was replaced with `Protocol::WebSocket::Client` + `IO::Socket::SSL` +> (zero event-loop dependency). Left as-written below per this project's ADR +> rule against editing an ADR's original decision text — this note is the +> pointer forward, not a correction of the historical record. Every other +> decision in this ADR (auth, method mapping, envelope) is unaffected. **`AnyEvent::WebSocket::Client`** (`libanyevent-websocket-client-perl` 0.55-1, apt-available on Debian trixie, depends on `libanyevent-perl` 7.170, also diff --git a/.claude/cos/adrs/ADR-014-websocket-transport-library-change.md b/.claude/cos/adrs/ADR-014-websocket-transport-library-change.md new file mode 100644 index 0000000..0bce7d8 --- /dev/null +++ b/.claude/cos/adrs/ADR-014-websocket-transport-library-change.md @@ -0,0 +1,129 @@ +# ADR-014: WebSocket Transport Library Change — AnyEvent → Protocol::WebSocket + +**Date**: 2026-09-01 +**Status**: Accepted +**Deciders**: Kevin Adams +**Supersedes**: ADR-012 (Question 1 / library choice section only — the rest of ADR-012's design, method mapping, and auth decisions are unaffected and still in effect) + +## Context + +Less than 24 hours after v4.0.0 shipped, the first real user (#290) hit a hard +failure on every attempt to add a TrueNAS SCALE 25.04+ storage through the +Proxmox WebUI: + +``` +create storage failed: TrueNAS WebSocket connect to 192.168.69.92 failed: +AnyEvent::CondVar: recursive blocking wait attempted at TrueNAS.pm line 308 +``` + +### Root cause (confirmed by direct reproduction, 2026-09-01) + +ADR-012's live verification — every read call, a full `alloc_image`/ +`free_image`/snapshot cycle, multipath, a REST-path regression check — was +all done by calling the plugin module's functions directly from standalone +Perl scripts. Every one of those scripts was the *only* thing in its process +using `AnyEvent`, so `_ws_connect`'s blocking `->recv()` was always the +outermost (and only) event-loop wait. That verification was real and correct +for what it tested — but it never once ran *inside the actual `pvedaemon` +process*, which is the only way a real user ever exercises this code. + +`pveproxy` and `pvedaemon` are themselves built on `AnyEvent` as their core +reactor (Proxmox's own API server stack). When the WebUI's "Add Storage" +action calls into the plugin (`activate_storage` → `_ws_connect`), it runs +*inside* that daemon's already-active event loop. The plugin's own blocking +`->recv()` is a second, nested loop-wait on top of the daemon's own — +exactly what `AnyEvent` refuses to allow, by design, because nesting +blocking waits is genuinely unsafe with several backend implementations. + +This was reproduced directly: `pvesm add` (a standalone CLI tool, its own +fresh process) did **not** trigger the bug — confirming the hypothesis. +Issuing the same request via the real REST API (`POST /api2/json/storage` +through `pveproxy`, matching exactly what the WebUI sends) reproduced the +exact reported error on the first try, on a real lab node (`pve03-hq`). + +**This is not a one-off edge case or a race condition** — it fails on every +real invocation of the WebUI/API "Add Storage" path against any +WebSocket-transport host, unconditionally. `AnyEvent::WebSocket::Client`'s +blocking style cannot work inside `pveproxy`/`pvedaemon`, ever, regardless of +how carefully it's used. + +## Decision + +Replace the WebSocket transport implementation with **`Protocol::WebSocket::Client` ++ `IO::Socket::SSL`** — a plain, synchronous socket client with **zero event-loop +dependency of any kind**. `Protocol::WebSocket::Client` is purely a protocol +state machine: it encodes/decodes handshake and frame bytes via callbacks +(`write`/`read`/`connect`/`error`) and does no I/O itself. The plugin owns a +blocking `IO::Socket::SSL` handle directly and feeds bytes in/out via a +`select()`-with-timeout read loop (`_ws_pump`). Because this never touches +`AnyEvent` (or any other reactor) at all, there is no shared event-loop state +to conflict with `pveproxy`/`pvedaemon`'s own — nesting is structurally +impossible, not just carefully avoided. + +This was ADR-012's original "Option A" alternative, set aside at the time in +favor of `AnyEvent::WebSocket::Client` specifically because the latter was +independently proven working by `boomshankerx`'s implementation. That +evidence was real, but incomplete — it (like this project's own initial +testing) never proved the blocking-`recv()` pattern safe *inside a daemon +that is itself built on the same event-loop library*. This ADR corrects that. + +### What did NOT change + +- `_api_ws`'s method-mapping logic (the REST-path-to-JSON-RPC translation + table) — completely unchanged. Only `_ws_connect`/`_ws_call`'s internals + changed; every call site above them is identical. +- The JSON-RPC envelope, auth call (`auth.login_with_api_key`), and every + method-mapping decision from ADR-012 — all still correct and unaffected. +- `packaging/DEBIAN/control.j2`'s dependency list changes (`libanyevent-perl` + / `libanyevent-websocket-client-perl` → `libprotocol-websocket-perl`; + `libio-socket-ssl-perl` added explicitly since it's now used directly, not + just transitively via `LWP::Protocol::https`), but nothing else about the + package changes. + +## Live re-verification (2026-09-01) + +Re-ran the complete verification suite from ADR-012 against the new +implementation, against the same real hosts: + +- Every read (query) call — pass +- Full `alloc_image`/`path`/`volume_size_info`/`free_image` write cycle + against `.92` — pass, zero orphans +- Full snapshot cycle (`volume_snapshot`/`_info`/`_rollback`/`_delete`) — + pass +- Multipath (`activate_volume`/`deactivate_volume`, real 2-path + `dm-multipath` device) — pass +- **The actual bug reproduction, re-run against the fix**: same + `POST /api2/json/storage` request via the real `pveproxy`/`pvedaemon` on + `pve03-hq` that previously reproduced the error — now returns HTTP 200, + storage created and confirmed `active`, plugin log shows clean transport + detection and activation with no errors. This is the verification that + actually matters for this ADR — everything else was already known-good + from ADR-012 and was re-run to confirm the transport swap didn't regress it. + +## Consequences + +- `packaging/DEBIAN/control.j2` Depends: `libanyevent-perl` and + `libanyevent-websocket-client-perl` removed; `libprotocol-websocket-perl` + and `libio-socket-ssl-perl` added. Both confirmed apt-available on Debian + trixie. CI's lint-job module install list updated to match. +- `$VERSION` bumped to `4.0.1` — this ships as a patch release fixing a + release-day P1, not a new feature. +- **Process lesson, not just a code lesson**: standalone-script verification + of plugin logic is necessary but not sufficient for a Proxmox storage + plugin — it cannot catch bugs that only manifest from the plugin running + inside `pveproxy`/`pvedaemon`'s own process. Any future transport-level or + connection-lifecycle change to this plugin should be verified via a real + `POST` to the local Proxmox REST API (or an actual WebUI action) on a lab + node, not just direct module calls, before being considered proven. + +## References + +- ADR-012 (original WebSocket transport design — superseded in the library + choice only, everything else stands) +- #290 (the bug report that triggered this ADR) +- [AnyEvent documentation on recursive condvar waits](https://metacpan.org/pod/AnyEvent) — + the `->recv` "recursive blocking wait" guard is documented AnyEvent + behavior, not a bug in AnyEvent itself +- `Protocol::WebSocket::Client` source (`/usr/share/perl5/Protocol/WebSocket/Client.pm` + on Debian trixie) — confirmed I/O-agnostic, callback-driven design during + implementation diff --git a/.claude/cos/adrs/README.md b/.claude/cos/adrs/README.md index 187e6db..ee7eb65 100644 --- a/.claude/cos/adrs/README.md +++ b/.claude/cos/adrs/README.md @@ -36,3 +36,4 @@ ADRs document significant decisions made about this project. They follow an RFC- | [ADR-011](ADR-011-apt-components-optional-features.md) | APT Components for Optional Feature Packages | Accepted | | [ADR-012](ADR-012-websocket-transport-v4.md) | WebSocket JSON-RPC 2.0 Transport for v4.0.0 (Rivendell) | Accepted | | [ADR-013](ADR-013-branching-strategy.md) | Role-Based Branching Strategy | Accepted | +| [ADR-014](ADR-014-websocket-transport-library-change.md) | WebSocket Transport Library Change — AnyEvent → Protocol::WebSocket | Accepted | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5a1645b..d55aeac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,7 +36,7 @@ jobs: sudo apt-get install -y \ shellcheck libperl-critic-perl \ libwww-perl libio-socket-ssl-perl libjson-perl \ - libanyevent-perl libanyevent-websocket-client-perl + libprotocol-websocket-perl # Stub PVE modules not available outside Proxmox hosts mkdir -p /tmp/pve-stub/PVE/Storage diff --git a/CHANGELOG.md b/CHANGELOG.md index 72bd865..0d9205c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,42 @@ See each [GitHub Release](https://github.com/TheGrandWazoo/truenas-proxmox/relea --- +## [4.0.1] — 2026-09-01 + +### Fixed +- **WebSocket transport failed on every real "Add Storage" attempt against a + TrueNAS SCALE 25.04+ host** — `create storage failed: TrueNAS WebSocket + connect to ... failed: AnyEvent::CondVar: recursive blocking wait + attempted` (#290, reported the day after v4.0.0 shipped). Root cause: + `pveproxy`/`pvedaemon` are themselves built on `AnyEvent` as their core + reactor, so the plugin's blocking `AnyEvent::WebSocket::Client` `->recv()` + nested inside the daemon's own already-running event loop, which + `AnyEvent` refuses by design. Not a race condition — this failed on every + real WebUI/API use, unconditionally; v4.0.0's testing never caught it + because every verification script ran standalone, never inside the actual + daemon process. See [ADR-014](.claude/cos/adrs/ADR-014-websocket-transport-library-change.md). +- Replaced the transport with `Protocol::WebSocket::Client` + `IO::Socket::SSL` + — a plain synchronous socket client with zero event-loop dependency of any + kind, so there's no shared reactor state to conflict with. `_api_ws`'s + method-mapping logic is completely unchanged; only the connection/call + internals (`_ws_connect`, `_ws_call`) were rewritten. +- Fixed a `perlcritic` `RequireFinalReturn` finding surfaced while rewriting + `_ws_call` (unrelated to the bug itself, caught during the same pass). + +### Changed +- `packaging/DEBIAN/control.j2` dependencies: `libanyevent-perl` and + `libanyevent-websocket-client-perl` removed; `libprotocol-websocket-perl` + and `libio-socket-ssl-perl` added (both apt-available on Debian trixie). + +### Verified +- Full re-run of every check from v4.0.0's release (read path, write path, + snapshots, multipath) against the new transport — all pass, zero + regressions. +- **The actual bug reproduction, re-run against the fix**: the exact + `POST /api2/json/storage` request that reproduced #290 via the real + `pveproxy`/`pvedaemon` on a lab node now returns HTTP 200 and a working, + `active` storage — this is the verification that matters for this release. + ## [4.0.0] — 2026-08-31 — Rivendell See [ADR-012](.claude/cos/adrs/ADR-012-websocket-transport-v4.md) (Accepted) diff --git a/packaging/DEBIAN/control.j2 b/packaging/DEBIAN/control.j2 index 34a9bb0..1d0ba37 100644 --- a/packaging/DEBIAN/control.j2 +++ b/packaging/DEBIAN/control.j2 @@ -2,7 +2,7 @@ Package: truenas-proxmox Version: ${VERSION} Architecture: all Maintainer: KSA Technologies, LLC -Depends: librest-client-perl, open-iscsi, libanyevent-perl, libanyevent-websocket-client-perl +Depends: librest-client-perl, open-iscsi, libprotocol-websocket-perl, libio-socket-ssl-perl Replaces: freenas-proxmox Breaks: freenas-proxmox (<< 3.0.0) Section: perl diff --git a/perl5/PVE/Storage/Custom/TrueNAS.pm b/perl5/PVE/Storage/Custom/TrueNAS.pm index bb8317d..526e313 100644 --- a/perl5/PVE/Storage/Custom/TrueNAS.pm +++ b/perl5/PVE/Storage/Custom/TrueNAS.pm @@ -18,14 +18,23 @@ package PVE::Storage::Custom::TrueNAS; # # NOTE: every API call this plugin makes over the WebSocket transport # (_api_ws, _ws_connect, _ws_call) has been live-verified against a real -# TrueNAS SCALE host (.92, 25.04.2.6) as of 2026-08-31 — see ADR-012 for the -# full history. Covered: every read (query) call, a complete +# TrueNAS SCALE host (.92, 25.04.2.6) — see ADR-012 for the full history. +# Covered: every read (query) call, a complete # alloc_image/path/volume_size_info/free_image cycle (target/extent/ # targetextent/dataset create+delete, zero orphans left behind), and a full -# snapshot cycle (volume_snapshot/_info/_rollback/_delete). Two real bugs -# were caught and fixed along the way (see ADR-012): missing int() on -# regex-captured ids, and iscsi.extent.delete's positional -# (id, remove, force) signature, which is NOT (id, {force=>bool}). +# snapshot cycle (volume_snapshot/_info/_rollback/_delete). Several real bugs +# were caught and fixed along the way (see ADR-012 and ADR-014): missing +# int() on regex-captured ids, iscsi.extent.delete's positional +# (id, remove, force) signature (NOT (id, {force=>bool})), and — the big +# one — the transport itself was rewritten from AnyEvent::WebSocket::Client +# to Protocol::WebSocket + IO::Socket::SSL after a real user hit +# "AnyEvent::CondVar: recursive blocking wait attempted" in production +# (#290). Root cause: pveproxy/pvedaemon are themselves built on AnyEvent as +# their core reactor, so any blocking AnyEvent ->recv() call from a plugin +# running inside that daemon nests inside its already-running loop, which +# AnyEvent refuses by design. See ADR-014 for the full story — this is why +# the transport below is a plain synchronous socket client with zero +# event-loop dependency, not a style preference. # # Per-VM target architecture: # Each VM gets its own iSCSI target (proxmox-vm-). @@ -42,14 +51,14 @@ use LWP::UserAgent (); use HTTP::Request (); use URI::Escape qw(uri_escape uri_unescape); use Sys::Syslog qw(syslog); -use AnyEvent (); -use AnyEvent::WebSocket::Client (); +use IO::Socket::SSL qw(SSL_VERIFY_NONE SSL_VERIFY_PEER); +use Protocol::WebSocket::Client (); use PVE::Storage::Plugin; use base qw(PVE::Storage::Plugin); -our $VERSION = '4.0.0'; +our $VERSION = '4.0.1'; # Per-host runtime state cache my $state = {}; @@ -279,10 +288,15 @@ sub _api_rest { # ── Private: WebSocket JSON-RPC 2.0 transport (TrueNAS SCALE 25.04+) ───────── # -# See ADR-012 for the full design and live-verification history. Summary: -# - AnyEvent::WebSocket::Client, used in blocking style (condvar ->recv) to -# match this plugin's existing per-invocation synchronous lifecycle — no -# persistent event loop needed, same as the LWP::UserAgent path above. +# See ADR-012 for the original design and ADR-014 for why the transport +# below is a plain synchronous socket client, not an AnyEvent-based one. +# Summary: +# - Protocol::WebSocket::Client is I/O-agnostic — it only encodes/decodes +# handshake and frame bytes via callbacks (write/read/connect/error); WE +# own a plain blocking IO::Socket::SSL and feed bytes in/out ourselves. +# No event loop of any kind, so no risk of nesting inside pveproxy's own +# AnyEvent reactor (#290 — the reason this isn't AnyEvent::WebSocket::Client +# anymore). # - Auth is auth.login_with_api_key (single positional [api_key] param) — # NOT auth.login_ex, which requires a username field this plugin has no # way to know for an arbitrary API key. @@ -290,6 +304,33 @@ sub _api_rest { # object, not a plain 1/0 — ref() on it is truthy. Never assume a non-hash # JSON-RPC result is a plain scalar without checking ref() eq 'HASH' first. +# Blocks until $sock is readable (or dies on timeout), then reads whatever's +# available and feeds it to the Protocol::WebSocket::Client state machine — +# which synchronously invokes whichever on(...) callback the bytes complete +# (connect, read, or error). Shared by the handshake wait and the per-call +# response wait below. +sub _ws_pump { + my ($scfg, $sock, $client, $deadline, $what) = @_; + my $host = $scfg->{truenas_host}; + + my $remaining = $deadline - time(); + die "TrueNAS WebSocket $what to $host timed out after 30s\n" if $remaining <= 0; + + my $rin = ''; + vec($rin, fileno($sock), 1) = 1; + my $nfound = select(my $rout = $rin, undef, undef, $remaining); + die "TrueNAS WebSocket $what to $host timed out after 30s\n" unless $nfound; + + my $buf; + my $n = $sock->sysread($buf, 65536); + unless (defined $n && $n > 0) { + delete $state->{$host}{ws}; + die "TrueNAS WebSocket connection to $host closed unexpectedly during $what\n"; + } + $client->read($buf); + return; +} + # Opens (and caches) an authenticated WebSocket connection for $scfg's host. sub _ws_connect { my ($scfg) = @_; @@ -299,34 +340,48 @@ sub _ws_connect { my $token = $state->{$host}{api_token} or die "TrueNAS API token not resolved for '$host'\n"; - my $url = "wss://$host/api/current"; - my $client = AnyEvent::WebSocket::Client->new( - ssl_no_verify => ($scfg->{truenas_ssl_verify} // 0) ? 0 : 1, - timeout => 30, - ); + my $sock = IO::Socket::SSL->new( + PeerHost => $host, + PeerPort => 443, + Timeout => 30, + SSL_verify_mode => ($scfg->{truenas_ssl_verify} // 0) ? SSL_VERIFY_PEER : SSL_VERIFY_NONE, + SSL_hostname => $host, + SSL_verifycn_name => $host, + ) or die "TrueNAS WebSocket TCP/TLS connect to $host failed: $IO::Socket::SSL::SSL_ERROR\n"; + $sock->autoflush(1); + $sock->blocking(1); - my $conn = eval { $client->connect($url)->recv }; - die "TrueNAS WebSocket connect to $host failed: $@" if $@; + my $handshake_done = 0; + my @inbound; + my $client = Protocol::WebSocket::Client->new(url => "wss://$host/api/current"); + $client->on(write => sub { + my (undef, $buf) = @_; + print { $sock } $buf; + }); + $client->on(connect => sub { + $handshake_done = 1; + }); + $client->on(read => sub { + my (undef, $buf) = @_; + push @inbound, $buf; + }); + $client->on(error => sub { + my (undef, $err) = @_; + die "TrueNAS WebSocket handshake to $host failed: $err\n"; + }); + + $client->connect; # synchronously fires the 'write' callback above + + my $deadline = time() + 30; + _ws_pump($scfg, $sock, $client, $deadline, 'handshake') until $handshake_done; $state->{$host}{ws} = { - conn => $conn, - pending => {}, + sock => $sock, + client => $client, + inbound => \@inbound, next_id => 1, }; - $conn->on(each_message => sub { - my (undef, $message) = @_; - my $body = eval { decode_json($message->body) }; - return unless $body; - my $id = $body->{id}; - my $pending = $state->{$host}{ws}{pending}; - return unless defined $id && $pending->{$id}; - (delete $pending->{$id})->send($body); - }); - $conn->on(finish => sub { - delete $state->{$host}{ws}; - }); - my $auth = _ws_call($scfg, 'auth.login_with_api_key', [$token]); unless (!$auth->{error} && $auth->{result}) { my $reason = $auth->{error}{message} // 'authentication rejected'; @@ -348,19 +403,25 @@ sub _ws_call { my $id = $ws->{next_id}++; my $req = { jsonrpc => '2.0', id => $id, method => $method, params => $params // [] }; + $ws->{client}->write(encode_json($req)); # frames + masks + fires 'write' - my $cv = AnyEvent->condvar; - $ws->{pending}{$id} = $cv; - $ws->{conn}->send(encode_json($req)); - - my $timeout_w = AnyEvent->timer(after => 30, cb => sub { - return unless $ws->{pending}{$id}; - delete $ws->{pending}{$id}; - $cv->send({ error => { message => "TrueNAS WebSocket call '$method' timed out after 30s" } }); - }); - my $resp = $cv->recv; - undef $timeout_w; - return $resp; + my $deadline = time() + 30; + my $result; + until (defined $result) { + while (my $msg = shift @{ $ws->{inbound} }) { + my $body = eval { decode_json($msg) }; + next unless $body && defined $body->{id}; + if ($body->{id} == $id) { + $result = $body; + last; + } + # Not our response (shouldn't normally happen in this serialized + # request/response pattern) — drop and keep waiting for ours. + } + _ws_pump($scfg, $ws->{sock}, $ws->{client}, $deadline, "call '$method'") + unless defined $result; + } + return $result; } # Translates a REST-style (method, path, data) call onto its JSON-RPC 2.0