API endpoint and associated RPC changes to resolve VMs IP's (#188)

* API endpoint and associated RPC changes to resolve VMs IP's

* Fix "Missing expected argument '<name>'" error when doing "tart set"

* Implement TestIPEndpoint() and IP() method in controller HTTP client

internal/controller/api.go

@@ -121,6 +121,9 @@ func (controller *Controller) initAPI() *gin.Engine {
 	v1.GET("/vms/:name/port-forward", func(c *gin.Context) {
 		controller.portForwardVM(c).Respond(c)
 	})
+	v1.GET("/vms/:name/ip", func(c *gin.Context) {
+		controller.ip(c).Respond(c)
+	})
 	v1.DELETE("/vms/:name", func(c *gin.Context) {
 		controller.deleteVM(c).Respond(c)
 	})

internal/controller/api_vms_ip.go

@@ -0,0 +1,69 @@
+package controller
+
+import (
+	"context"
+	"github.com/cirruslabs/orchard/internal/responder"
+	v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
+	"github.com/cirruslabs/orchard/rpc"
+	"github.com/gin-gonic/gin"
+	"github.com/google/uuid"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+func (controller *Controller) ip(ctx *gin.Context) responder.Responder {
+	if responder := controller.authorize(ctx, v1.ServiceAccountRoleComputeWrite); responder != nil {
+		return responder
+	}
+
+	// Retrieve and parse path and query parameters
+	name := ctx.Param("name")
+
+	waitRaw := ctx.Query("wait")
+	wait, err := strconv.ParseUint(waitRaw, 10, 16)
+	if err != nil {
+		return responder.Code(http.StatusBadRequest)
+	}
+	waitContext, waitContextCancel := context.WithTimeout(ctx, time.Duration(wait)*time.Second)
+	defer waitContextCancel()
+
+	// Look-up the VM
+	vm, responderImpl := controller.waitForVM(waitContext, name)
+	if responderImpl != nil {
+		return responderImpl
+	}
+
+	// Send an IP resolution request and wait for the result
+	session := uuid.New().String()
+	boomerangConnCh, cancel := controller.ipRendezvous.Request(ctx, session)
+	defer cancel()
+
+	err = controller.workerNotifier.Notify(ctx, vm.Worker, &rpc.WatchInstruction{
+		Action: &rpc.WatchInstruction_ResolveIpAction{
+			ResolveIpAction: &rpc.WatchInstruction_ResolveIP{
+				Session: session,
+				VmUid:   vm.UID,
+			},
+		},
+	})
+	if err != nil {
+		controller.logger.Warnf("failed to request VM's IP from the worker %s: %v",
+			vm.Worker, err)
+
+		return responder.Code(http.StatusServiceUnavailable)
+	}
+
+	select {
+	case ip := <-boomerangConnCh:
+		result := struct {
+			IP string `json:"ip"`
+		}{
+			IP: ip,
+		}
+
+		return responder.JSON(http.StatusOK, &result)
+	case <-ctx.Done():
+		return responder.Error(ctx.Err())
+	}
+}

internal/controller/api_vms_portforward.go

@@ -59,7 +59,7 @@ func (controller *Controller) portForward(
 ) responder.Responder {
 	// Request and wait for a connection with a worker
 	session := uuid.New().String()
-	boomerangConnCh, cancel := controller.proxy.Request(ctx, session)
+	boomerangConnCh, cancel := controller.connRendezvous.Request(ctx, session)
 	defer cancel()
 
 	// send request to worker to initiate port-forwarding connection back to us

internal/controller/controller.go

@@ -6,7 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/cirruslabs/orchard/internal/controller/notifier"
-	"github.com/cirruslabs/orchard/internal/controller/proxy"
+	"github.com/cirruslabs/orchard/internal/controller/rendezvous"
 	"github.com/cirruslabs/orchard/internal/controller/scheduler"
 	"github.com/cirruslabs/orchard/internal/controller/sshserver"
 	storepkg "github.com/cirruslabs/orchard/internal/controller/store"
@@ -54,7 +54,8 @@ type Controller struct {
 	logger               *zap.SugaredLogger
 	grpcServer           *grpc.Server
 	workerNotifier       *notifier.Notifier
-	proxy                *proxy.Proxy
+	connRendezvous       *rendezvous.Rendezvous[net.Conn]
+	ipRendezvous         *rendezvous.Rendezvous[string]
 	enableSwaggerDocs    bool
 	workerOfflineTimeout time.Duration
 	maxWorkersPerLicense uint
@@ -69,7 +70,8 @@ type Controller struct {
 
 func New(opts ...Option) (*Controller, error) {
 	controller := &Controller{
-		proxy:                proxy.NewProxy(),
+		connRendezvous:       rendezvous.New[net.Conn](),
+		ipRendezvous:         rendezvous.New[string](),
 		workerOfflineTimeout: 3 * time.Minute,
 		maxWorkersPerLicense: maxWorkersPerDefaultLicense,
 	}
@@ -125,7 +127,7 @@ func New(opts ...Option) (*Controller, error) {
 	// Instantiate the SSH server (if configured)
 	if controller.sshListenAddr != "" && controller.sshSigner != nil {
 		controller.sshServer, err = sshserver.NewSSHServer(controller.sshListenAddr, controller.sshSigner,
-			store, controller.proxy, controller.workerNotifier, controller.sshNoClientAuth, controller.logger)
+			store, controller.connRendezvous, controller.workerNotifier, controller.sshNoClientAuth, controller.logger)
 		if err != nil {
 			return nil, err
 		}

internal/controller/proxy/proxy.go

@@ -1,49 +0,0 @@
-package proxy
-
-import (
-	"context"
-	"errors"
-	"github.com/cirruslabs/orchard/internal/concurrentmap"
-	"net"
-)
-
-var ErrInvalidToken = errors.New("invalid proxy token")
-
-type Proxy struct {
-	sessions *concurrentmap.ConcurrentMap[*TokenSlot]
-}
-
-type TokenSlot struct {
-	ctx context.Context
-	ch  chan net.Conn
-}
-
-func NewProxy() *Proxy {
-	return &Proxy{
-		sessions: concurrentmap.NewConcurrentMap[*TokenSlot](),
-	}
-}
-
-func (proxy *Proxy) Request(ctx context.Context, session string) (chan net.Conn, func()) {
-	tokenSlot := &TokenSlot{
-		ctx: ctx,
-		ch:  make(chan net.Conn),
-	}
-
-	proxy.sessions.Store(session, tokenSlot)
-
-	return tokenSlot.ch, func() {
-		proxy.sessions.Delete(session)
-	}
-}
-
-func (proxy *Proxy) Respond(session string, conn net.Conn) (context.Context, error) {
-	tokenSlot, ok := proxy.sessions.Load(session)
-	if !ok {
-		return nil, ErrInvalidToken
-	}
-
-	tokenSlot.ch <- conn
-
-	return tokenSlot.ctx, nil
-}

internal/controller/rendezvous/rendezvous.go

@@ -0,0 +1,48 @@
+package rendezvous
+
+import (
+	"context"
+	"errors"
+	"github.com/cirruslabs/orchard/internal/concurrentmap"
+)
+
+var ErrInvalidToken = errors.New("invalid rendezvous token")
+
+type Rendezvous[T any] struct {
+	sessions *concurrentmap.ConcurrentMap[*TokenSlot[T]]
+}
+
+type TokenSlot[T any] struct {
+	ctx context.Context
+	ch  chan T
+}
+
+func New[T any]() *Rendezvous[T] {
+	return &Rendezvous[T]{
+		sessions: concurrentmap.NewConcurrentMap[*TokenSlot[T]](),
+	}
+}
+
+func (rendezvous *Rendezvous[T]) Request(ctx context.Context, session string) (chan T, func()) {
+	tokenSlot := &TokenSlot[T]{
+		ctx: ctx,
+		ch:  make(chan T),
+	}
+
+	rendezvous.sessions.Store(session, tokenSlot)
+
+	return tokenSlot.ch, func() {
+		rendezvous.sessions.Delete(session)
+	}
+}
+
+func (rendezvous *Rendezvous[T]) Respond(session string, conn T) (context.Context, error) {
+	tokenSlot, ok := rendezvous.sessions.Load(session)
+	if !ok {
+		return nil, ErrInvalidToken
+	}
+
+	tokenSlot.ch <- conn
+
+	return tokenSlot.ctx, nil
+}

internal/controller/rendezvous/rendezvous_test.go

@@ -1,8 +1,8 @@
-package proxy_test
+package rendezvous_test
 
 import (
 	"context"
-	"github.com/cirruslabs/orchard/internal/controller/proxy"
+	"github.com/cirruslabs/orchard/internal/controller/rendezvous"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 	"net"
@@ -15,7 +15,7 @@ func TestProxy(t *testing.T) {
 
 	expectedConn, _ := net.Pipe()
 
-	proxy := proxy.NewProxy()
+	proxy := rendezvous.New[net.Conn]()
 
 	token := uuid.New().String()
 

internal/controller/rpc.go

@@ -1,6 +1,7 @@
 package controller
 
 import (
+	"context"
 	v1pkg "github.com/cirruslabs/orchard/pkg/resource/v1"
 	"github.com/cirruslabs/orchard/rpc"
 	"google.golang.org/grpc/metadata"
@@ -61,8 +62,8 @@ func (controller *Controller) PortForward(stream rpc.Controller_PortForwardServe
 		}),
 	}
 
-	// make proxy aware of the connection
-	proxyCtx, err := controller.proxy.Respond(sessionMetadataValue[0], conn)
+	// make connection rendezvous aware of the connection
+	proxyCtx, err := controller.connRendezvous.Respond(sessionMetadataValue[0], conn)
 	if err != nil {
 		return err
 	}
@@ -74,3 +75,22 @@ func (controller *Controller) PortForward(stream rpc.Controller_PortForwardServe
 		return stream.Context().Err()
 	}
 }
+
+func (controller *Controller) ResolveIP(ctx context.Context, request *rpc.ResolveIPResult) (*emptypb.Empty, error) {
+	if !controller.authorizeGRPC(ctx, v1pkg.ServiceAccountRoleComputeWrite) {
+		return nil, status.Errorf(codes.Unauthenticated, "auth failed")
+	}
+
+	sessionMetadataValue := metadata.ValueFromIncomingContext(ctx, rpc.MetadataWorkerPortForwardingSessionKey)
+	if len(sessionMetadataValue) == 0 {
+		return nil, status.Errorf(codes.InvalidArgument, "no session in metadata")
+	}
+
+	// Respond with the resolved IP address
+	_, err := controller.ipRendezvous.Respond(sessionMetadataValue[0], request.Ip)
+	if err != nil {
+		return nil, err
+	}
+
+	return &emptypb.Empty{}, nil
+}

internal/controller/sshserver/sshserver.go

@@ -6,7 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/cirruslabs/orchard/internal/controller/notifier"
-	proxypkg "github.com/cirruslabs/orchard/internal/controller/proxy"
+	"github.com/cirruslabs/orchard/internal/controller/rendezvous"
 	storepkg "github.com/cirruslabs/orchard/internal/controller/store"
 	"github.com/cirruslabs/orchard/internal/proxy"
 	"github.com/cirruslabs/orchard/pkg/resource/v1"
@@ -31,7 +31,7 @@ type SSHServer struct {
 	listener       net.Listener
 	serverConfig   *ssh.ServerConfig
 	store          storepkg.Store
-	proxy          *proxypkg.Proxy
+	connRendezvous *rendezvous.Rendezvous[net.Conn]
 	workerNotifier *notifier.Notifier
 	logger         *zap.SugaredLogger
 }
@@ -40,14 +40,14 @@ func NewSSHServer(
 	address string,
 	signer ssh.Signer,
 	store storepkg.Store,
-	proxy *proxypkg.Proxy,
+	connRendezvous *rendezvous.Rendezvous[net.Conn],
 	workerNotifier *notifier.Notifier,
 	noClientAuth bool,
 	logger *zap.SugaredLogger,
 ) (*SSHServer, error) {
 	server := &SSHServer{
 		store:          store,
-		proxy:          proxy,
+		connRendezvous: connRendezvous,
 		workerNotifier: workerNotifier,
 		logger:         logger,
 	}
@@ -232,7 +232,7 @@ func (server *SSHServer) handleDirectTCPIP(ctx context.Context, newChannel ssh.N
 	// The user wants to connect to an existing VM, request and wait
 	// for a connection with the worker before accepting the channel
 	session := uuid.New().String()
-	boomerangConnCh, cancel := server.proxy.Request(ctx, session)
+	boomerangConnCh, cancel := server.connRendezvous.Request(ctx, session)
 	defer cancel()
 
 	err = server.workerNotifier.Notify(ctx, vm.Worker, &rpc.WatchInstruction{

internal/tests/ip_endpoint_test.go

@@ -0,0 +1,62 @@
+package tests_test
+
+import (
+	"context"
+	"github.com/cirruslabs/orchard/internal/tests/devcontroller"
+	"github.com/cirruslabs/orchard/internal/tests/wait"
+	v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/crypto/ssh"
+	"net"
+	"testing"
+	"time"
+)
+
+func TestIPEndpoint(t *testing.T) {
+	// Run the Controller
+	devClient, _, _ := devcontroller.StartIntegrationTestEnvironment(t)
+
+	// Create a VM to which we'll connect via Controller's SSH server
+	err := devClient.VMs().Create(context.Background(), &v1.VM{
+		Meta: v1.Meta{
+			Name: "test-vm",
+		},
+		Image:    "ghcr.io/cirruslabs/macos-sonoma-base:latest",
+		CPU:      4,
+		Memory:   8 * 1024,
+		Headless: true,
+		Status:   v1.VMStatusPending,
+	})
+	require.NoError(t, err)
+
+	// Wait for the VM to start
+	require.True(t, wait.Wait(2*time.Minute, func() bool {
+		vm, err := devClient.VMs().Get(context.Background(), "test-vm")
+		require.NoError(t, err)
+
+		return vm.Status == v1.VMStatusRunning
+	}), "failed to wait for the VM to start")
+
+	// Retrieve the VM's IP
+	ip, err := devClient.VMs().IP(context.Background(), "test-vm", 30)
+	require.NoError(t, err)
+
+	// Connect to the VM over SSH to make sure the provided IP is valid
+	sshClient, err := ssh.Dial("tcp", ip+":22", &ssh.ClientConfig{
+		User: "admin",
+		Auth: []ssh.AuthMethod{
+			ssh.Password("admin"),
+		},
+		HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+			return nil
+		},
+	})
+	require.NoError(t, err)
+
+	sshSession, err := sshClient.NewSession()
+	require.NoError(t, err)
+
+	output, err := sshSession.CombinedOutput("uname -a")
+	require.NoError(t, err)
+	require.Contains(t, string(output), "Darwin")
+}

internal/worker/rpc.go

@@ -57,6 +57,8 @@ func (worker *Worker) watchRPC(ctx context.Context) error {
 			go worker.handlePortForward(ctxWithMetadata, client, action.PortForwardAction)
 		case *rpc.WatchInstruction_SyncVmsAction:
 			worker.requestVMSyncing()
+		case *rpc.WatchInstruction_ResolveIpAction:
+			worker.handleGetIP(ctxWithMetadata, client, action.ResolveIpAction)
 		}
 	}
 }
@@ -129,3 +131,46 @@ func (worker *Worker) handlePortForward(
 
 	_ = proxy.Connections(vmConn, grpcConn)
 }
+
+func (worker *Worker) handleGetIP(
+	ctx context.Context,
+	client rpc.ControllerClient,
+	resolveIP *rpc.WatchInstruction_ResolveIP,
+) {
+	grpcMetadata := metadata.Join(
+		worker.grpcMetadata(),
+		metadata.Pairs(rpc.MetadataWorkerPortForwardingSessionKey, resolveIP.Session),
+	)
+	ctxWithMetadata := metadata.NewOutgoingContext(ctx, grpcMetadata)
+
+	// Find the desired VM
+	vm, ok := lo.Find(worker.vmm.List(), func(item *vmmanager.VM) bool {
+		return item.Resource.UID == resolveIP.VmUid
+	})
+	if !ok {
+		worker.logger.Warnf("failed to resolve IP for the VM with UID %q: VM not found",
+			resolveIP.VmUid)
+
+		return
+	}
+
+	// Obtain VM's IP address
+	ip, err := vm.IP(ctx)
+	if err != nil {
+		worker.logger.Warnf("failed to resolve IP for the VM with UID %q: \"tart ip\" failed: %v",
+			resolveIP.VmUid, err)
+
+		return
+	}
+
+	_, err = client.ResolveIP(ctxWithMetadata, &rpc.ResolveIPResult{
+		Session: resolveIP.Session,
+		Ip:      ip,
+	})
+	if err != nil {
+		worker.logger.Warnf("failed to resolve IP for the VM with UID %q: "+
+			"failed to call back to the controller: %v", resolveIP.VmUid, err)
+
+		return
+	}
+}

internal/worker/vmmanager/vm.go

@@ -216,7 +216,7 @@ func (vm *VM) cloneAndConfigure(ctx context.Context) error {
 	//
 	// See https://github.com/cirruslabs/orchard/issues/181 for more details.
 	if vm.Resource.NetBridged != "" {
-		_, _, err = tart.Tart(ctx, vm.logger, "set", "--random-mac")
+		_, _, err = tart.Tart(ctx, vm.logger, "set", "--random-mac", vm.id())
 		if err != nil {
 			return err
 		}