pkg/hostagent: Use in-process SSH client on executing requirement scripts

go.mod

@@ -117,7 +117,7 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
 	github.com/yuin/gopher-lua v1.1.1 // indirect
-	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/crypto v0.45.0
 	golang.org/x/mod v0.29.0 // indirect
 	golang.org/x/oauth2 v0.32.0 // indirect
 	golang.org/x/term v0.37.0 // indirect

pkg/driver/vz/vm_darwin.go

@@ -113,18 +113,18 @@ func startVM(ctx context.Context, inst *limatype.Instance, sshLocalPort int) (vm
 								useSSHOverVsock = b
 							}
 						}
+						hostAddress := net.JoinHostPort(inst.SSHAddress, strconv.Itoa(usernetSSHLocalPort))
 						if !useSSHOverVsock {
 							logrus.Info("LIMA_SSH_OVER_VSOCK is false, skipping detection of SSH server on vsock port")
-						} else if err := usernetClient.WaitOpeningSSHPort(ctx, inst); err == nil {
-							hostAddress := net.JoinHostPort(inst.SSHAddress, strconv.Itoa(usernetSSHLocalPort))
-							if err := wrapper.startVsockForwarder(ctx, 22, hostAddress); err == nil {
-								logrus.Infof("Detected SSH server is listening on the vsock port; changed %s to proxy for the vsock port", hostAddress)
-								usernetSSHLocalPort = 0 // disable gvisor ssh port forwarding
-							} else {
-								logrus.WithError(err).Warn("Failed to detect SSH server on vsock port, falling back to usernet forwarder")
-							}
+						} else if err := usernetClient.WaitOpeningSSHPort(ctx, inst); err != nil {
+							logrus.WithError(err).Info("Failed to wait for the guest SSH server to become available, falling back to usernet forwarder")
+						} else if err := wrapper.checkSSHOverVsockAvailable(ctx, inst); err != nil {
+							logrus.WithError(err).Info("Failed to detect SSH server on vsock port, falling back to usernet forwarder")
+						} else if err := wrapper.startVsockForwarder(ctx, 22, hostAddress); err != nil {
+							logrus.WithError(err).Info("Failed to start SSH server forwarder on vsock port, falling back to usernet forwarder")
 						} else {
-							logrus.WithError(err).Warn("Failed to wait for the guest SSH server to become available, falling back to usernet forwarder")
+							logrus.Infof("Detected SSH server is listening on the vsock port; changed %s to proxy for the vsock port", hostAddress)
+							usernetSSHLocalPort = 0 // disable gvisor ssh port forwarding
 						}
 						err := usernetClient.ConfigureDriver(ctx, inst, usernetSSHLocalPort)
 						if err != nil {

pkg/driver/vz/vsock_forwarder.go

@@ -12,17 +12,14 @@ import (
 
 	"github.com/containers/gvisor-tap-vsock/pkg/tcpproxy"
 	"github.com/sirupsen/logrus"
+
+	"github.com/lima-vm/lima/v2/pkg/limatype"
+	"github.com/lima-vm/lima/v2/pkg/sshutil"
 )
 
 func (m *virtualMachineWrapper) startVsockForwarder(ctx context.Context, vsockPort uint32, hostAddress string) error {
-	// Test if the vsock port is open
-	conn, err := m.dialVsock(ctx, vsockPort)
-	if err != nil {
-		return err
-	}
-	conn.Close()
 	// Start listening on localhost:hostPort and forward to vsock:vsockPort
-	_, _, err = net.SplitHostPort(hostAddress)
+	_, _, err := net.SplitHostPort(hostAddress)
 	if err != nil {
 		return err
 	}
@@ -73,3 +70,9 @@ func (m *virtualMachineWrapper) dialVsock(_ context.Context, port uint32) (conn
 	}
 	return nil, err
 }
+
+func (m *virtualMachineWrapper) checkSSHOverVsockAvailable(ctx context.Context, inst *limatype.Instance) error {
+	return sshutil.WaitSSHReady(ctx, func(ctx context.Context) (net.Conn, error) {
+		return m.dialVsock(ctx, uint32(22))
+	}, "vsock:22", *inst.Config.User.Name, 1)
+}

pkg/hostagent/requirements.go

@@ -6,8 +6,11 @@ package hostagent
 import (
 	"errors"
 	"fmt"
+	"os"
 	"runtime"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/lima-vm/sshocker/pkg/ssh"
@@ -103,39 +106,65 @@ func (a *HostAgent) waitForRequirement(r requirement) error {
 	if err != nil {
 		return err
 	}
+	var stdout, stderr string
 	sshConfig := a.sshConfig
-	if r.noMaster || runtime.GOOS == "windows" {
-		// Remove ControlMaster, ControlPath, and ControlPersist options,
-		// because Cygwin-based SSH clients do not support multiplexing when executing commands.
-		// References:
-		//   https://inbox.sourceware.org/cygwin/c98988a5-7e65-4282-b2a1-bb8e350d5fab@acm.org/T/
-		//   https://stackoverflow.com/questions/20959792/is-ssh-controlmaster-with-cygwin-on-windows-actually-possible
-		// By removing these options:
-		//   - Avoids execution failures when the control master is not yet available.
-		//   - Prevents error messages such as:
-		//     > mux_client_request_session: read from master failed: Connection reset by peer
-		//     > ControlSocket ....sock already exists, disabling multiplexing
-		//     > mm_send_fd: sendmsg(2): Connection reset by peer\\r\\nmux_client_request_session: send fds failed\\r\\n
-		sshConfig = &ssh.SSHConfig{
-			ConfigFile:     sshConfig.ConfigFile,
-			Persist:        false,
-			AdditionalArgs: sshutil.DisableControlMasterOptsFromSSHArgs(sshConfig.AdditionalArgs),
+	if r.external || determineUseExternalSSH() {
+		if r.noMaster || runtime.GOOS == "windows" {
+			// Remove ControlMaster, ControlPath, and ControlPersist options,
+			// because Cygwin-based SSH clients do not support multiplexing when executing commands.
+			// References:
+			//   https://inbox.sourceware.org/cygwin/c98988a5-7e65-4282-b2a1-bb8e350d5fab@acm.org/T/
+			//   https://stackoverflow.com/questions/20959792/is-ssh-controlmaster-with-cygwin-on-windows-actually-possible
+			// By removing these options:
+			//   - Avoids execution failures when the control master is not yet available.
+			//   - Prevents error messages such as:
+			//     > mux_client_request_session: read from master failed: Connection reset by peer
+			//     > ControlSocket ....sock already exists, disabling multiplexing
+			//     > mm_send_fd: sendmsg(2): Connection reset by peer\\r\\nmux_client_request_session: send fds failed\\r\\n
+			sshConfig = &ssh.SSHConfig{
+				ConfigFile:     sshConfig.ConfigFile,
+				Persist:        false,
+				AdditionalArgs: sshutil.DisableControlMasterOptsFromSSHArgs(sshConfig.AdditionalArgs),
+			}
 		}
+		stdout, stderr, err = ssh.ExecuteScript(a.instSSHAddress, a.sshLocalPort, sshConfig, script, r.description)
+	} else {
+		stdout, stderr, err = sshutil.ExecuteScriptViaInProcessClient(a.instSSHAddress, a.sshLocalPort, *a.instConfig.User.Name, script, r.description)
 	}
-	stdout, stderr, err := ssh.ExecuteScript(a.instSSHAddress, a.sshLocalPort, sshConfig, script, r.description)
 	logrus.Debugf("stdout=%q, stderr=%q, err=%v", stdout, stderr, err)
 	if err != nil {
 		return fmt.Errorf("stdout=%q, stderr=%q: %w", stdout, stderr, err)
 	}
 	return nil
 }
 
+var determineUseExternalSSH = sync.OnceValue(func() bool {
+	var useExternalSSH bool
+	// allow overriding via LIMA_EXTERNAL_SSH_REQUIREMENT environment variable
+	if envVar := os.Getenv("LIMA_EXTERNAL_SSH_REQUIREMENT"); envVar != "" {
+		if b, err := strconv.ParseBool(envVar); err != nil {
+			logrus.WithError(err).Warnf("invalid LIMA_EXTERNAL_SSH_REQUIREMENT value %q", envVar)
+		} else {
+			useExternalSSH = b
+		}
+	}
+	if useExternalSSH {
+		logrus.Info("using external ssh command for executing requirement scripts")
+	} else {
+		logrus.Info("using in-process ssh client for executing requirement scripts")
+	}
+	return useExternalSSH
+})
+
 type requirement struct {
 	description string
 	script      string
 	debugHint   string
 	fatal       bool
 	noMaster    bool
+	// Execute the script externally via the ssh command instead of using the in-process client.
+	// noMaster will be ignored if external is false.
+	external bool
 }
 
 func (a *HostAgent) essentialRequirements() []requirement {
@@ -158,6 +187,7 @@ If any private key under ~/.ssh is protected with a passphrase, you need to have
 true
 `,
 		debugHint: `The persistent ssh ControlMaster should be started immediately.`,
+		external:  true,
 	}
 	if *a.instConfig.Plain {
 		req = append(req, startControlMasterReq)

pkg/networks/usernet/client.go

@@ -140,8 +140,9 @@ func (c *Client) WaitOpeningSSHPort(ctx context.Context, inst *limatype.Instance
 	if err != nil {
 		return err
 	}
+	user := *inst.Config.User.Name
 	// -1 avoids both sides timing out simultaneously.
-	u := fmt.Sprintf("%s/extension/wait_port?ip=%s&port=22&timeout=%d", c.base, ipAddr, timeoutSeconds-1)
+	u := fmt.Sprintf("%s/extension/wait-ssh-server?ip=%s&port=22&timeout=%d&user=%s", c.base, ipAddr, timeoutSeconds-1, user)
 	res, err := httpclientutil.Get(ctx, c.client, u)
 	if err != nil {
 		return err

pkg/networks/usernet/gvproxy.go

@@ -22,6 +22,8 @@ import (
 	"github.com/containers/gvisor-tap-vsock/pkg/virtualnetwork"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/lima-vm/lima/v2/pkg/sshutil"
 )
 
 type GVisorNetstackOpts struct {
@@ -243,7 +245,7 @@ func httpServe(ctx context.Context, g *errgroup.Group, ln net.Listener, mux http
 
 func muxWithExtension(n *virtualnetwork.VirtualNetwork) *http.ServeMux {
 	m := n.Mux()
-	m.HandleFunc("/extension/wait_port", func(w http.ResponseWriter, r *http.Request) {
+	m.HandleFunc("/extension/wait-ssh-server", func(w http.ResponseWriter, r *http.Request) {
 		ip := r.URL.Query().Get("ip")
 		if net.ParseIP(ip) == nil {
 			msg := fmt.Sprintf("invalid ip address: %s", ip)
@@ -255,8 +257,14 @@ func muxWithExtension(n *virtualnetwork.VirtualNetwork) *http.ServeMux {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-		port := uint16(port16)
-		addr := fmt.Sprintf("%s:%d", ip, port)
+		addr := net.JoinHostPort(ip, fmt.Sprintf("%d", uint16(port16)))
+
+		user := r.URL.Query().Get("user")
+		if user == "" {
+			msg := "user query parameter is required"
+			http.Error(w, msg, http.StatusBadRequest)
+			return
+		}
 
 		timeoutSeconds := 10
 		if timeoutString := r.URL.Query().Get("timeout"); timeoutString != "" {
@@ -267,27 +275,14 @@ func muxWithExtension(n *virtualnetwork.VirtualNetwork) *http.ServeMux {
 			}
 			timeoutSeconds = int(timeout16)
 		}
-		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds)*time.Second)
-		defer cancel()
+		dialContext := func(ctx context.Context) (net.Conn, error) {
+			return n.DialContextTCP(ctx, addr)
+		}
 		// Wait until the port is available.
-		for {
-			conn, err := n.DialContextTCP(ctx, addr)
-			if err == nil {
-				conn.Close()
-				logrus.Debugf("Port is available on %s", addr)
-				w.WriteHeader(http.StatusOK)
-				break
-			}
-			select {
-			case <-ctx.Done():
-				msg := fmt.Sprintf("timed out waiting for port to become available on %s", addr)
-				logrus.Warn(msg)
-				http.Error(w, msg, http.StatusRequestTimeout)
-				return
-			default:
-			}
-			logrus.Debugf("Waiting for port to become available on %s", addr)
-			time.Sleep(1 * time.Second)
+		if err = sshutil.WaitSSHReady(r.Context(), dialContext, addr, user, timeoutSeconds); err != nil {
+			http.Error(w, err.Error(), http.StatusRequestTimeout)
+		} else {
+			w.WriteHeader(http.StatusOK)
 		}
 	})
 	return m

pkg/osutil/osutil_unix.go

@@ -8,6 +8,7 @@ package osutil
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -36,3 +37,7 @@ func Sysctl(ctx context.Context, name string) (string, error) {
 	}
 	return strings.TrimSuffix(string(stdout), "\n"), nil
 }
+
+func IsConnectionResetError(err error) bool {
+	return errors.Is(err, syscall.ECONNRESET)
+}

pkg/osutil/osutil_windows.go

@@ -57,3 +57,7 @@ func SignalName(sig os.Signal) string {
 func Sysctl(_ context.Context, _ string) (string, error) {
 	return "", errors.New("sysctl: unimplemented on Windows")
 }
+
+func IsConnectionResetError(err error) bool {
+	return errors.Is(err, syscall.WSAECONNRESET)
+}