From bf1ecd576752dd56ddf2602f7866f3492fc39a06 Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Mon, 3 Feb 2025 11:04:18 -0500 Subject: [PATCH 1/6] Init UnixPortforwardAdapter --- .../adapters/__init__.py | 4 +-- .../adapters/fabric.py | 4 +-- .../adapters/novnc.py | 4 +-- .../adapters/pexpect.py | 4 +-- .../adapters/portforward.py | 30 +++++++++++++------ .../jumpstarter_driver_network/driver_test.py | 17 +++++++++-- 6 files changed, 43 insertions(+), 20 deletions(-) diff --git a/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/__init__.py b/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/__init__.py index cc043bced..fa23bcc2b 100644 --- a/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/__init__.py +++ b/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/__init__.py @@ -1,6 +1,6 @@ from .fabric import FabricAdapter from .novnc import NovncAdapter from .pexpect import PexpectAdapter -from .portforward import PortforwardAdapter +from .portforward import TcpPortforwardAdapter, UnixPortforwardAdapter -__all__ = ["FabricAdapter", "NovncAdapter", "PexpectAdapter", "PortforwardAdapter"] +__all__ = ["FabricAdapter", "NovncAdapter", "PexpectAdapter", "TcpPortforwardAdapter", "UnixPortforwardAdapter"] diff --git a/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/fabric.py b/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/fabric.py index 59ec08b41..afff5e88f 100644 --- a/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/fabric.py +++ b/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/fabric.py @@ -4,11 +4,11 @@ from fabric.config import Config from fabric.connection import Connection -from .portforward import PortforwardAdapter +from .portforward import TcpPortforwardAdapter @dataclass(kw_only=True) -class FabricAdapter(PortforwardAdapter): +class FabricAdapter(TcpPortforwardAdapter): user: str | None = None config: Config | None = None forward_agent: bool | None = None diff --git a/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/novnc.py b/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/novnc.py index 7ccd93cc7..8756f3a0e 100644 --- a/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/novnc.py +++ b/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/novnc.py @@ -2,12 +2,12 @@ from urllib.parse import urlencode, urlunparse from ..streams import WebsocketServerStream -from .portforward import PortforwardAdapter +from .portforward import TcpPortforwardAdapter from jumpstarter.streams import forward_stream @dataclass(kw_only=True) -class NovncAdapter(PortforwardAdapter): +class NovncAdapter(TcpPortforwardAdapter): async def __aenter__(self): addr = await super().__aenter__() return urlunparse( diff --git a/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/pexpect.py b/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/pexpect.py index 31a7e4e89..653f88567 100644 --- a/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/pexpect.py +++ b/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/pexpect.py @@ -3,11 +3,11 @@ from pexpect.fdpexpect import fdspawn -from .portforward import PortforwardAdapter +from .portforward import TcpPortforwardAdapter @dataclass(kw_only=True) -class PexpectAdapter(PortforwardAdapter): +class PexpectAdapter(TcpPortforwardAdapter): async def __aenter__(self): addr = await super().__aenter__() diff --git a/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/portforward.py b/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/portforward.py index c180fb340..45024a1b9 100644 --- a/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/portforward.py +++ b/packages/jumpstarter-driver-network/jumpstarter_driver_network/adapters/portforward.py @@ -1,15 +1,28 @@ from dataclasses import dataclass from jumpstarter.client.adapters import ClientAdapter -from jumpstarter.common import TemporaryTcpListener +from jumpstarter.common import TemporaryTcpListener, TemporaryUnixListener from jumpstarter.streams import forward_stream @dataclass(kw_only=True) class PortforwardAdapter(ClientAdapter): + method: str = "connect" + + async def __aexit__(self, exc_type, exc_value, traceback): + return await self.listener.__aexit__(exc_type, exc_value, traceback) + + async def handler(self, conn): + async with conn: + async with self.client.stream_async(self.method) as stream: + async with forward_stream(conn, stream): + pass + + +@dataclass(kw_only=True) +class TcpPortforwardAdapter(PortforwardAdapter): local_host: str = "127.0.0.1" local_port: int = 0 - method: str = "connect" async def __aenter__(self): self.listener = TemporaryTcpListener( @@ -18,11 +31,10 @@ async def __aenter__(self): return await self.listener.__aenter__() - async def __aexit__(self, exc_type, exc_value, traceback): - return await self.listener.__aexit__(exc_type, exc_value, traceback) - async def handler(self, conn): - async with conn: - async with self.client.stream_async(self.method) as stream: - async with forward_stream(conn, stream): - pass +@dataclass(kw_only=True) +class UnixPortforwardAdapter(PortforwardAdapter): + async def __aenter__(self): + self.listener = TemporaryUnixListener(self.handler) + + return await self.listener.__aenter__() diff --git a/packages/jumpstarter-driver-network/jumpstarter_driver_network/driver_test.py b/packages/jumpstarter-driver-network/jumpstarter_driver_network/driver_test.py index a0e2a4dce..80e4c4864 100644 --- a/packages/jumpstarter-driver-network/jumpstarter_driver_network/driver_test.py +++ b/packages/jumpstarter-driver-network/jumpstarter_driver_network/driver_test.py @@ -6,7 +6,7 @@ import pytest from anyio.from_thread import start_blocking_portal -from .adapters import PortforwardAdapter +from .adapters import TcpPortforwardAdapter, UnixPortforwardAdapter from .driver import EchoNetwork, TcpNetwork, UdpNetwork, UnixNetwork from jumpstarter.common import TemporaryTcpListener, TemporaryUnixListener from jumpstarter.common.utils import serve @@ -41,13 +41,24 @@ def test_tcp_network_portforward(): with start_blocking_portal() as portal: with portal.wrap_async_context_manager(TemporaryTcpListener(echo_handler, local_host="127.0.0.1")) as inner: with serve(TcpNetwork(host=inner[0], port=inner[1])) as client: - with PortforwardAdapter(client=client) as addr: + with TcpPortforwardAdapter(client=client) as addr: stream = socket.socket(socket.AF_INET, socket.SOCK_STREAM) stream.connect(addr) stream.send(b"hello") assert stream.recv(5) == b"hello" +def test_unix_network_portforward(): + with start_blocking_portal() as portal: + with portal.wrap_async_context_manager(TemporaryUnixListener(echo_handler)) as inner: + with serve(UnixNetwork(path=inner)) as client: + with UnixPortforwardAdapter(client=client) as addr: + stream = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + stream.connect(str(addr)) + stream.send(b"hello") + assert stream.recv(5) == b"hello" + + def test_udp_network(): with serve( UdpNetwork( @@ -90,7 +101,7 @@ def test_tcp_network_performance(): stderr=subprocess.DEVNULL, ) - with PortforwardAdapter(client=client) as addr: + with TcpPortforwardAdapter(client=client) as addr: subprocess.run( [ "iperf3", From 2eaad8383efa845f2f5ce02124ccc9dc308d90cc Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Mon, 3 Feb 2025 11:44:58 -0500 Subject: [PATCH 2/6] Add docs for portforward adapters --- docs/source/api-reference/adapters/index.md | 5 ++ docs/source/api-reference/adapters/network.md | 51 +++++++++++++++++++ docs/source/api-reference/index.md | 1 + 3 files changed, 57 insertions(+) create mode 100644 docs/source/api-reference/adapters/index.md create mode 100644 docs/source/api-reference/adapters/network.md diff --git a/docs/source/api-reference/adapters/index.md b/docs/source/api-reference/adapters/index.md new file mode 100644 index 000000000..19071a00c --- /dev/null +++ b/docs/source/api-reference/adapters/index.md @@ -0,0 +1,5 @@ +# Adapter Packages + +```{toctree} +network.md +``` diff --git a/docs/source/api-reference/adapters/network.md b/docs/source/api-reference/adapters/network.md new file mode 100644 index 000000000..5acbf0f05 --- /dev/null +++ b/docs/source/api-reference/adapters/network.md @@ -0,0 +1,51 @@ +# Network adapters + +Network adapters are for transforming network connections exposed by drivers + +```{eval-rst} +.. autoclass:: jumpstarter_driver_network.adapters.TcpPortforwardAdapter + :members: +``` + +```{eval-rst} +.. autoclass:: jumpstarter_driver_network.adapters.UnixPortforwardAdapter + :members: +``` + +## Examples +```yaml +export: + tcp_port: + type: "jumpstarter_driver_network.driver.TcpNetwork" + config: + host: localhost + port: 80 + unix_socket: + type: "jumpstarter_driver_network.driver.UnixNetwork" + config: + path: /tmp/test.sock +``` + +Forward a remote TCP port to a local TCP port + +```{testcode} +# random port on localhost +with TcpPortforwardAdapter(client.tcp_port) as addr: + print(addr[0], addr[1]) # 127.0.0.1 38406 + +# specific address and port +with TcpPortforwardAdapter(client.tcp_port, local_host="192.0.2.1", local_port=8080) as addr: + print(addr[0], addr[1]) # 192.0.2.1 8080 +``` + +Forward a remote Unix domain socket to a local socket + +```{testcode} +with UnixPortforwardAdapter(client.unix_socket) as addr: + print(addr) # /tmp/jumpstarter-w30wxu64/socket + +# the type of the remote socket and the local one doesn't have to match +# e.g. forward a remote Unix domain socket to a local TCP port +with TcpPortforwardAdapter(client.unix_socket) as addr: + print(addr[0], addr[1]) # 127.0.0.1 38406 +``` diff --git a/docs/source/api-reference/index.md b/docs/source/api-reference/index.md index 5d3d1e755..be73ea2b2 100644 --- a/docs/source/api-reference/index.md +++ b/docs/source/api-reference/index.md @@ -6,4 +6,5 @@ This section provides details on the Jumpstarter core API and contrib drivers. drivers.md adapters.md drivers/index.md +adapters/index.md ``` From 8bdbdcd32a0db1c7865957cbf585a6a80d906816 Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Mon, 3 Feb 2025 11:50:23 -0500 Subject: [PATCH 3/6] Document NovncAdapter --- docs/source/api-reference/adapters/network.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/source/api-reference/adapters/network.md b/docs/source/api-reference/adapters/network.md index 5acbf0f05..afebd48e3 100644 --- a/docs/source/api-reference/adapters/network.md +++ b/docs/source/api-reference/adapters/network.md @@ -12,6 +12,11 @@ Network adapters are for transforming network connections exposed by drivers :members: ``` +```{eval-rst} +.. autoclass:: jumpstarter_driver_network.adapters.NovncAdapter + :members: +``` + ## Examples ```yaml export: @@ -49,3 +54,11 @@ with UnixPortforwardAdapter(client.unix_socket) as addr: with TcpPortforwardAdapter(client.unix_socket) as addr: print(addr[0], addr[1]) # 127.0.0.1 38406 ``` + +Connect to a remote TCP port with a web-based VNC client + +```{testcode} +with NovncAdapter(client.tcp_port) as url: + print(url) # https://novnc.com/noVNC/vnc.html?autoconnect=1&reconnect=1&host=127.0.0.1&port=36459 + # open the url in browser to access the VNC client +``` From 5c242c5fbb6624a3a566843a6376f47c3e5dd945 Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Mon, 3 Feb 2025 11:53:28 -0500 Subject: [PATCH 4/6] Document PexpectAdapter --- docs/source/api-reference/adapters/network.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/source/api-reference/adapters/network.md b/docs/source/api-reference/adapters/network.md index afebd48e3..ef9dc7ea9 100644 --- a/docs/source/api-reference/adapters/network.md +++ b/docs/source/api-reference/adapters/network.md @@ -17,6 +17,11 @@ Network adapters are for transforming network connections exposed by drivers :members: ``` +```{eval-rst} +.. autoclass:: jumpstarter_driver_network.adapters.PexpectAdapter + :members: +``` + ## Examples ```yaml export: @@ -62,3 +67,13 @@ with NovncAdapter(client.tcp_port) as url: print(url) # https://novnc.com/noVNC/vnc.html?autoconnect=1&reconnect=1&host=127.0.0.1&port=36459 # open the url in browser to access the VNC client ``` + +Interact with a remote TCP port as if it's a serial console + +```{testcode} +with PexpectAdapter(client.tcp_port) as expect: + expect.expect("localhost login:") + expect.send("root\n") + expect.expect("Password:") + expect.send("secret\n") +``` From ee4e176508fa061e8776de23fdb3e70075985c41 Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Mon, 3 Feb 2025 11:55:09 -0500 Subject: [PATCH 5/6] Document FabricAdapter --- docs/source/api-reference/adapters/network.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/source/api-reference/adapters/network.md b/docs/source/api-reference/adapters/network.md index ef9dc7ea9..11b9d9031 100644 --- a/docs/source/api-reference/adapters/network.md +++ b/docs/source/api-reference/adapters/network.md @@ -22,6 +22,11 @@ Network adapters are for transforming network connections exposed by drivers :members: ``` +```{eval-rst} +.. autoclass:: jumpstarter_driver_network.adapters.FabricAdapter + :members: +``` + ## Examples ```yaml export: @@ -77,3 +82,10 @@ with PexpectAdapter(client.tcp_port) as expect: expect.expect("Password:") expect.send("secret\n") ``` + +Connect to a remote TCP port with the fabric SSH client + +```{testcode} +with FabricAdapter(client=client.tcp_port, connect_kwargs={"password": "secret"}) as conn: + conn.run("uname") +``` From 135c98384d85d2997b3d831f5405742b93dc63ca Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Mon, 3 Feb 2025 11:57:29 -0500 Subject: [PATCH 6/6] Add links to upstream documentation --- docs/source/api-reference/adapters/network.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/api-reference/adapters/network.md b/docs/source/api-reference/adapters/network.md index 11b9d9031..8788b8639 100644 --- a/docs/source/api-reference/adapters/network.md +++ b/docs/source/api-reference/adapters/network.md @@ -75,6 +75,8 @@ with NovncAdapter(client.tcp_port) as url: Interact with a remote TCP port as if it's a serial console +See [pexpect](https://pexpect.readthedocs.io/en/stable/api/fdpexpect.html) for API documentation + ```{testcode} with PexpectAdapter(client.tcp_port) as expect: expect.expect("localhost login:") @@ -85,6 +87,8 @@ with PexpectAdapter(client.tcp_port) as expect: Connect to a remote TCP port with the fabric SSH client +See [fabric](https://docs.fabfile.org/en/latest/api/connection.html#fabric.connection.Connection) for API documentation + ```{testcode} with FabricAdapter(client=client.tcp_port, connect_kwargs={"password": "secret"}) as conn: conn.run("uname")