SSH handler/listener plugins (#1398)

* SSH handler/listener plugins

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Readme updated

* Fix listener tests

* pyclassrole

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Trigger rebuild

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Trigger build

* pre-commit default language version 3.10

* Language version

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.pre-commit-config.yaml

@@ -144,7 +144,7 @@ repos:
  rev: 3.9.2
  hooks:
  - id: flake8
- language_version: python3
+ language_version: python3.10
  additional_dependencies:
  - flake8-2020 >= 1.6.0
  - flake8-docstrings >= 1.5.0

README.md

@@ -2341,25 +2341,25 @@ To run standalone benchmark for `proxy.py`, use the following command from repo
 
 ```console
 ❯ proxy -h
-usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT]
- [--tunnel-username TUNNEL_USERNAME]
+usage: -m [-h] [--threadless] [--threaded] [--num-workers NUM_WORKERS]
+ [--enable-events] [--local-executor LOCAL_EXECUTOR]
+ [--backlog BACKLOG] [--hostname HOSTNAME]
+ [--hostnames HOSTNAMES [HOSTNAMES ...]] [--port PORT]
+ [--ports PORTS [PORTS ...]] [--port-file PORT_FILE]
+ [--unix-socket-path UNIX_SOCKET_PATH]
+ [--num-acceptors NUM_ACCEPTORS] [--tunnel-hostname TUNNEL_HOSTNAME]
+ [--tunnel-port TUNNEL_PORT] [--tunnel-username TUNNEL_USERNAME]
  [--tunnel-ssh-key TUNNEL_SSH_KEY]
  [--tunnel-ssh-key-passphrase TUNNEL_SSH_KEY_PASSPHRASE]
- [--tunnel-remote-port TUNNEL_REMOTE_PORT] [--threadless]
- [--threaded] [--num-workers NUM_WORKERS] [--enable-events]
- [--local-executor LOCAL_EXECUTOR] [--backlog BACKLOG]
- [--hostname HOSTNAME] [--hostnames HOSTNAMES [HOSTNAMES ...]]
- [--port PORT] [--ports PORTS [PORTS ...]] [--port-file PORT_FILE]
- [--unix-socket-path UNIX_SOCKET_PATH]
- [--num-acceptors NUM_ACCEPTORS] [--version] [--log-level LOG_LEVEL]
- [--log-file LOG_FILE] [--log-format LOG_FORMAT]
- [--open-file-limit OPEN_FILE_LIMIT]
+ [--tunnel-remote-port TUNNEL_REMOTE_PORT] [--version]
+ [--log-level LOG_LEVEL] [--log-file LOG_FILE]
+ [--log-format LOG_FORMAT] [--open-file-limit OPEN_FILE_LIMIT]
  [--plugins PLUGINS [PLUGINS ...]] [--enable-dashboard]
  [--basic-auth BASIC_AUTH] [--enable-ssh-tunnel]
  [--work-klass WORK_KLASS] [--pid-file PID_FILE] [--openssl OPENSSL]
- [--data-dir DATA_DIR] [--enable-proxy-protocol] [--enable-conn-pool]
- [--key-file KEY_FILE] [--cert-file CERT_FILE]
- [--client-recvbuf-size CLIENT_RECVBUF_SIZE]
+ [--data-dir DATA_DIR] [--ssh-listener-klass SSH_LISTENER_KLASS]
+ [--enable-proxy-protocol] [--enable-conn-pool] [--key-file KEY_FILE]
+ [--cert-file CERT_FILE] [--client-recvbuf-size CLIENT_RECVBUF_SIZE]
  [--server-recvbuf-size SERVER_RECVBUF_SIZE]
  [--max-sendbuf-size MAX_SENDBUF_SIZE] [--timeout TIMEOUT]
  [--disable-http-proxy] [--disable-headers DISABLE_HEADERS]
@@ -2379,25 +2379,10 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT]
  [--filtered-client-ips FILTERED_CLIENT_IPS]
  [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
 
-proxy.py v2.4.4rc6.dev85+g9335918b
+proxy.py v2.4.4rc6.dev164+g73497f30
 
 options:
  -h, --help show this help message and exit
- --tunnel-hostname TUNNEL_HOSTNAME
- Default: None. Remote hostname or IP address to which
- SSH tunnel will be established.
- --tunnel-port TUNNEL_PORT
- Default: 22. SSH port of the remote host.
- --tunnel-username TUNNEL_USERNAME
- Default: None. Username to use for establishing SSH
- tunnel.
- --tunnel-ssh-key TUNNEL_SSH_KEY
- Default: None. Private key path in pem format
- --tunnel-ssh-key-passphrase TUNNEL_SSH_KEY_PASSPHRASE
- Default: None. Private key passphrase
- --tunnel-remote-port TUNNEL_REMOTE_PORT
- Default: 8899. Remote port which will be forwarded
- locally for proxy.
  --threadless Default: True. Enabled by default on Python 3.8+ (mac,
  linux). When disabled a new thread is spawned to
  handle each client connection.
@@ -2434,6 +2419,21 @@ options:
  --host and --port flags are ignored
  --num-acceptors NUM_ACCEPTORS
  Defaults to number of CPU cores.
+ --tunnel-hostname TUNNEL_HOSTNAME
+ Default: None. Remote hostname or IP address to which
+ SSH tunnel will be established.
+ --tunnel-port TUNNEL_PORT
+ Default: 22. SSH port of the remote host.
+ --tunnel-username TUNNEL_USERNAME
+ Default: None. Username to use for establishing SSH
+ tunnel.
+ --tunnel-ssh-key TUNNEL_SSH_KEY
+ Default: None. Private key path in pem format
+ --tunnel-ssh-key-passphrase TUNNEL_SSH_KEY_PASSPHRASE
+ Default: None. Private key passphrase
+ --tunnel-remote-port TUNNEL_REMOTE_PORT
+ Default: 8899. Remote port which will be forwarded
+ locally for proxy.
  --version, -v Prints proxy.py version.
  --log-level LOG_LEVEL
  Valid options: DEBUG, INFO (default), WARNING, ERROR,
@@ -2461,6 +2461,9 @@ options:
  --openssl OPENSSL Default: openssl. Path to openssl binary. By default,
  assumption is that openssl is in your PATH.
  --data-dir DATA_DIR Default: ~/.proxypy. Path to proxypy data directory.
+ --ssh-listener-klass SSH_LISTENER_KLASS
+ Default: proxy.core.ssh.listener.SshTunnelListener. An
+ implementation of BaseSshTunnelListener
  --enable-proxy-protocol
  Default: False. If used, will enable proxy protocol.
  Only version 1 is currently supported.

docs/conf.py

@@ -324,6 +324,7 @@
  (_py_class_role, 're.Pattern'),
  (_py_class_role, 'proxy.core.base.tcp_server.T'),
  (_py_class_role, 'proxy.common.types.RePattern'),
+ (_py_class_role, 'BaseSshTunnelHandler'),
  (_py_obj_role, 'proxy.core.work.threadless.T'),
  (_py_obj_role, 'proxy.core.work.work.T'),
  (_py_obj_role, 'proxy.core.base.tcp_server.T'),

proxy/core/ssh/listener.py

@@ -8,21 +8,20 @@
  :copyright: (c) 2013-present by Abhinav Singh and contributors.
  :license: BSD, see LICENSE for more details.
 """
+import sys
+import socket
 import logging
 import argparse
-from typing import TYPE_CHECKING, Any, Set, Callable, Optional
+from typing import TYPE_CHECKING, Any, Set, Optional, cast
 
 
 try:
- from paramiko import SSHClient, AutoAddPolicy
- from paramiko.transport import Transport
- if TYPE_CHECKING: # pragma: no cover
- from paramiko.channel import Channel
-
+ if TYPE_CHECKING: # pragma: no cover
  from ...common.types import HostPort
 except ImportError: # pragma: no cover
  pass
 
+from .base import BaseSshTunnelHandler, BaseSshTunnelListener
 from ...common.flag import flags
 
 
@@ -72,18 +71,27 @@
 )
 
 
-class SshTunnelListener:
+class SshTunnelListener(BaseSshTunnelListener):
  """Connects over SSH and forwards a remote port to local host.
 
  Incoming connections are delegated to provided callback."""
 
  def __init__(
- self,
- flags: argparse.Namespace,
- on_connection_callback: Callable[['Channel', 'HostPort', 'HostPort'], None],
+ self,
+ flags: argparse.Namespace,
+ handler: BaseSshTunnelHandler,
+ *args: Any,
+ **kwargs: Any,
  ) -> None:
+ paramiko_logger = logging.getLogger('paramiko')
+ paramiko_logger.setLevel(logging.WARNING)
+
+ # pylint: disable=import-outside-toplevel
+ from paramiko import SSHClient
+ from paramiko.transport import Transport
+
  self.flags = flags
- self.on_connection_callback = on_connection_callback
+ self.handler = handler
  self.ssh: Optional[SSHClient] = None
  self.transport: Optional[Transport] = None
  self.forwarded: Set['HostPort'] = set()
@@ -92,24 +100,20 @@ def start_port_forward(self, remote_addr: 'HostPort') -> None:
  assert self.transport is not None
  self.transport.request_port_forward(
  *remote_addr,
- handler=self.on_connection_callback,
+ handler=self.handler.on_connection,
  )
  self.forwarded.add(remote_addr)
- logger.info('%s:%d forwarding successful...' % remote_addr)
+ logger.debug('%s:%d forwarding successful...' % remote_addr)
 
  def stop_port_forward(self, remote_addr: 'HostPort') -> None:
  assert self.transport is not None
  self.transport.cancel_port_forward(*remote_addr)
  self.forwarded.remove(remote_addr)
 
- def __enter__(self) -> 'SshTunnelListener':
- self.setup()
- return self
-
- def __exit__(self, *args: Any) -> None:
- self.shutdown()
-
  def setup(self) -> None:
+ # pylint: disable=import-outside-toplevel
+ from paramiko import SSHClient, AutoAddPolicy
+
  self.ssh = SSHClient()
  self.ssh.load_system_host_keys()
  self.ssh.set_missing_host_key_policy(AutoAddPolicy())
@@ -119,14 +123,30 @@ def setup(self) -> None:
  username=self.flags.tunnel_username,
  key_filename=self.flags.tunnel_ssh_key,
  passphrase=self.flags.tunnel_ssh_key_passphrase,
+ compress=True,
+ timeout=10,
+ auth_timeout=7,
  )
- logger.info(
- 'SSH connection established to %s:%d...' % (
+ logger.debug(
+ 'SSH connection established to %s:%d...'
+ % (
  self.flags.tunnel_hostname,
  self.flags.tunnel_port,
  ),
  )
  self.transport = self.ssh.get_transport()
+ assert self.transport
+ sock = cast(socket.socket, self.transport.sock) # type: ignore[redundant-cast]
+ # Enable TCP keep-alive
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+ # Keep-alive interval (in seconds)
+ if sys.platform != 'darwin':
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 30)
+ # Keep-alive probe interval (in seconds)
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 5)
+ # Number of keep-alive probes before timeout
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 5)
+ self.start_port_forward(('', self.flags.tunnel_remote_port))
 
  def shutdown(self) -> None:
  for remote_addr in list(self.forwarded):
@@ -136,3 +156,10 @@ def shutdown(self) -> None:
  self.transport.close()
  if self.ssh is not None:
  self.ssh.close()
+ self.handler.shutdown()
+
+ def is_alive(self) -> bool:
+ return self.transport.is_alive() if self.transport else False
+
+ def is_active(self) -> bool:
+ return self.transport.is_active() if self.transport else False

proxy/http/proxy/server.py

@@ -726,9 +726,9 @@ def generate_upstream_certificate(
  ):
  raise HttpProtocolException(
  f'For certificate generation all the following flags are mandatory: '
- f'--ca-cert-file:{ self.flags.ca_cert_file }, '
- f'--ca-key-file:{ self.flags.ca_key_file }, '
- f'--ca-signing-key-file:{ self.flags.ca_signing_key_file }',
+ f'--ca-cert-file:{ self.flags.ca_cert_file}, '
+ f'--ca-key-file:{ self.flags.ca_key_file}, '
+ f'--ca-signing-key-file:{ self.flags.ca_signing_key_file}',
  )
  cert_file_path = HttpProxyPlugin.generated_cert_file_path(
  self.flags.ca_cert_dir, text_(self.request.host),

proxy/http/websocket/frame.py

@@ -128,8 +128,8 @@ def build(self) -> bytes:
  )
  else:
  raise ValueError(
- f'Invalid payload_length { self.payload_length },'
- f'maximum allowed { 1 << 64 }',
+ f'Invalid payload_length { self.payload_length},'
+ f'maximum allowed { 1 << 64}',
  )
  if self.masked and self.data:
  mask = secrets.token_bytes(4) if self.mask is None else self.mask