Feature/impersonate 6.0

.github/workflows/test.yaml

@@ -12,7 +12,8 @@ jobs:
  runs-on: ${{ matrix.os }}
  strategy:
  matrix:
- os: [ubuntu-22.04, macos-11, windows-2019]
+ # os: [ubuntu-22.04, macos-11, windows-2019]
+ os: [ubuntu-22.04, macos-11]
  steps:
  - uses: actions/checkout@v3
  - uses: actions/setup-python@v4

Makefile

@@ -1,7 +1,7 @@
 .ONESHELL:
 SHELL := bash
-VERSION := 0.5.4
-CURL_VERSION := curl-7.84.0
+VERSION := 0.6.0
+CURL_VERSION := curl-8.1.1
 
 .preprocessed: curl_cffi/include/curl/curl.h curl_cffi/cacert.pem .so_downloaded
  touch .preprocessed
@@ -32,7 +32,7 @@ curl_cffi/cacert.pem:
  curl https://curl.se/ca/cacert.pem -o curl_cffi/cacert.pem
 
 .so_downloaded:
- python preprocess/download_so.py
+ python preprocess/download_so.py $(VERSION)
  touch .so_downloaded
 
 preprocess: .preprocessed

README-zh.md

@@ -15,6 +15,14 @@ TLS 或者 JA3 指纹。如果你莫名其妙地被某个网站封锁了，可
 - 支持 `asyncio`，并且每个请求都可以换代理。
 - 支持 http 2.0，requests 不支持。
 
+|库|requests|aiohttp|httpx|pycurl|curl_cffi|
+|---|---|---|---|---|---|
+|http2|❌|❌|✅|✅|✅|
+|sync|✅|❌|✅|✅|✅|
+|async|❌|✅|✅|❌|✅|
+|指纹|❌|❌|❌|❌|✅|
+|速度|🐇|🐇🐇|🐇|🐇🐇|🐇🐇|
+
 ## 安装
 
  pip install curl_cffi --upgrade
@@ -23,8 +31,14 @@ TLS 或者 JA3 指纹。如果你莫名其妙地被某个网站封锁了，可
 在其他小众平台，你可能需要先编译并安装 `curl-impersonate` 并且设置 `LD_LIBRARY_PATH` 这些
 环境变量。
 
+安装测试版:
+
+ pip install curl_cffi --pre
+
 ## 使用
 
+尽量模仿比较新的浏览器，不要直接从下边的例子里复制 `chrome110` 去用。
+
 ### 类 requests
 
 ```python
@@ -59,14 +73,21 @@ print(r.json())
 # {'cookies': {'foo': 'bar'}}
 ```
 
-支持模拟的浏览器版本，和 [curl-impersonate](https:/lwthiker/curl-impersonate) 一致：
+支持模拟的浏览器版本，和我 [fork](https:/yifeikong/curl-impersonate) 的 [curl-impersonate](https:/lwthiker/curl-impersonate) 一致：
+
+不过只支持类似 Chrome 的浏览器。
 
 - chrome99
 - chrome100
 - chrome101
 - chrome104
 - chrome107
 - chrome110
+- chrome116
+- chrome117
+- chrome118
+- chrome119
+- chrome120
 - chrome99_android
 - edge99
 - edge101
@@ -125,7 +146,10 @@ print(body.decode())
 
 更多细节请查看 [英文文档](https://curl-cffi.readthedocs.io)。
 
-如果你用 scrapy 的话，可以参考这个中间件：[tieyongjie/scrapy-fingerprint](https:/tieyongjie/scrapy-fingerprint)
+如果你用 scrapy 的话，可以参考这些中间件：
+
+- [tieyongjie/scrapy-fingerprint](https:/tieyongjie/scrapy-fingerprint)
+- [jxlil/scrapy-impersonate](https:/jxlil/scrapy-impersonate)
 
 有问题和建议请优先提 issue，中英文均可，也可以加微信群交流讨论：
 

README.md

@@ -40,6 +40,8 @@ To install beta releases:
 
 ## Usage
 
+Use the latest impersonate versions, do NOT copy `chrome110` here without changing.
+
 ### requests-like
 
 ```python
@@ -74,14 +76,21 @@ print(r.json())
 # {'cookies': {'foo': 'bar'}}
 ```
 
-Supported impersonate versions, as supported by [curl-impersonate](https:/lwthiker/curl-impersonate):
+Supported impersonate versions, as supported by my [fork](https:/yifeikong/curl-impersonate) of [curl-impersonate](https:/lwthiker/curl-impersonate):
+
+However, only Chrome-like browsers are supported.
 
 - chrome99
 - chrome100
 - chrome101
 - chrome104
 - chrome107
 - chrome110
+- chrome116
+- chrome117
+- chrome118
+- chrome119
+- chrome120
 - chrome99_android
 - edge99
 - edge101
@@ -140,7 +149,10 @@ print(body.decode())
 
 See the [docs](https://curl-cffi.readthedocs.io) for more details. 
 
-If you are using scrapy, check out this middleware: [tieyongjie/scrapy-fingerprint](https:/tieyongjie/scrapy-fingerprint)
+If you are using scrapy, check out these middlewares:
+
+- [tieyongjie/scrapy-fingerprint](https:/tieyongjie/scrapy-fingerprint)
+- [jxlil/scrapy-impersonate](https:/jxlil/scrapy-impersonate)
 
 ## Acknowledgement
 

curl_cffi/__version__.py

@@ -7,5 +7,5 @@
 # __description__ = metadata.metadata("curl_cffi")["Summary"]
 # __version__ = metadata.version("curl_cffi")
 __description__ = "libcurl ffi bindings for Python, with impersonation support"
-__version__ = "0.5.10"
+__version__ = "0.6.0"
 __curl_version__ = Curl().version().decode()

curl_cffi/const.py

@@ -527,3 +527,12 @@ class CurlHttpVersion(IntEnum):
  V2TLS = 4 # use version 2 for HTTPS, version 1.1 for HTTP */
  V2_PRIOR_KNOWLEDGE = 5 # please use HTTP 2 without HTTP/1.1 Upgrade */
  V3 = 30 # Makes use of explicit HTTP/3 without fallback.
+
+
+class CurlWsFlag(IntEnum):
+ TEXT = 1 << 0
+ BINARY = 1 << 1
+ CONT = 1 << 2
+ CLOSE = 1 << 3
+ PING = 1 << 4
+ OFFSET = 1 << 5

curl_cffi/curl.py

@@ -5,7 +5,7 @@
 from typing import Any, List, Tuple, Union
 
 from ._wrapper import ffi, lib # type: ignore
-from .const import CurlHttpVersion, CurlInfo, CurlOpt
+from .const import CurlHttpVersion, CurlInfo, CurlOpt, CurlWsFlag
 
 try:
  import certifi
@@ -107,6 +107,10 @@ def _set_error_buffer(self):
  self.setopt(CurlOpt.VERBOSE, 1)
  lib._curl_easy_setopt(self._curl, CurlOpt.DEBUGFUNCTION, lib.debug_function)
 
+ def debug(self):
+ self.setopt(CurlOpt.VERBOSE, 1)
+ lib._curl_easy_setopt(self._curl, CurlOpt.DEBUGFUNCTION, lib.debug_function)
+
  def __del__(self):
  self.close()
 
@@ -335,3 +339,25 @@ def close(self):
  self._curl = None
  ffi.release(self._error_buffer)
  self._resolve = ffi.NULL
+
+ def ws_recv(self, n: int = 1024):
+ buffer = ffi.new("char[]", n)
+ n_recv = ffi.new("int *")
+ p_frame = ffi.new("struct curl_ws_frame **")
+
+ ret = lib.curl_ws_recv(self._curl, buffer, n, n_recv, p_frame)
+ self._check_error(ret, "WS_RECV")
+ frame = p_frame[0]
+ # print(frame.offset, frame.bytesleft)
+
+ return ffi.buffer(buffer)[: n_recv[0]], frame
+
+ def ws_send(self, payload: bytes, flags: CurlWsFlag = CurlWsFlag.BINARY) -> int:
+ n_sent = ffi.new("int *")
+ buffer = ffi.from_buffer(payload)
+ ret = lib.curl_ws_send(self._curl, buffer, len(buffer), n_sent, 0, flags)
+ self._check_error(ret, "WS_SEND")
+ return n_sent
+
+ def ws_close(self):
+ self.ws_send(b"", CurlWsFlag.CLOSE)

curl_cffi/ffi/cdef.c

@@ -46,3 +46,14 @@ struct CURLMsg *curl_multi_info_read(void* curlm, int *msg_in_queue);
 extern "Python" void socket_function(void *curl, int sockfd, int what, void *clientp, void *socketp);
 extern "Python" void timer_function(void *curlm, int timeout_ms, void *clientp);
 
+// websocket
+struct curl_ws_frame {
+ int age; /* zero */
+ int flags; /* See the CURLWS_* defines */
+ long offset; /* the offset of this data into the frame */
+ long bytesleft; /* number of pending bytes left of the payload */
+ ...;
+};
+
+int curl_ws_recv(void *curl, void *buffer, int buflen, int *recv, struct curl_ws_frame **meta);
+int curl_ws_send(void *curl, void *buffer, int buflen, int *sent, int fragsize, unsigned int sendflags);

curl_cffi/requests/session.py

@@ -20,6 +20,7 @@
 from .errors import RequestsError
 from .headers import Headers, HeaderTypes
 from .models import Request, Response
+from .websockets import WebSocket
 
 try:
  import gevent
@@ -47,6 +48,11 @@ class BrowserType(str, Enum):
  chrome104 = "chrome104"
  chrome107 = "chrome107"
  chrome110 = "chrome110"
+ chrome116 = "chrome116"
+ chrome117 = "chrome117"
+ chrome118 = "chrome118"
+ chrome119 = "chrome119"
+ chrome120 = "chrome120"
  chrome99_android = "chrome99_android"
  safari15_3 = "safari15_3"
  safari15_5 = "safari15_5"
@@ -580,6 +586,13 @@ def stream(self, *args, **kwargs):
  finally:
  rsp.close()
 
+ def connect(self, url, *args, **kwargs):
+ self._set_curl_options(self.curl, "GET", url, *args, **kwargs)
+ # https://curl.se/docs/websocket.html
+ self.curl.setopt(CurlOpt.CONNECT_ONLY, 2)
+ self.curl.perform()
+ return WebSocket(self, self.curl)
+
  def request(
  self,
  method: str,
@@ -752,7 +765,7 @@ def __init__(
  ```
  """
  super().__init__(**kwargs)
- self.loop = loop
+ self._loop = loop
  self._acurl = async_curl
  self.max_clients = max_clients
  self._closed = False
@@ -763,10 +776,14 @@ def __init__(
  ):
  warnings.warn(WINDOWS_WARN)
 
+ @property
+ def loop(self):
+ if self._loop is None:
+ self._loop = asyncio.get_running_loop()
+ return self._loop
+
  @property
  def acurl(self):
- if self.loop is None:
- self.loop = asyncio.get_running_loop()
  if self._acurl is None:
  self._acurl = AsyncCurl(loop=self.loop)
  return self._acurl
@@ -827,6 +844,14 @@ async def stream(self, *args, **kwargs):
  finally:
  await rsp.aclose()
 
+ async def connect(self, url, *args, **kwargs):
+ curl = await self.pop_curl()
+ # curl.debug()
+ self._set_curl_options(curl, "GET", url, *args, **kwargs)
+ curl.setopt(CurlOpt.CONNECT_ONLY, 2) # https://curl.se/docs/websocket.html
+ await self.loop.run_in_executor(None, curl.perform)
+ return WebSocket(self, curl)
+
  async def request(
  self,
  method: str,

curl_cffi/requests/websockets.py

@@ -0,0 +1,54 @@
+import asyncio
+from curl_cffi.const import CurlECode, CurlWsFlag
+from curl_cffi.curl import CurlError
+
+
+class WebSocket:
+ def __init__(self, session, curl):
+ self.session = session
+ self.curl = curl
+ self._loop = None
+
+ def recv_fragment(self):
+ return self.curl.ws_recv()
+
+ def recv(self):
+ chunks = []
+ # TODO use select here
+ while True:
+ try:
+ chunk, frame = self.curl.ws_recv()
+ chunks.append(chunk)
+ if frame.bytesleft == 0:
+ break
+ except CurlError as e:
+ if e.code == CurlECode.AGAIN:
+ pass
+ else:
+ raise
+
+ return b"".join(chunks)
+
+ def send(self, payload: bytes, flags: CurlWsFlag = CurlWsFlag.BINARY):
+ return self.curl.ws_send(payload, flags)
+
+ def close(self):
+ # FIXME how to reset. or can a curl handle connect to two websockets?
+ self.curl.close()
+
+ @property
+ def loop(self):
+ if self._loop is None:
+ self._loop = asyncio.get_running_loop()
+ return self._loop
+
+ async def arecv(self):
+ return await self.loop.run_in_executor(None, self.recv)
+
+ async def asend(self, payload: bytes, flags: CurlWsFlag = CurlWsFlag.BINARY):
+ return await self.loop.run_in_executor(None, self.send, payload, flags)
+
+ async def aclose(self):
+ await self.loop.run_in_executor(None, self.close)
+ self.curl.reset()
+ self.session.push_curl(curl)

examples/websocket.py

@@ -0,0 +1,21 @@
+import asyncio
+from curl_cffi import requests
+
+with requests.Session() as s:
+ w = s.connect("ws://localhost:8765")
+ w.send(b"Foo")
+ reply = w.recv()
+ print(reply)
+ assert reply == b"Hello Foo!"
+
+
+async def async_examples():
+ async with requests.AsyncSession() as s:
+ w = await s.connect("ws://localhost:8765")
+ await w.asend(b"Bar")
+ reply = await w.arecv()
+ print(reply)
+ assert reply == b"Hello Bar!"
+
+
+asyncio.run(async_examples())

examples/websocket_server.py

@@ -0,0 +1,18 @@
+import asyncio
+import websockets
+
+async def hello(websocket):
+ name = (await websocket.recv()).decode()
+ print(f"<<< {name}")
+
+ greeting = f"Hello {name}!"
+
+ await websocket.send(greeting)
+ print(f">>> {greeting}")
+
+async def main():
+ async with websockets.serve(hello, "localhost", 8765):
+ await asyncio.Future() # run forever
+
+if __name__ == "__main__":
+ asyncio.run(main())