From 5ae2216b513cc4e2ba54d43c15ce8bf11105df5f Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Sun, 1 Oct 2023 22:21:55 +0800 Subject: [PATCH 01/19] Upgrade curl-impersonate version --- Makefile | 6 +++--- README-zh.md | 1 + README.md | 1 + curl_cffi/requests/session.py | 1 + preprocess/download_so.py | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 41a0154a..771cd60c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .ONESHELL: SHELL := bash -VERSION := 0.5.4 -CURL_VERSION := curl-7.84.0 +VERSION := 0.6.0-alpha.1 +CURL_VERSION := curl-8.1.1 .preprocessed: curl_cffi/include/curl/curl.h curl_cffi/cacert.pem .so_downloaded touch .preprocessed @@ -32,7 +32,7 @@ curl_cffi/cacert.pem: curl https://curl.se/ca/cacert.pem -o curl_cffi/cacert.pem .so_downloaded: - python preprocess/download_so.py + python preprocess/download_so.py $(VERSION) touch .so_downloaded preprocess: .preprocessed diff --git a/README-zh.md b/README-zh.md index 50246dce..34b6c417 100644 --- a/README-zh.md +++ b/README-zh.md @@ -67,6 +67,7 @@ print(r.json()) - chrome104 - chrome107 - chrome110 +- chrome116 - chrome99_android - edge99 - edge101 diff --git a/README.md b/README.md index 91f64904..ff8e9fd9 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ Supported impersonate versions, as supported by [curl-impersonate](https://githu - chrome104 - chrome107 - chrome110 +- chrome116 - chrome99_android - edge99 - edge101 diff --git a/curl_cffi/requests/session.py b/curl_cffi/requests/session.py index fa417df0..e16f8269 100644 --- a/curl_cffi/requests/session.py +++ b/curl_cffi/requests/session.py @@ -47,6 +47,7 @@ class BrowserType(str, Enum): chrome104 = "chrome104" chrome107 = "chrome107" chrome110 = "chrome110" + chrome116 = "chrome116" chrome99_android = "chrome99_android" safari15_3 = "safari15_3" safari15_5 = "safari15_5" diff --git a/preprocess/download_so.py b/preprocess/download_so.py index 5ba305fd..7c1256b6 100644 --- a/preprocess/download_so.py +++ b/preprocess/download_so.py @@ -6,7 +6,7 @@ uname = platform.uname() -VERSION = "0.5.4" +VERSION = sys.argv[1] if uname.system == "Windows": LIBDIR = "./lib" From 04b467154039eb2cf5912ff20ea2f6ba9bcd7cd4 Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Mon, 2 Oct 2023 16:57:05 +0800 Subject: [PATCH 02/19] Add websocket support --- .github/workflows/test.yaml | 3 +- curl_cffi/const.py | 9 ++++++ curl_cffi/curl.py | 24 ++++++++++++++- curl_cffi/ffi/cdef.c | 11 +++++++ curl_cffi/requests/session.py | 16 ++++++++++ curl_cffi/requests/websockets.py | 52 ++++++++++++++++++++++++++++++++ examples/websocket.py | 7 +++++ examples/websocket_server.py | 18 +++++++++++ preprocess/generate_consts.py | 9 ++++++ 9 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 curl_cffi/requests/websockets.py create mode 100644 examples/websocket.py create mode 100644 examples/websocket_server.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 395be6f1..86bc96ee 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,7 +12,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, macos-11, windows-2019] + # os: [ubuntu-22.04, macos-11, windows-2019] + os: [ubuntu-22.04, macos-11] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/curl_cffi/const.py b/curl_cffi/const.py index c29b76ad..edfb1353 100644 --- a/curl_cffi/const.py +++ b/curl_cffi/const.py @@ -527,3 +527,12 @@ class CurlHttpVersion(IntEnum): V2TLS = 4 # use version 2 for HTTPS, version 1.1 for HTTP */ V2_PRIOR_KNOWLEDGE = 5 # please use HTTP 2 without HTTP/1.1 Upgrade */ V3 = 30 # Makes use of explicit HTTP/3 without fallback. + + +class CurlWsFlag(IntEnum): + TEXT = 1 << 0 + BINARY = 1 << 1 + CONT = 1 << 2 + CLOSE = 1 << 3 + PING = 1 << 4 + OFFSET = 1 << 5 diff --git a/curl_cffi/curl.py b/curl_cffi/curl.py index d267d686..42eccb4d 100644 --- a/curl_cffi/curl.py +++ b/curl_cffi/curl.py @@ -5,7 +5,7 @@ from typing import Any, List, Tuple, Union from ._wrapper import ffi, lib # type: ignore -from .const import CurlHttpVersion, CurlInfo, CurlOpt +from .const import CurlHttpVersion, CurlInfo, CurlOpt, CurlWsFlag try: import certifi @@ -335,3 +335,25 @@ def close(self): self._curl = None ffi.release(self._error_buffer) self._resolve = ffi.NULL + + def ws_recv(self, n: int = 1024): + buffer = ffi.new("char[]", n) + n_recv = ffi.new("int *") + p_frame = ffi.new("struct curl_ws_frame **") + + ret = lib.curl_ws_recv(self._curl, buffer, n, n_recv, p_frame) + self._check_error(ret, "WS_RECV") + frame = p_frame[0] + # print(frame.offset, frame.bytesleft) + + return ffi.buffer(buffer)[: n_recv[0]], frame + + def ws_send(self, payload: bytes, flags: CurlWsFlag = CurlWsFlag.BINARY) -> int: + n_sent = ffi.new("int *") + buffer = ffi.from_buffer(payload) + ret = lib.curl_ws_send(self._curl, buffer, len(buffer), n_sent, 0, flags) + self._check_error(ret, "WS_SEND") + return n_sent + + def ws_close(self): + self.ws_send(b"", CurlWsFlag.CLOSE) diff --git a/curl_cffi/ffi/cdef.c b/curl_cffi/ffi/cdef.c index 25b47289..b5dc98ba 100644 --- a/curl_cffi/ffi/cdef.c +++ b/curl_cffi/ffi/cdef.c @@ -46,3 +46,14 @@ struct CURLMsg *curl_multi_info_read(void* curlm, int *msg_in_queue); extern "Python" void socket_function(void *curl, int sockfd, int what, void *clientp, void *socketp); extern "Python" void timer_function(void *curlm, int timeout_ms, void *clientp); +// websocket +struct curl_ws_frame { + int age; /* zero */ + int flags; /* See the CURLWS_* defines */ + long offset; /* the offset of this data into the frame */ + long bytesleft; /* number of pending bytes left of the payload */ + ...; +}; + +int curl_ws_recv(void *curl, void *buffer, int buflen, int *recv, struct curl_ws_frame **meta); +int curl_ws_send(void *curl, void *buffer, int buflen, int *sent, int fragsize, unsigned int sendflags); diff --git a/curl_cffi/requests/session.py b/curl_cffi/requests/session.py index e16f8269..cff35bb9 100644 --- a/curl_cffi/requests/session.py +++ b/curl_cffi/requests/session.py @@ -20,6 +20,7 @@ from .errors import RequestsError from .headers import Headers, HeaderTypes from .models import Request, Response +from .websockets import WebSocket, AsyncWebSocket try: import gevent @@ -581,6 +582,13 @@ def stream(self, *args, **kwargs): finally: rsp.close() + def connect(self, url, *args, **kwargs): + self._set_curl_options(self.curl, "GET", url, *args, **kwargs) + # https://curl.se/docs/websocket.html + self.curl.setopt(CurlOpt.CONNECT_ONLY, 2) + self.curl.perform() + return WebSocket(self) + def request( self, method: str, @@ -828,6 +836,14 @@ async def stream(self, *args, **kwargs): finally: await rsp.aclose() + async def connect(self, *args, **kwargs): + curl = await self.pop_curl() + self._set_curl_options(*args, **kwargs) + curl.setopt(CurlOpt.CONNECT_ONLY, 2) # https://curl.se/docs/websocket.html + task = self.acurl.add_handle(curl) + await task + return AsyncWebSocket(self, curl) + async def request( self, method: str, diff --git a/curl_cffi/requests/websockets.py b/curl_cffi/requests/websockets.py new file mode 100644 index 00000000..0113f23b --- /dev/null +++ b/curl_cffi/requests/websockets.py @@ -0,0 +1,52 @@ +from curl_cffi.const import CurlECode, CurlWsFlag +from curl_cffi.curl import CurlError + + +class WebSocket: + def __init__(self, session): + self.session = session + + @property + def c(self): + return self.session.curl + + def recv_fragment(self): + return self.c.ws_recv() + + def recv(self): + chunks = [] + # TODO use select here + while True: + try: + chunk, frame = self.c.ws_recv() + chunks.append(chunk) + if frame.bytesleft == 0: + break + except CurlError as e: + if e.code == CurlECode.AGAIN: + pass + else: + raise + + return b"".join(chunks) + + def send(self, payload: bytes, flags: CurlWsFlag = CurlWsFlag.BINARY): + return self.c.ws_send(payload, flags) + + def close(self): + return self.c.close() + + +class AsyncWebSocket: + def __init__(self, session, curl): + self.session = session + self.curl = curl + + async def recv(self): + return await self.curl.ws_recv() + + async def send(self): + return await self.curl.ws_send() + + async def close(self): + return await self.curl.close() diff --git a/examples/websocket.py b/examples/websocket.py new file mode 100644 index 00000000..28350373 --- /dev/null +++ b/examples/websocket.py @@ -0,0 +1,7 @@ +from curl_cffi import requests + +with requests.Session() as s: + w = s.connect("ws://localhost:8765") + w.send(b"Foo") + reply = w.recv() + assert reply == b"Hello Foo!" diff --git a/examples/websocket_server.py b/examples/websocket_server.py new file mode 100644 index 00000000..accd928e --- /dev/null +++ b/examples/websocket_server.py @@ -0,0 +1,18 @@ +import asyncio +import websockets + +async def hello(websocket): + name = (await websocket.recv()).decode() + print(f"<<< {name}") + + greeting = f"Hello {name}!" + + await websocket.send(greeting) + print(f">>> {greeting}") + +async def main(): + async with websockets.serve(hello, "localhost", 8765): + await asyncio.Future() # run forever + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/preprocess/generate_consts.py b/preprocess/generate_consts.py index 33813b0f..17bd94ca 100644 --- a/preprocess/generate_consts.py +++ b/preprocess/generate_consts.py @@ -68,3 +68,12 @@ f.write(" V2TLS = 4 # use version 2 for HTTPS, version 1.1 for HTTP */\n") f.write(" V2_PRIOR_KNOWLEDGE = 5 # please use HTTP 2 without HTTP/1.1 Upgrade */\n") f.write(" V3 = 30 # Makes use of explicit HTTP/3 without fallback.\n") + + + f.write("class CurlWsFlag(IntEnum):\n") + f.write(" TEXT = (1<<0)\n") + f.write(" BINARY = (1<<1)\n") + f.write(" CONT = (1<<2)\n") + f.write(" CLOSE = (1<<3)\n") + f.write(" PING = (1<<4)\n") + f.write(" OFFSET = (1<<5)\n") From b150a561051f4774677584964ee59c77863a2b12 Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Mon, 2 Oct 2023 18:09:41 +0800 Subject: [PATCH 03/19] Support websocket with asyncio --- curl_cffi/curl.py | 4 +++ curl_cffi/requests/session.py | 24 ++++++++++-------- curl_cffi/requests/websockets.py | 42 +++++++++++++++++--------------- examples/websocket.py | 14 +++++++++++ 4 files changed, 54 insertions(+), 30 deletions(-) diff --git a/curl_cffi/curl.py b/curl_cffi/curl.py index 42eccb4d..e08c2d18 100644 --- a/curl_cffi/curl.py +++ b/curl_cffi/curl.py @@ -107,6 +107,10 @@ def _set_error_buffer(self): self.setopt(CurlOpt.VERBOSE, 1) lib._curl_easy_setopt(self._curl, CurlOpt.DEBUGFUNCTION, lib.debug_function) + def debug(self): + self.setopt(CurlOpt.VERBOSE, 1) + lib._curl_easy_setopt(self._curl, CurlOpt.DEBUGFUNCTION, lib.debug_function) + def __del__(self): self.close() diff --git a/curl_cffi/requests/session.py b/curl_cffi/requests/session.py index cff35bb9..f5286b6c 100644 --- a/curl_cffi/requests/session.py +++ b/curl_cffi/requests/session.py @@ -20,7 +20,7 @@ from .errors import RequestsError from .headers import Headers, HeaderTypes from .models import Request, Response -from .websockets import WebSocket, AsyncWebSocket +from .websockets import WebSocket try: import gevent @@ -587,7 +587,7 @@ def connect(self, url, *args, **kwargs): # https://curl.se/docs/websocket.html self.curl.setopt(CurlOpt.CONNECT_ONLY, 2) self.curl.perform() - return WebSocket(self) + return WebSocket(self, self.curl) def request( self, @@ -761,7 +761,7 @@ def __init__( ``` """ super().__init__(**kwargs) - self.loop = loop + self._loop = loop self._acurl = async_curl self.max_clients = max_clients self._closed = False @@ -772,10 +772,14 @@ def __init__( ): warnings.warn(WINDOWS_WARN) + @property + def loop(self): + if self._loop is None: + self._loop = asyncio.get_running_loop() + return self._loop + @property def acurl(self): - if self.loop is None: - self.loop = asyncio.get_running_loop() if self._acurl is None: self._acurl = AsyncCurl(loop=self.loop) return self._acurl @@ -836,13 +840,13 @@ async def stream(self, *args, **kwargs): finally: await rsp.aclose() - async def connect(self, *args, **kwargs): + async def connect(self, url, *args, **kwargs): curl = await self.pop_curl() - self._set_curl_options(*args, **kwargs) + # curl.debug() + self._set_curl_options(curl, "GET", url, *args, **kwargs) curl.setopt(CurlOpt.CONNECT_ONLY, 2) # https://curl.se/docs/websocket.html - task = self.acurl.add_handle(curl) - await task - return AsyncWebSocket(self, curl) + await self.loop.run_in_executor(None, curl.perform) + return WebSocket(self, curl) async def request( self, diff --git a/curl_cffi/requests/websockets.py b/curl_cffi/requests/websockets.py index 0113f23b..76a548fb 100644 --- a/curl_cffi/requests/websockets.py +++ b/curl_cffi/requests/websockets.py @@ -1,24 +1,23 @@ +import asyncio from curl_cffi.const import CurlECode, CurlWsFlag from curl_cffi.curl import CurlError class WebSocket: - def __init__(self, session): + def __init__(self, session, curl): self.session = session - - @property - def c(self): - return self.session.curl + self.curl = curl + self._loop = None def recv_fragment(self): - return self.c.ws_recv() + return self.curl.ws_recv() def recv(self): chunks = [] # TODO use select here while True: try: - chunk, frame = self.c.ws_recv() + chunk, frame = self.curl.ws_recv() chunks.append(chunk) if frame.bytesleft == 0: break @@ -31,22 +30,25 @@ def recv(self): return b"".join(chunks) def send(self, payload: bytes, flags: CurlWsFlag = CurlWsFlag.BINARY): - return self.c.ws_send(payload, flags) + return self.curl.ws_send(payload, flags) def close(self): - return self.c.close() - + # FIXME how to reset. or can a curl handle connect to two websockets? + self.curl.close() -class AsyncWebSocket: - def __init__(self, session, curl): - self.session = session - self.curl = curl + @property + def loop(self): + if self._loop is None: + self._loop = asyncio.get_running_loop() + return self._loop - async def recv(self): - return await self.curl.ws_recv() + async def arecv(self): + return await self.loop.run_in_executor(None, self.recv) - async def send(self): - return await self.curl.ws_send() + async def asend(self, payload: bytes, flags: CurlWsFlag = CurlWsFlag.BINARY): + return await self.loop.run_in_executor(None, self.send, payload, flags) - async def close(self): - return await self.curl.close() + async def aclose(self): + await self.loop.run_in_executor(None, self.close) + self.curl.reset() + self.session.push_curl(curl) diff --git a/examples/websocket.py b/examples/websocket.py index 28350373..175df755 100644 --- a/examples/websocket.py +++ b/examples/websocket.py @@ -1,7 +1,21 @@ +import asyncio from curl_cffi import requests with requests.Session() as s: w = s.connect("ws://localhost:8765") w.send(b"Foo") reply = w.recv() + print(reply) assert reply == b"Hello Foo!" + + +async def async_examples(): + async with requests.AsyncSession() as s: + w = await s.connect("ws://localhost:8765") + await w.asend(b"Bar") + reply = await w.arecv() + print(reply) + assert reply == b"Hello Bar!" + + +asyncio.run(async_examples()) From efb322c394265ec416552d302786f899b0a19095 Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Sat, 18 Nov 2023 17:51:46 +0800 Subject: [PATCH 04/19] Update readme --- README-zh.md | 27 +++++++++++++++++++++++++-- README.md | 15 +++++++++++++-- curl_cffi/requests/session.py | 4 ++++ preprocess/download_so.py | 4 ++-- 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/README-zh.md b/README-zh.md index 34b6c417..bb0febe9 100644 --- a/README-zh.md +++ b/README-zh.md @@ -15,6 +15,14 @@ TLS 或者 JA3 指纹。如果你莫名其妙地被某个网站封锁了,可 - 支持 `asyncio`,并且每个请求都可以换代理。 - 支持 http 2.0,requests 不支持。 +|库|requests|aiohttp|httpx|pycurl|curl_cffi| +|---|---|---|---|---|---| +|http2|❌|❌|✅|✅|✅| +|sync|✅|❌|✅|✅|✅| +|async|❌|✅|✅|❌|✅| +|指纹|❌|❌|❌|❌|✅| +|速度|🐇|🐇🐇|🐇|🐇🐇|🐇🐇| + ## 安装 pip install curl_cffi --upgrade @@ -23,8 +31,14 @@ TLS 或者 JA3 指纹。如果你莫名其妙地被某个网站封锁了,可 在其他小众平台,你可能需要先编译并安装 `curl-impersonate` 并且设置 `LD_LIBRARY_PATH` 这些 环境变量。 +安装测试版: + + pip install curl_cffi --pre + ## 使用 +尽量模仿比较新的浏览器,不要直接从下边的例子里复制 `chrome110` 去用。 + ### 类 requests ```python @@ -59,7 +73,9 @@ print(r.json()) # {'cookies': {'foo': 'bar'}} ``` -支持模拟的浏览器版本,和 [curl-impersonate](https://github.com/lwthiker/curl-impersonate) 一致: +支持模拟的浏览器版本,和我 [fork](https://github.com/yifeikong/curl-impersonate) 的 [curl-impersonate](https://github.com/lwthiker/curl-impersonate) 一致: + +不过只支持类似 Chrome 的浏览器。 - chrome99 - chrome100 @@ -68,6 +84,10 @@ print(r.json()) - chrome107 - chrome110 - chrome116 +- chrome117 +- chrome118 +- chrome119 +- chrome120 - chrome99_android - edge99 - edge101 @@ -126,7 +146,10 @@ print(body.decode()) 更多细节请查看 [英文文档](https://curl-cffi.readthedocs.io)。 -如果你用 scrapy 的话,可以参考这个中间件:[tieyongjie/scrapy-fingerprint](https://github.com/tieyongjie/scrapy-fingerprint) +如果你用 scrapy 的话,可以参考这些中间件: + +- [tieyongjie/scrapy-fingerprint](https://github.com/tieyongjie/scrapy-fingerprint) +- [jxlil/scrapy-impersonate](https://github.com/jxlil/scrapy-impersonate) 有问题和建议请优先提 issue,中英文均可,也可以加微信群交流讨论: diff --git a/README.md b/README.md index ff8e9fd9..e9019eb1 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ To install beta releases: ## Usage +Use the latest impersonate versions, do NOT copy `chrome110` here without changing. + ### requests-like ```python @@ -74,7 +76,9 @@ print(r.json()) # {'cookies': {'foo': 'bar'}} ``` -Supported impersonate versions, as supported by [curl-impersonate](https://github.com/lwthiker/curl-impersonate): +Supported impersonate versions, as supported by my [fork](https://github.com/yifeikong/curl-impersonate) of [curl-impersonate](https://github.com/lwthiker/curl-impersonate): + +However, only Chrome-like browsers are supported. - chrome99 - chrome100 @@ -83,6 +87,10 @@ Supported impersonate versions, as supported by [curl-impersonate](https://githu - chrome107 - chrome110 - chrome116 +- chrome117 +- chrome118 +- chrome119 +- chrome120 - chrome99_android - edge99 - edge101 @@ -141,7 +149,10 @@ print(body.decode()) See the [docs](https://curl-cffi.readthedocs.io) for more details. -If you are using scrapy, check out this middleware: [tieyongjie/scrapy-fingerprint](https://github.com/tieyongjie/scrapy-fingerprint) +If you are using scrapy, check out these middlewares: + +- [tieyongjie/scrapy-fingerprint](https://github.com/tieyongjie/scrapy-fingerprint) +- [jxlil/scrapy-impersonate](https://github.com/jxlil/scrapy-impersonate) ## Acknowledgement diff --git a/curl_cffi/requests/session.py b/curl_cffi/requests/session.py index f5286b6c..15fc4dac 100644 --- a/curl_cffi/requests/session.py +++ b/curl_cffi/requests/session.py @@ -49,6 +49,10 @@ class BrowserType(str, Enum): chrome107 = "chrome107" chrome110 = "chrome110" chrome116 = "chrome116" + chrome117 = "chrome117" + chrome118 = "chrome118" + chrome119 = "chrome119" + chrome120 = "chrome120" chrome99_android = "chrome99_android" safari15_3 = "safari15_3" safari15_5 = "safari15_5" diff --git a/preprocess/download_so.py b/preprocess/download_so.py index 7c1256b6..c56f0dd4 100644 --- a/preprocess/download_so.py +++ b/preprocess/download_so.py @@ -34,13 +34,13 @@ def reporthook(blocknum, blocksize, totalsize): url = "" filename = "./curl-impersonate.tar.gz" else: - url = f"https://github.com/lwthiker/curl-impersonate/releases/download/v{VERSION}/libcurl-impersonate-v{VERSION}.{uname.machine}-macos.tar.gz" + url = f"https://github.com/yifeikong/curl-impersonate/releases/download/v{VERSION}/libcurl-impersonate-v{VERSION}.{uname.machine}-macos.tar.gz" filename = "./curl-impersonate.tar.gz" elif uname.system == "Windows": url = f"https://github.com/yifeikong/curl-impersonate-win/releases/download/v{VERSION}/curl-impersonate-chrome.tar.gz" filename = "./curl-impersonate.tar.gz" else: - url = f"https://github.com/lwthiker/curl-impersonate/releases/download/v{VERSION}/libcurl-impersonate-v{VERSION}.{uname.machine}-linux-gnu.tar.gz" + url = f"https://github.com/yifeikong/curl-impersonate/releases/download/v{VERSION}/libcurl-impersonate-v{VERSION}.{uname.machine}-linux-gnu.tar.gz" filename = "./curl-impersonate.tar.gz" if url: From d4242602ae3f09896db0b81ad7ba8b24764abd76 Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Sat, 2 Dec 2023 11:51:27 +0800 Subject: [PATCH 05/19] Update version --- Makefile | 2 +- curl_cffi/__version__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 771cd60c..cf8e2c94 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .ONESHELL: SHELL := bash -VERSION := 0.6.0-alpha.1 +VERSION := 0.6.0 CURL_VERSION := curl-8.1.1 .preprocessed: curl_cffi/include/curl/curl.h curl_cffi/cacert.pem .so_downloaded diff --git a/curl_cffi/__version__.py b/curl_cffi/__version__.py index b0cfd8d3..246cdb22 100644 --- a/curl_cffi/__version__.py +++ b/curl_cffi/__version__.py @@ -7,5 +7,5 @@ # __description__ = metadata.metadata("curl_cffi")["Summary"] # __version__ = metadata.version("curl_cffi") __description__ = "libcurl ffi bindings for Python, with impersonation support" -__version__ = "0.5.10" +__version__ = "0.6.0" __curl_version__ = Curl().version().decode() diff --git a/pyproject.toml b/pyproject.toml index 9d886faa..9e7e9654 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "curl_cffi" -version = "0.5.10" +version = "0.6.0" authors = [{ name = "Yifei Kong", email = "kong@yifei.me" }] description = "libcurl ffi bindings for Python, with impersonation support" license = { file = "LICENSE" } From e1d486d1df1ad9007bfd9d5b69f9133c2a5706ba Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Mon, 25 Dec 2023 22:23:28 +0800 Subject: [PATCH 06/19] Initial working version for 0.6 --- Makefile | 4 ++-- README-zh.md | 3 --- README.md | 3 --- curl_cffi/__version__.py | 2 +- curl_cffi/requests/session.py | 14 +++++++----- pyproject.toml | 4 +++- tests/unittest/conftest.py | 37 +++++++++++++++++++++++++++++++ tests/unittest/test_websockets.py | 6 +++++ 8 files changed, 58 insertions(+), 15 deletions(-) create mode 100644 tests/unittest/test_websockets.py diff --git a/Makefile b/Makefile index cf8e2c94..cc5a44bd 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .ONESHELL: SHELL := bash -VERSION := 0.6.0 +VERSION := 0.6.0b1 CURL_VERSION := curl-8.1.1 .preprocessed: curl_cffi/include/curl/curl.h curl_cffi/cacert.pem .so_downloaded @@ -15,7 +15,7 @@ $(CURL_VERSION): tar -xf $(CURL_VERSION).tar.xz curl-impersonate-$(VERSION)/chrome/patches: $(CURL_VERSION) - curl -L "https://github.com/lwthiker/curl-impersonate/archive/refs/tags/v$(VERSION).tar.gz" \ + curl -L "https://github.com/yifeikong/curl-impersonate/archive/refs/tags/v$(VERSION).tar.gz" \ -o "curl-impersonate-$(VERSION).tar.gz" tar -xf curl-impersonate-$(VERSION).tar.gz diff --git a/README-zh.md b/README-zh.md index bb0febe9..3e70a16f 100644 --- a/README-zh.md +++ b/README-zh.md @@ -84,10 +84,7 @@ print(r.json()) - chrome107 - chrome110 - chrome116 -- chrome117 -- chrome118 - chrome119 -- chrome120 - chrome99_android - edge99 - edge101 diff --git a/README.md b/README.md index e9019eb1..71e88ac4 100644 --- a/README.md +++ b/README.md @@ -87,10 +87,7 @@ However, only Chrome-like browsers are supported. - chrome107 - chrome110 - chrome116 -- chrome117 -- chrome118 - chrome119 -- chrome120 - chrome99_android - edge99 - edge101 diff --git a/curl_cffi/__version__.py b/curl_cffi/__version__.py index 246cdb22..9acc7803 100644 --- a/curl_cffi/__version__.py +++ b/curl_cffi/__version__.py @@ -7,5 +7,5 @@ # __description__ = metadata.metadata("curl_cffi")["Summary"] # __version__ = metadata.version("curl_cffi") __description__ = "libcurl ffi bindings for Python, with impersonation support" -__version__ = "0.6.0" +__version__ = "0.6.0b1" __curl_version__ = Curl().version().decode() diff --git a/curl_cffi/requests/session.py b/curl_cffi/requests/session.py index 15fc4dac..1b985b7c 100644 --- a/curl_cffi/requests/session.py +++ b/curl_cffi/requests/session.py @@ -49,19 +49,23 @@ class BrowserType(str, Enum): chrome107 = "chrome107" chrome110 = "chrome110" chrome116 = "chrome116" - chrome117 = "chrome117" - chrome118 = "chrome118" chrome119 = "chrome119" - chrome120 = "chrome120" chrome99_android = "chrome99_android" safari15_3 = "safari15_3" safari15_5 = "safari15_5" + chrome = "chrome119" + @classmethod def has(cls, item): return item in cls.__members__ +class BrowserSpec: + """A more structured way of selecting browsers """ + # TODO + + def _update_url_params(url: str, params: Dict) -> str: """Add GET params to provided URL being aware of existing. @@ -586,7 +590,7 @@ def stream(self, *args, **kwargs): finally: rsp.close() - def connect(self, url, *args, **kwargs): + def ws_connect(self, url, *args, **kwargs): self._set_curl_options(self.curl, "GET", url, *args, **kwargs) # https://curl.se/docs/websocket.html self.curl.setopt(CurlOpt.CONNECT_ONLY, 2) @@ -844,7 +848,7 @@ async def stream(self, *args, **kwargs): finally: await rsp.aclose() - async def connect(self, url, *args, **kwargs): + async def ws_connect(self, url, *args, **kwargs): curl = await self.pop_curl() # curl.debug() self._set_curl_options(curl, "GET", url, *args, **kwargs) diff --git a/pyproject.toml b/pyproject.toml index 9e7e9654..cfef96b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "curl_cffi" -version = "0.6.0" +version = "0.6.0b1" authors = [{ name = "Yifei Kong", email = "kong@yifei.me" }] description = "libcurl ffi bindings for Python, with impersonation support" license = { file = "LICENSE" } @@ -42,6 +42,7 @@ dev = [ "trio-typing==0.7.0", "trustme==0.9.0", "uvicorn==0.18.3", + "websockets==11.0.3", ] build = [ "cibuildwheel", @@ -58,6 +59,7 @@ test = [ "trio-typing==0.7.0", "trustme==0.9.0", "uvicorn==0.18.3", + "websockets==11.0.3", ] diff --git a/tests/unittest/conftest.py b/tests/unittest/conftest.py index a94f10f8..414cda8d 100644 --- a/tests/unittest/conftest.py +++ b/tests/unittest/conftest.py @@ -4,6 +4,7 @@ import threading import time import typing +import websockets from asyncio import sleep from collections import defaultdict from urllib.parse import parse_qs @@ -558,6 +559,29 @@ async def watch_restarts(self): # pragma: nocover await self.startup() +async def hello(websocket): + name = (await websocket.recv()).decode() + print(f"<<< {name}") + + greeting = f"Hello {name}!" + + await websocket.send(greeting) + print(f">>> {greeting}") + + +class TestWebsocketServer: + def __init__(self, port): + self.url = f"ws://localhost:{port}" + self.port = port + + def run(self): + async def serve(port): + async with websockets.serve(hello, "localhost", port): + await asyncio.Future() # run forever + + asyncio.run(serve(self.port)) + + def serve_in_thread(server: Server): thread = threading.Thread(target=server.run) thread.start() @@ -570,6 +594,19 @@ def serve_in_thread(server: Server): thread.join() +@pytest.fixture(scope="session") +def ws_server(): + server = TestWebsocketServer(port=8964) + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + try: + time.sleep(2) # FIXME find a reliable way to check the server is up + yield server + finally: + pass + # thread.join() + + @pytest.fixture(scope="session") def server(): config = Config(app=app, lifespan="off", loop="asyncio") diff --git a/tests/unittest/test_websockets.py b/tests/unittest/test_websockets.py new file mode 100644 index 00000000..903d31cc --- /dev/null +++ b/tests/unittest/test_websockets.py @@ -0,0 +1,6 @@ +from curl_cffi.requests import Session + + +def test_websocket(ws_server): + with Session() as s: + s.ws_connect(ws_server.url) From 530455d6ac1acc44cbbd142959c2bb9c482faf3b Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Mon, 25 Dec 2023 22:37:03 +0800 Subject: [PATCH 07/19] Update manylinux images --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index cfef96b8..ffbfe355 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ test-requires = "pytest" test-command = "pytest {project}/tests/unittest" test-extras = ["test"] test-skip = "pp*" +manylinux-x86_64-image = "manylinux_2_28" [tool.cibuildwheel.macos] before-all = "gmake preprocess" From c7f360c61279701067c48c09cf790e09e03b3427 Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Tue, 26 Dec 2023 00:01:50 +0800 Subject: [PATCH 08/19] Add websockets test --- tests/unittest/conftest.py | 12 +++++------- tests/unittest/test_websockets.py | 8 ++++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/unittest/conftest.py b/tests/unittest/conftest.py index 414cda8d..21b1a81d 100644 --- a/tests/unittest/conftest.py +++ b/tests/unittest/conftest.py @@ -559,14 +559,12 @@ async def watch_restarts(self): # pragma: nocover await self.startup() -async def hello(websocket): +async def echo(websocket): name = (await websocket.recv()).decode() - print(f"<<< {name}") + # print(f"<<< {name}") - greeting = f"Hello {name}!" - - await websocket.send(greeting) - print(f">>> {greeting}") + await websocket.send(name) + # print(f">>> {name}") class TestWebsocketServer: @@ -576,7 +574,7 @@ def __init__(self, port): def run(self): async def serve(port): - async with websockets.serve(hello, "localhost", port): + async with websockets.serve(echo, "localhost", port): await asyncio.Future() # run forever asyncio.run(serve(self.port)) diff --git a/tests/unittest/test_websockets.py b/tests/unittest/test_websockets.py index 903d31cc..5a8f862e 100644 --- a/tests/unittest/test_websockets.py +++ b/tests/unittest/test_websockets.py @@ -4,3 +4,11 @@ def test_websocket(ws_server): with Session() as s: s.ws_connect(ws_server.url) + + +def test_hello(ws_server): + with Session() as s: + ws = s.ws_connect(ws_server.url) + ws.send(b"foo") + content = ws.recv() + assert content == "foo" From e74586266b6cbd3a6aac1464b55caad70dc2c1bf Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Tue, 26 Dec 2023 00:03:19 +0800 Subject: [PATCH 09/19] Bump version and revert manylinux image --- Makefile | 2 +- curl_cffi/__version__.py | 2 +- pyproject.toml | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index cc5a44bd..fc727b60 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .ONESHELL: SHELL := bash -VERSION := 0.6.0b1 +VERSION := 0.6.0b2 CURL_VERSION := curl-8.1.1 .preprocessed: curl_cffi/include/curl/curl.h curl_cffi/cacert.pem .so_downloaded diff --git a/curl_cffi/__version__.py b/curl_cffi/__version__.py index 9acc7803..01ade491 100644 --- a/curl_cffi/__version__.py +++ b/curl_cffi/__version__.py @@ -7,5 +7,5 @@ # __description__ = metadata.metadata("curl_cffi")["Summary"] # __version__ = metadata.version("curl_cffi") __description__ = "libcurl ffi bindings for Python, with impersonation support" -__version__ = "0.6.0b1" +__version__ = "0.6.0b2" __curl_version__ = Curl().version().decode() diff --git a/pyproject.toml b/pyproject.toml index ffbfe355..2ca0d01d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "curl_cffi" -version = "0.6.0b1" +version = "0.6.0b2" authors = [{ name = "Yifei Kong", email = "kong@yifei.me" }] description = "libcurl ffi bindings for Python, with impersonation support" license = { file = "LICENSE" } @@ -85,7 +85,6 @@ test-requires = "pytest" test-command = "pytest {project}/tests/unittest" test-extras = ["test"] test-skip = "pp*" -manylinux-x86_64-image = "manylinux_2_28" [tool.cibuildwheel.macos] before-all = "gmake preprocess" From 5069ea72c8e57a2f3d2e066863dacf00385a55f5 Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Tue, 26 Dec 2023 00:11:45 +0800 Subject: [PATCH 10/19] Change localhost to 127.0.0.1 in tests --- tests/unittest/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittest/conftest.py b/tests/unittest/conftest.py index 21b1a81d..f8eff760 100644 --- a/tests/unittest/conftest.py +++ b/tests/unittest/conftest.py @@ -569,12 +569,12 @@ async def echo(websocket): class TestWebsocketServer: def __init__(self, port): - self.url = f"ws://localhost:{port}" + self.url = f"ws://127.0.0.1:{port}" self.port = port def run(self): async def serve(port): - async with websockets.serve(echo, "localhost", port): + async with websockets.serve(echo, "127.0.0.1", port): await asyncio.Future() # run forever asyncio.run(serve(self.port)) From 63b277be438dde6225422a946046f88b3fd1e85a Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Tue, 26 Dec 2023 00:14:28 +0800 Subject: [PATCH 11/19] Fix websocket tests --- tests/unittest/test_websockets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest/test_websockets.py b/tests/unittest/test_websockets.py index 5a8f862e..736d21d5 100644 --- a/tests/unittest/test_websockets.py +++ b/tests/unittest/test_websockets.py @@ -11,4 +11,4 @@ def test_hello(ws_server): ws = s.ws_connect(ws_server.url) ws.send(b"foo") content = ws.recv() - assert content == "foo" + assert content == b"foo" From fd3bfa7bf3c8f4f277cfe6b753e5084aaabf9c8d Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Tue, 26 Dec 2023 00:26:31 +0800 Subject: [PATCH 12/19] Disable windows temporially --- .github/workflows/build.yaml | 3 ++- curl_cffi/requests/models.py | 3 +++ tests/unittest/conftest.py | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3dcd9256..eebb6976 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -18,7 +18,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, macos-11, windows-2019] + # os: [ubuntu-22.04, macos-11, windows-2019] + os: [ubuntu-22.04, macos-11] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/curl_cffi/requests/models.py b/curl_cffi/requests/models.py index f2d50f2e..741d4d21 100644 --- a/curl_cffi/requests/models.py +++ b/curl_cffi/requests/models.py @@ -190,4 +190,7 @@ async def acontent(self) -> bytes: return b"".join(chunks) async def aclose(self): + import time + print(time.time()) await self.stream_task # type: ignore + print(time.time()) diff --git a/tests/unittest/conftest.py b/tests/unittest/conftest.py index f8eff760..3005e3e5 100644 --- a/tests/unittest/conftest.py +++ b/tests/unittest/conftest.py @@ -574,6 +574,7 @@ def __init__(self, port): def run(self): async def serve(port): + # GitHub actions only likes 127, not localhost, wtf... async with websockets.serve(echo, "127.0.0.1", port): await asyncio.Future() # run forever From b7203ad60fd0825d0ccf267d75f3df87f94f22ec Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Tue, 26 Dec 2023 21:04:50 +0800 Subject: [PATCH 13/19] Adding chrome120 and safari17_2_ios --- curl_cffi/requests/models.py | 3 --- curl_cffi/requests/session.py | 4 +++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/curl_cffi/requests/models.py b/curl_cffi/requests/models.py index 741d4d21..f2d50f2e 100644 --- a/curl_cffi/requests/models.py +++ b/curl_cffi/requests/models.py @@ -190,7 +190,4 @@ async def acontent(self) -> bytes: return b"".join(chunks) async def aclose(self): - import time - print(time.time()) await self.stream_task # type: ignore - print(time.time()) diff --git a/curl_cffi/requests/session.py b/curl_cffi/requests/session.py index 1b985b7c..0a910931 100644 --- a/curl_cffi/requests/session.py +++ b/curl_cffi/requests/session.py @@ -50,11 +50,13 @@ class BrowserType(str, Enum): chrome110 = "chrome110" chrome116 = "chrome116" chrome119 = "chrome119" + chrome120 = "chrome120" chrome99_android = "chrome99_android" safari15_3 = "safari15_3" safari15_5 = "safari15_5" + safari17_2_ios = "safari17_2_ios" - chrome = "chrome119" + chrome = "chrome120" @classmethod def has(cls, item): From 6799e09560d5e15b4f485f9701df2edc09135d10 Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Wed, 27 Dec 2023 09:50:14 +0800 Subject: [PATCH 14/19] Bump version to 0.6.0b4, add support for chrome120 and safari17.2 on ios --- Makefile | 2 +- bump_version.sh | 12 ++++++++ curl_cffi/__version__.py | 2 +- curl_cffi/const.py | 49 ++++++++++++++++---------------- curl_cffi/requests/exceptions.py | 3 ++ preprocess/generate_consts.py | 1 + pyproject.toml | 2 +- 7 files changed, 43 insertions(+), 28 deletions(-) create mode 100755 bump_version.sh create mode 100644 curl_cffi/requests/exceptions.py diff --git a/Makefile b/Makefile index fc727b60..5aab1267 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .ONESHELL: SHELL := bash -VERSION := 0.6.0b2 +VERSION := 0.6.0b4 CURL_VERSION := curl-8.1.1 .preprocessed: curl_cffi/include/curl/curl.h curl_cffi/cacert.pem .so_downloaded diff --git a/bump_version.sh b/bump_version.sh new file mode 100755 index 00000000..9591d5e2 --- /dev/null +++ b/bump_version.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +VERSION=$1 + +# Makefile +gsed "s/^VERSION := .*/VERSION := ${VERSION}/g" -i Makefile + +# curl_cffi/__version__.py +gsed "s/^__version__ = .*/__version__ = \"${VERSION}\"/g" -i curl_cffi/__version__.py + +# pyproject.toml +gsed "s/^version = .*/version = \"${VERSION}\"/g" -i pyproject.toml diff --git a/curl_cffi/__version__.py b/curl_cffi/__version__.py index 01ade491..5b27dc03 100644 --- a/curl_cffi/__version__.py +++ b/curl_cffi/__version__.py @@ -7,5 +7,5 @@ # __description__ = metadata.metadata("curl_cffi")["Summary"] # __version__ = metadata.version("curl_cffi") __description__ = "libcurl ffi bindings for Python, with impersonation support" -__version__ = "0.6.0b2" +__version__ = "0.6.0b4" __curl_version__ = Curl().version().decode() diff --git a/curl_cffi/const.py b/curl_cffi/const.py index edfb1353..f81068e3 100644 --- a/curl_cffi/const.py +++ b/curl_cffi/const.py @@ -106,7 +106,7 @@ class CurlOpt(IntEnum): SSL_CTX_DATA = 10000 + 109 FTP_CREATE_MISSING_DIRS = 0 + 110 PROXYAUTH = 0 + 111 - FTP_RESPONSE_TIMEOUT = 0 + 112 + SERVER_RESPONSE_TIMEOUT = 0 + 112 IPRESOLVE = 0 + 113 MAXFILESIZE = 0 + 114 INFILESIZE_LARGE = 30000 + 115 @@ -303,14 +303,21 @@ class CurlOpt(IntEnum): MIME_OPTIONS = 0 + 315 SSH_HOSTKEYFUNCTION = 20000 + 316 SSH_HOSTKEYDATA = 10000 + 317 - HTTPBASEHEADER = 10000 + 318 - SSL_SIG_HASH_ALGS = 10000 + 319 - SSL_ENABLE_ALPS = 0 + 320 - SSL_CERT_COMPRESSION = 10000 + 321 - SSL_ENABLE_TICKET = 0 + 322 - HTTP2_PSEUDO_HEADERS_ORDER = 10000 + 323 - HTTP2_NO_SERVER_PUSH = 0 + 324 - SSL_PERMUTE_EXTENSIONS = 0 + 325 + PROTOCOLS_STR = 10000 + 318 + REDIR_PROTOCOLS_STR = 10000 + 319 + WS_OPTIONS = 0 + 320 + CA_CACHE_TIMEOUT = 0 + 321 + QUICK_EXIT = 0 + 322 + HTTPBASEHEADER = 10000 + 323 + SSL_SIG_HASH_ALGS = 10000 + 324 + SSL_ENABLE_ALPS = 0 + 325 + SSL_CERT_COMPRESSION = 10000 + 326 + SSL_ENABLE_TICKET = 0 + 327 + HTTP2_PSEUDO_HEADERS_ORDER = 10000 + 328 + HTTP2_SETTINGS = 10000 + 329 + SSL_PERMUTE_EXTENSIONS = 0 + 330 + HTTP2_WINDOW_UPDATE = 0 + 331 + ECH = 10000 + 332 if locals().get("WRITEDATA"): FILE = locals().get("WRITEDATA") @@ -328,22 +335,16 @@ class CurlInfo(IntEnum): NAMELOOKUP_TIME = 0x300000 + 4 CONNECT_TIME = 0x300000 + 5 PRETRANSFER_TIME = 0x300000 + 6 - SIZE_UPLOAD = 0x300000 + 7 SIZE_UPLOAD_T = 0x600000 + 7 - SIZE_DOWNLOAD = 0x300000 + 8 SIZE_DOWNLOAD_T = 0x600000 + 8 - SPEED_DOWNLOAD = 0x300000 + 9 SPEED_DOWNLOAD_T = 0x600000 + 9 - SPEED_UPLOAD = 0x300000 + 10 SPEED_UPLOAD_T = 0x600000 + 10 HEADER_SIZE = 0x200000 + 11 REQUEST_SIZE = 0x200000 + 12 SSL_VERIFYRESULT = 0x200000 + 13 FILETIME = 0x200000 + 14 FILETIME_T = 0x600000 + 14 - CONTENT_LENGTH_DOWNLOAD = 0x300000 + 15 CONTENT_LENGTH_DOWNLOAD_T = 0x600000 + 15 - CONTENT_LENGTH_UPLOAD = 0x300000 + 16 CONTENT_LENGTH_UPLOAD_T = 0x600000 + 16 STARTTRANSFER_TIME = 0x300000 + 17 CONTENT_TYPE = 0x100000 + 18 @@ -357,7 +358,6 @@ class CurlInfo(IntEnum): NUM_CONNECTS = 0x200000 + 26 SSL_ENGINES = 0x400000 + 27 COOKIELIST = 0x400000 + 28 - LASTSOCKET = 0x200000 + 29 FTP_ENTRY_PATH = 0x100000 + 30 REDIRECT_URL = 0x100000 + 31 PRIMARY_IP = 0x100000 + 32 @@ -371,12 +371,10 @@ class CurlInfo(IntEnum): PRIMARY_PORT = 0x200000 + 40 LOCAL_IP = 0x100000 + 41 LOCAL_PORT = 0x200000 + 42 - TLS_SESSION = 0x400000 + 43 ACTIVESOCKET = 0x500000 + 44 TLS_SSL_PTR = 0x400000 + 45 HTTP_VERSION = 0x200000 + 46 PROXY_SSL_VERIFYRESULT = 0x200000 + 47 - PROTOCOL = 0x200000 + 48 SCHEME = 0x100000 + 49 TOTAL_TIME_T = 0x600000 + 50 NAMELOOKUP_TIME_T = 0x600000 + 51 @@ -492,7 +490,7 @@ class CurlECode(IntEnum): TFTP_UNKNOWNID = 72 REMOTE_FILE_EXISTS = 73 TFTP_NOSUCHUSER = 74 - CONV_FAILED = 75 + OBSOLETE75 = 75 OBSOLETE76 = 76 SSL_CACERT_BADFILE = 77 REMOTE_FILE_NOT_FOUND = 78 @@ -517,6 +515,7 @@ class CurlECode(IntEnum): PROXY = 97 SSL_CLIENTCERT = 98 UNRECOVERABLE_POLL = 99 + ECH_REQUIRED = 100 class CurlHttpVersion(IntEnum): @@ -530,9 +529,9 @@ class CurlHttpVersion(IntEnum): class CurlWsFlag(IntEnum): - TEXT = 1 << 0 - BINARY = 1 << 1 - CONT = 1 << 2 - CLOSE = 1 << 3 - PING = 1 << 4 - OFFSET = 1 << 5 + TEXT = (1<<0) + BINARY = (1<<1) + CONT = (1<<2) + CLOSE = (1<<3) + PING = (1<<4) + OFFSET = (1<<5) diff --git a/curl_cffi/requests/exceptions.py b/curl_cffi/requests/exceptions.py new file mode 100644 index 00000000..efbda06f --- /dev/null +++ b/curl_cffi/requests/exceptions.py @@ -0,0 +1,3 @@ +from .errors import RequestsError + +RequestsException = RequestsError diff --git a/preprocess/generate_consts.py b/preprocess/generate_consts.py index 17bd94ca..c456754a 100644 --- a/preprocess/generate_consts.py +++ b/preprocess/generate_consts.py @@ -68,6 +68,7 @@ f.write(" V2TLS = 4 # use version 2 for HTTPS, version 1.1 for HTTP */\n") f.write(" V2_PRIOR_KNOWLEDGE = 5 # please use HTTP 2 without HTTP/1.1 Upgrade */\n") f.write(" V3 = 30 # Makes use of explicit HTTP/3 without fallback.\n") + f.write("\n\n") f.write("class CurlWsFlag(IntEnum):\n") diff --git a/pyproject.toml b/pyproject.toml index 2ca0d01d..19d0af9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "curl_cffi" -version = "0.6.0b2" +version = "0.6.0b4" authors = [{ name = "Yifei Kong", email = "kong@yifei.me" }] description = "libcurl ffi bindings for Python, with impersonation support" license = { file = "LICENSE" } From 92a2ddfc217b94c86d58b0aee5bbc70ecb046e95 Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Wed, 27 Dec 2023 23:01:30 +0800 Subject: [PATCH 15/19] Implement the websocket_client-like API --- README-zh.md | 1 + README.md | 1 + curl_cffi/curl.py | 3 +- curl_cffi/requests/__init__.py | 2 + curl_cffi/requests/session.py | 27 ++++++-- curl_cffi/requests/websockets.py | 69 +++++++++++++++++-- examples/websocket_server.py | 18 ----- examples/websockets/long_running.py | 41 +++++++++++ examples/websockets/long_running_async.py | 38 ++++++++++ .../short_running.py} | 8 +-- tests/unittest/test_smoke.py | 20 ++++++ 11 files changed, 194 insertions(+), 34 deletions(-) delete mode 100644 examples/websocket_server.py create mode 100644 examples/websockets/long_running.py create mode 100644 examples/websockets/long_running_async.py rename examples/{websocket.py => websockets/short_running.py} (66%) create mode 100644 tests/unittest/test_smoke.py diff --git a/README-zh.md b/README-zh.md index 3e70a16f..3909ab37 100644 --- a/README-zh.md +++ b/README-zh.md @@ -157,6 +157,7 @@ print(body.decode()) - 该项目 fork 自:[multippt/python_curl_cffi](https://github.com/multippt/python_curl_cffi), MIT 协议发布。 - Headers/Cookies 代码来自 [httpx](https://github.com/encode/httpx/blob/master/httpx/_models.py), BSD 协议发布。 - Asyncio 支持是受 Tornado 的 curl http client 启发而做。 +- WebSocket API 的设计来自 [websocket_client](https://github.com/websocket-client/websocket-client)。 ## 赞助 diff --git a/README.md b/README.md index 71e88ac4..2f369b40 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,7 @@ If you are using scrapy, check out these middlewares: - Originally forked from [multippt/python_curl_cffi](https://github.com/multippt/python_curl_cffi), which is under the MIT license. - Headers/Cookies files are copied from [httpx](https://github.com/encode/httpx/blob/master/httpx/_models.py), which is under the BSD license. - Asyncio support is inspired by Tornado's curl http client. +- The WebSocket API is inspired by [websocket_client](https://github.com/websocket-client/websocket-client) ## Sponsor diff --git a/curl_cffi/curl.py b/curl_cffi/curl.py index e08c2d18..ce31f599 100644 --- a/curl_cffi/curl.py +++ b/curl_cffi/curl.py @@ -347,8 +347,9 @@ def ws_recv(self, n: int = 1024): ret = lib.curl_ws_recv(self._curl, buffer, n, n_recv, p_frame) self._check_error(ret, "WS_RECV") + + # Frame meta explained: https://curl.se/libcurl/c/curl_ws_meta.html frame = p_frame[0] - # print(frame.offset, frame.bytesleft) return ffi.buffer(buffer)[: n_recv[0]], frame diff --git a/curl_cffi/requests/__init__.py b/curl_cffi/requests/__init__.py index 24beb15f..80ab213f 100644 --- a/curl_cffi/requests/__init__.py +++ b/curl_cffi/requests/__init__.py @@ -15,6 +15,7 @@ "Headers", "Request", "Response", + "WebSocket", ] from functools import partial @@ -27,6 +28,7 @@ from .errors import RequestsError from .headers import Headers, HeaderTypes from .session import AsyncSession, BrowserType, Session +from .websockets import WebSocket # ThreadType = Literal["eventlet", "gevent", None] diff --git a/curl_cffi/requests/session.py b/curl_cffi/requests/session.py index 0a910931..a6463d8b 100644 --- a/curl_cffi/requests/session.py +++ b/curl_cffi/requests/session.py @@ -64,7 +64,8 @@ def has(cls, item): class BrowserSpec: - """A more structured way of selecting browsers """ + """A more structured way of selecting browsers""" + # TODO @@ -270,7 +271,7 @@ def _set_curl_options( h.update(headers) # remove Host header if it's unnecessary, otherwise curl maybe confused. - # Host header will be automatically add by curl if it's not present. + # Host header will be automatically added by curl if it's not present. # https://github.com/yifeikong/curl_cffi/issues/119 host_header = h.get("Host") if host_header is not None: @@ -592,12 +593,30 @@ def stream(self, *args, **kwargs): finally: rsp.close() - def ws_connect(self, url, *args, **kwargs): + def ws_connect( + self, + url, + *args, + on_message: Optional[Callable[[WebSocket, str], None]] = None, + on_error: Optional[Callable[[WebSocket, str], None]] = None, + on_open: Optional[Callable] = None, + on_close: Optional[Callable] = None, + **kwargs, + ): self._set_curl_options(self.curl, "GET", url, *args, **kwargs) + # https://curl.se/docs/websocket.html self.curl.setopt(CurlOpt.CONNECT_ONLY, 2) self.curl.perform() - return WebSocket(self, self.curl) + + return WebSocket( + self, + self.curl, + on_message=on_message, + on_error=on_error, + on_open=on_open, + on_close=on_close, + ) def request( self, diff --git a/curl_cffi/requests/websockets.py b/curl_cffi/requests/websockets.py index 76a548fb..9fbbbd8a 100644 --- a/curl_cffi/requests/websockets.py +++ b/curl_cffi/requests/websockets.py @@ -1,25 +1,53 @@ +from typing import Callable, Optional, Tuple import asyncio from curl_cffi.const import CurlECode, CurlWsFlag from curl_cffi.curl import CurlError +ON_MESSAGE_T = Callable[["WebSocket", bytes], None] +ON_ERROR_T = Callable[["WebSocket", CurlError], None] +ON_OPEN_T = Callable[["WebSocket"], None] +ON_CLOSE_T = Callable[["WebSocket"], None] + + class WebSocket: - def __init__(self, session, curl): + def __init__( + self, + session, + curl, + on_message: Optional[ON_MESSAGE_T] = None, + on_error: Optional[ON_ERROR_T] = None, + on_open: Optional[ON_OPEN_T] = None, + on_close: Optional[ON_CLOSE_T] = None, + ): self.session = session self.curl = curl + self.on_message = on_message + self.on_error = on_error + self.on_open = on_open + self.on_close = on_close + self.keep_running = True self._loop = None def recv_fragment(self): return self.curl.ws_recv() - def recv(self): + def recv(self) -> Tuple[bytes, int]: + """ + Receive a frame as bytes. + + libcurl split frames into fragments, so we have to collect all the chunks for + a frame. + """ chunks = [] + flags = 0 # TODO use select here while True: try: chunk, frame = self.curl.ws_recv() + flags = frame.flags chunks.append(chunk) - if frame.bytesleft == 0: + if frame.bytesleft == 0 and flags & CurlWsFlag.CONT == 0: break except CurlError as e: if e.code == CurlECode.AGAIN: @@ -27,13 +55,40 @@ def recv(self): else: raise - return b"".join(chunks) + return b"".join(chunks), flags def send(self, payload: bytes, flags: CurlWsFlag = CurlWsFlag.BINARY): + """Send a data frame""" return self.curl.ws_send(payload, flags) - def close(self): - # FIXME how to reset. or can a curl handle connect to two websockets? + def run_forever(self): + """ + libcurl automatically handles pings and pongs. + + ref: https://curl.se/libcurl/c/libcurl-ws.html + """ + if self.on_open: + self.on_open(self) + try: + # Keep reading the messages and invoke callbacks + while self.keep_running: + try: + msg, flags = self.recv() + if self.on_message: + self.on_message(self, msg) + if flags & CurlWsFlag.CLOSE: + self.keep_running = False + except CurlError as e: + if self.on_error: + self.on_error(self, e) + finally: + if self.on_close: + self.on_close(self) + + def close(self, msg: bytes = b""): + # FIXME how to reset, or can a curl handle connect to two websockets? + self.send(msg, CurlWsFlag.CLOSE) + self.keep_running = False self.curl.close() @property @@ -51,4 +106,4 @@ async def asend(self, payload: bytes, flags: CurlWsFlag = CurlWsFlag.BINARY): async def aclose(self): await self.loop.run_in_executor(None, self.close) self.curl.reset() - self.session.push_curl(curl) + self.session.push_curl(self.curl) diff --git a/examples/websocket_server.py b/examples/websocket_server.py deleted file mode 100644 index accd928e..00000000 --- a/examples/websocket_server.py +++ /dev/null @@ -1,18 +0,0 @@ -import asyncio -import websockets - -async def hello(websocket): - name = (await websocket.recv()).decode() - print(f"<<< {name}") - - greeting = f"Hello {name}!" - - await websocket.send(greeting) - print(f">>> {greeting}") - -async def main(): - async with websockets.serve(hello, "localhost", 8765): - await asyncio.Future() # run forever - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/websockets/long_running.py b/examples/websockets/long_running.py new file mode 100644 index 00000000..a539e0be --- /dev/null +++ b/examples/websockets/long_running.py @@ -0,0 +1,41 @@ +from curl_cffi.requests import Session, WebSocket + + +msg_count = 0 + + +def on_message(ws: WebSocket, message): + global msg_count + + print("------------------------------------------------------") + print(message) + print("======================================================") + + msg_count += 1 + if msg_count >= 100: + ws.close() + + +def on_error(ws: WebSocket, error): + print(error) + + +def on_open(ws: WebSocket): + print("For websockets, you need to set $wss_proxy environment variable!\n" + "$https_proxy will not work!") + print(">>> Websocket open!") + + +def on_close(ws: WebSocket): + print("<<< Websocket closed!") + + +with Session() as s: + ws = s.ws_connect( + "wss://api.gemini.com/v1/marketdata/BTCUSD", + on_open=on_open, + on_close=on_close, + on_message=on_message, + on_error=on_error, + ) + ws.run_forever() diff --git a/examples/websockets/long_running_async.py b/examples/websockets/long_running_async.py new file mode 100644 index 00000000..8b69218c --- /dev/null +++ b/examples/websockets/long_running_async.py @@ -0,0 +1,38 @@ +""" +WIP: this has not been implemented yet. +""" +import asyncio +from curl_cffi import requests + + +async def on_message(ws, message): + print(message) + + +async def on_error(ws, error): + print(error) + + +async def on_open(ws): + print("For websockets, you need to set $wss_proxy environment variable!\n" + "$https_proxy will not work!") + print(">>> Websocket open!") + + +async def on_close(ws): + print("<<< Websocket closed!") + + +async def main(): + async with requests.AsyncSession() as s: + ws = await s.ws_connect( + "wss://api.gemini.com/v1/marketdata/BTCUSD", + on_open=on_open, + on_close=on_close, + on_message=on_message, + on_error=on_error, + ) + ws.run_forever() + + +asyncio.run(main()) diff --git a/examples/websocket.py b/examples/websockets/short_running.py similarity index 66% rename from examples/websocket.py rename to examples/websockets/short_running.py index 175df755..486f2a34 100644 --- a/examples/websocket.py +++ b/examples/websockets/short_running.py @@ -1,21 +1,21 @@ import asyncio from curl_cffi import requests +URL = "ws://echo.websocket.events" + with requests.Session() as s: - w = s.connect("ws://localhost:8765") + w = s.ws_connect(URL) w.send(b"Foo") reply = w.recv() print(reply) - assert reply == b"Hello Foo!" async def async_examples(): async with requests.AsyncSession() as s: - w = await s.connect("ws://localhost:8765") + w = await s.ws_connect(URL) await w.asend(b"Bar") reply = await w.arecv() print(reply) - assert reply == b"Hello Bar!" asyncio.run(async_examples()) diff --git a/tests/unittest/test_smoke.py b/tests/unittest/test_smoke.py new file mode 100644 index 00000000..40f46fbc --- /dev/null +++ b/tests/unittest/test_smoke.py @@ -0,0 +1,20 @@ +# Simple smoke test to real world websites +from curl_cffi import requests + + +URLS = [ + "https://www.zhihu.com", + "https://www.weibo.com", +] + + +def test_without_impersonate(): + for url in URLS: + r = requests.get(url) + assert r.status_code == 200 + + +def test_with_impersonate(): + for url in URLS: + r = requests.get(url, impersonate=requests.BrowserType.chrome) + assert r.status_code == 200 From 5e8a4775812f43d7da8c4168d7ce6da9f4c6deee Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Wed, 27 Dec 2023 23:12:24 +0800 Subject: [PATCH 16/19] Update readme with websocket examples --- README-zh.md | 22 +++++++++++++++++++++- README.md | 22 +++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/README-zh.md b/README-zh.md index 3909ab37..e8271061 100644 --- a/README-zh.md +++ b/README-zh.md @@ -14,12 +14,14 @@ TLS 或者 JA3 指纹。如果你莫名其妙地被某个网站封锁了,可 - 预编译,不需要再自己机器上再弄一遍。 - 支持 `asyncio`,并且每个请求都可以换代理。 - 支持 http 2.0,requests 不支持。 +- 支持 websocket。 |库|requests|aiohttp|httpx|pycurl|curl_cffi| |---|---|---|---|---|---| |http2|❌|❌|✅|✅|✅| |sync|✅|❌|✅|✅|✅| |async|❌|✅|✅|❌|✅| +|websocket|❌|✅|❌|❌|✅| |指纹|❌|❌|❌|❌|✅| |速度|🐇|🐇🐇|🐇|🐇🐇|🐇🐇| @@ -75,7 +77,7 @@ print(r.json()) 支持模拟的浏览器版本,和我 [fork](https://github.com/yifeikong/curl-impersonate) 的 [curl-impersonate](https://github.com/lwthiker/curl-impersonate) 一致: -不过只支持类似 Chrome 的浏览器。 +不过只支持类似 Chrome 的浏览器。Firefox 的支持进展可以查看 #55 - chrome99 - chrome100 @@ -85,11 +87,13 @@ print(r.json()) - chrome110 - chrome116 - chrome119 +- chrome120 - chrome99_android - edge99 - edge101 - safari15_3 - safari15_5 +- safari17_2_ios ### asyncio @@ -120,6 +124,22 @@ async with AsyncSession() as s: results = await asyncio.gather(*tasks) ``` +### WebSockets + +```python +from curl_cffi.requests import Session, WebSocket + +def on_message(ws: WebSocket, message): + print(message) + +with Session() as s: + ws = s.ws_connect( + "wss://api.gemini.com/v1/marketdata/BTCUSD", + on_message=on_message, + ) + ws.run_forever() +``` + ### 类 curl 另外,你还可以使用类似 curl 的底层 API: diff --git a/README.md b/README.md index 2f369b40..aebc3309 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,14 @@ website for no obvious reason, you can give this package a try. - Pre-compiled, so you don't have to compile on your machine. - Supports `asyncio` with proxy rotation on each request. - Supports http 2.0, which requests does not. +- Supports websocket. |library|requests|aiohttp|httpx|pycurl|curl_cffi| |---|---|---|---|---|---| |http2|❌|❌|✅|✅|✅| |sync|✅|❌|✅|✅|✅| |async|❌|✅|✅|❌|✅| +|websocket|❌|✅|❌|❌|✅| |fingerprints|❌|❌|❌|❌|✅| |speed|🐇|🐇🐇|🐇|🐇🐇|🐇🐇| @@ -78,7 +80,7 @@ print(r.json()) Supported impersonate versions, as supported by my [fork](https://github.com/yifeikong/curl-impersonate) of [curl-impersonate](https://github.com/lwthiker/curl-impersonate): -However, only Chrome-like browsers are supported. +However, only Chrome-like browsers are supported. Firefox support is tracked in #55 - chrome99 - chrome100 @@ -88,11 +90,13 @@ However, only Chrome-like browsers are supported. - chrome110 - chrome116 - chrome119 +- chrome120 - chrome99_android - edge99 - edge101 - safari15_3 - safari15_5 +- safari17_2_ios ### asyncio @@ -123,6 +127,22 @@ async with AsyncSession() as s: results = await asyncio.gather(*tasks) ``` +### WebSockets + +```python +from curl_cffi.requests import Session, WebSocket + +def on_message(ws: WebSocket, message): + print(message) + +with Session() as s: + ws = s.ws_connect( + "wss://api.gemini.com/v1/marketdata/BTCUSD", + on_message=on_message, + ) + ws.run_forever() +``` + ### curl-like Alternatively, you can use the low-level curl-like API: From 905cf94a8cc678fcdbc57e9e7a5df4a428217ede Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Thu, 28 Dec 2023 00:39:23 +0800 Subject: [PATCH 17/19] Fix websockets tests --- curl_cffi/curl.py | 2 +- tests/unittest/test_websockets.py | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/curl_cffi/curl.py b/curl_cffi/curl.py index ce31f599..e86b23bb 100644 --- a/curl_cffi/curl.py +++ b/curl_cffi/curl.py @@ -358,7 +358,7 @@ def ws_send(self, payload: bytes, flags: CurlWsFlag = CurlWsFlag.BINARY) -> int: buffer = ffi.from_buffer(payload) ret = lib.curl_ws_send(self._curl, buffer, len(buffer), n_sent, 0, flags) self._check_error(ret, "WS_SEND") - return n_sent + return n_sent[0] def ws_close(self): self.ws_send(b"", CurlWsFlag.CLOSE) diff --git a/tests/unittest/test_websockets.py b/tests/unittest/test_websockets.py index 736d21d5..816d4b27 100644 --- a/tests/unittest/test_websockets.py +++ b/tests/unittest/test_websockets.py @@ -9,6 +9,22 @@ def test_websocket(ws_server): def test_hello(ws_server): with Session() as s: ws = s.ws_connect(ws_server.url) - ws.send(b"foo") - content = ws.recv() - assert content == b"foo" + ws.send(b"Foo me once") + content, _ = ws.recv() + assert content == b"Foo me once" + + +def test_hello_twice(ws_server): + with Session() as s: + # w = s.ws_connect(ws_server.url) + w = s.ws_connect("ws://echo.websocket.events") + + w.send(b"Bar") + reply, _ = w.recv() + print(reply) + + for _ in range(10): + w.send(b"Bar") + reply, _ = w.recv() + assert reply == b"Bar" + print(reply) From f361e915a06b0a531fd767282daebc02887e9d5e Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Thu, 28 Dec 2023 00:47:55 +0800 Subject: [PATCH 18/19] Fix echo server --- tests/unittest/conftest.py | 9 +++++---- tests/unittest/test_websockets.py | 3 +-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/unittest/conftest.py b/tests/unittest/conftest.py index 3005e3e5..0316b14c 100644 --- a/tests/unittest/conftest.py +++ b/tests/unittest/conftest.py @@ -560,11 +560,12 @@ async def watch_restarts(self): # pragma: nocover async def echo(websocket): - name = (await websocket.recv()).decode() - # print(f"<<< {name}") + while True: + name = (await websocket.recv()).decode() + # print(f"<<< {name}") - await websocket.send(name) - # print(f">>> {name}") + await websocket.send(name) + # print(f">>> {name}") class TestWebsocketServer: diff --git a/tests/unittest/test_websockets.py b/tests/unittest/test_websockets.py index 816d4b27..8b26423b 100644 --- a/tests/unittest/test_websockets.py +++ b/tests/unittest/test_websockets.py @@ -16,8 +16,7 @@ def test_hello(ws_server): def test_hello_twice(ws_server): with Session() as s: - # w = s.ws_connect(ws_server.url) - w = s.ws_connect("ws://echo.websocket.events") + w = s.ws_connect(ws_server.url) w.send(b"Bar") reply, _ = w.recv() From 17489822e6bdf561f64f56a9d7e4f2f939122580 Mon Sep 17 00:00:00 2001 From: Yifei Kong Date: Sun, 31 Dec 2023 17:26:31 +0800 Subject: [PATCH 19/19] Re-enable windows --- .github/workflows/build.yaml | 4 +--- .github/workflows/test.yaml | 4 +--- Makefile | 2 +- curl_cffi/__version__.py | 2 +- preprocess/download_so.py | 8 ++------ pyproject.toml | 2 +- 6 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index eebb6976..64292bb2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -18,8 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # os: [ubuntu-22.04, macos-11, windows-2019] - os: [ubuntu-22.04, macos-11] + os: [ubuntu-22.04, macos-11, windows-2019] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -51,4 +50,3 @@ jobs: - uses: pypa/gh-action-pypi-publish@v1.5.0 with: password: ${{ secrets.PYPI_TOKEN }} - diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 86bc96ee..0ff09fd3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,7 +3,6 @@ on: push: branches: - main - - master - bugfix/* - feature/* jobs: @@ -12,8 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # os: [ubuntu-22.04, macos-11, windows-2019] - os: [ubuntu-22.04, macos-11] + os: [ubuntu-22.04, macos-11, windows-2019] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/Makefile b/Makefile index 5aab1267..10a549dc 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .ONESHELL: SHELL := bash -VERSION := 0.6.0b4 +VERSION := 0.6.0b6 CURL_VERSION := curl-8.1.1 .preprocessed: curl_cffi/include/curl/curl.h curl_cffi/cacert.pem .so_downloaded diff --git a/curl_cffi/__version__.py b/curl_cffi/__version__.py index 5b27dc03..c3bfd383 100644 --- a/curl_cffi/__version__.py +++ b/curl_cffi/__version__.py @@ -7,5 +7,5 @@ # __description__ = metadata.metadata("curl_cffi")["Summary"] # __version__ = metadata.version("curl_cffi") __description__ = "libcurl ffi bindings for Python, with impersonation support" -__version__ = "0.6.0b4" +__version__ = "0.6.0b6" __curl_version__ = Curl().version().decode() diff --git a/preprocess/download_so.py b/preprocess/download_so.py index c56f0dd4..03f37ca2 100644 --- a/preprocess/download_so.py +++ b/preprocess/download_so.py @@ -1,4 +1,3 @@ -import os import platform import shutil import sys @@ -32,17 +31,14 @@ def reporthook(blocknum, blocksize, totalsize): if uname.machine == "arm64": # TODO Download my own build of libcurl-impersonate for M1 Mac url = "" - filename = "./curl-impersonate.tar.gz" else: url = f"https://github.com/yifeikong/curl-impersonate/releases/download/v{VERSION}/libcurl-impersonate-v{VERSION}.{uname.machine}-macos.tar.gz" - filename = "./curl-impersonate.tar.gz" elif uname.system == "Windows": - url = f"https://github.com/yifeikong/curl-impersonate-win/releases/download/v{VERSION}/curl-impersonate-chrome.tar.gz" - filename = "./curl-impersonate.tar.gz" + url = f"https://github.com/yifeikong/curl-impersonate/releases/download/v{VERSION}/libcurl-impersonate-v{VERSION}.{uname.machine}-win32.tar.gz" else: url = f"https://github.com/yifeikong/curl-impersonate/releases/download/v{VERSION}/libcurl-impersonate-v{VERSION}.{uname.machine}-linux-gnu.tar.gz" - filename = "./curl-impersonate.tar.gz" +filename = "./curl-impersonate.tar.gz" if url: print(f"Download libcurl-impersonate-chrome from {url}") urlretrieve(url, filename, reporthook) diff --git a/pyproject.toml b/pyproject.toml index 19d0af9d..15422e8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "curl_cffi" -version = "0.6.0b4" +version = "0.6.0b6" authors = [{ name = "Yifei Kong", email = "kong@yifei.me" }] description = "libcurl ffi bindings for Python, with impersonation support" license = { file = "LICENSE" }