From 471f860a63688b950096bd796ce6ff645e817f36 Mon Sep 17 00:00:00 2001 From: Chayim Date: Wed, 2 Aug 2023 14:02:21 +0300 Subject: [PATCH 1/6] Fixing doc builds (#2869) --- .github/workflows/docs.yaml | 47 ++++++++++++ docs/conf.py | 4 +- docs/examples/redis-stream-example.ipynb | 2 +- docs/requirements.txt | 1 + redis/asyncio/client.py | 9 ++- redis/asyncio/connection.py | 9 ++- redis/commands/core.py | 95 +++++++++++++++++------- tasks.py | 2 +- 8 files changed, 133 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/docs.yaml diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 0000000000..61ec76e9f8 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,47 @@ +name: Docs CI + +on: + push: + branches: + - master + - '[0-9].[0-9]' + pull_request: + branches: + - master + - '[0-9].[0-9]' + schedule: + - cron: '0 1 * * *' # nightly build + +concurrency: + group: ${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + + build-docs: + name: Build docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.9 + cache: 'pip' + - name: install deps + run: | + sudo apt-get update -yqq + sudo apt-get install -yqq pandoc make + - name: run code linters + run: | + pip install -r requirements.txt -r dev_requirements.txt -r docs/requirements.txt + invoke build-docs + + - name: upload docs + uses: actions/upload-artifact@v3 + with: + name: redis-py-docs + path: | + docs/_build/html diff --git a/docs/conf.py b/docs/conf.py index cdbeb02c9a..8849752404 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,7 +60,7 @@ # General information about the project. project = "redis-py" -copyright = "2022, Redis Inc" +copyright = "2023, Redis Inc" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -287,4 +287,4 @@ epub_title = "redis-py" epub_author = "Redis Inc" epub_publisher = "Redis Inc" -epub_copyright = "2022, Redis Inc" +epub_copyright = "2023, Redis Inc" diff --git a/docs/examples/redis-stream-example.ipynb b/docs/examples/redis-stream-example.ipynb index 9303b527ca..a84bf19cb6 100644 --- a/docs/examples/redis-stream-example.ipynb +++ b/docs/examples/redis-stream-example.ipynb @@ -313,7 +313,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# stream groups\n", + "# Stream groups\n", "With the groups is possible track, for many consumers, and at the Redis side, which message have been already consumed.\n", "## add some data to streams\n", "Creating 2 streams with 10 messages each." diff --git a/docs/requirements.txt b/docs/requirements.txt index edecdffe4b..5b15c09268 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,3 +5,4 @@ sphinx_gallery ipython sphinx-autodoc-typehints furo +pandoc diff --git a/redis/asyncio/client.py b/redis/asyncio/client.py index 31e27a4462..0e3c879278 100644 --- a/redis/asyncio/client.py +++ b/redis/asyncio/client.py @@ -134,10 +134,13 @@ def from_url( There are several ways to specify a database number. The first value found will be used: - 1. A ``db`` querystring option, e.g. redis://localhost?db=0 - 2. If using the redis:// or rediss:// schemes, the path argument + + 1. A ``db`` querystring option, e.g. redis://localhost?db=0 + + 2. If using the redis:// or rediss:// schemes, the path argument of the url, e.g. redis://localhost/0 - 3. A ``db`` keyword argument to this function. + + 3. A ``db`` keyword argument to this function. If none of these options are specified, the default db=0 is used. diff --git a/redis/asyncio/connection.py b/redis/asyncio/connection.py index 22c5030e6c..d501989c83 100644 --- a/redis/asyncio/connection.py +++ b/redis/asyncio/connection.py @@ -948,10 +948,13 @@ def from_url(cls: Type[_CP], url: str, **kwargs) -> _CP: There are several ways to specify a database number. The first value found will be used: - 1. A ``db`` querystring option, e.g. redis://localhost?db=0 - 2. If using the redis:// or rediss:// schemes, the path argument + + 1. A ``db`` querystring option, e.g. redis://localhost?db=0 + + 2. If using the redis:// or rediss:// schemes, the path argument of the url, e.g. redis://localhost/0 - 3. A ``db`` keyword argument to this function. + + 3. A ``db`` keyword argument to this function. If none of these options are specified, the default db=0 is used. diff --git a/redis/commands/core.py b/redis/commands/core.py index 6abcd5a2ec..8b1b711df9 100644 --- a/redis/commands/core.py +++ b/redis/commands/core.py @@ -517,6 +517,7 @@ def client_list( """ Returns a list of currently connected clients. If type of client specified, only that type will be returned. + :param _type: optional. one of the client types (normal, master, replica, pubsub) :param client_id: optional. a list of client ids @@ -559,16 +560,17 @@ def client_reply( ) -> ResponseT: """ Enable and disable redis server replies. + ``reply`` Must be ON OFF or SKIP, - ON - The default most with server replies to commands - OFF - Disable server responses to commands - SKIP - Skip the response of the immediately following command. + ON - The default most with server replies to commands + OFF - Disable server responses to commands + SKIP - Skip the response of the immediately following command. Note: When setting OFF or SKIP replies, you will need a client object with a timeout specified in seconds, and will need to catch the TimeoutError. - The test_client_reply unit test illustrates this, and - conftest.py has a client with a timeout. + The test_client_reply unit test illustrates this, and + conftest.py has a client with a timeout. See https://redis.io/commands/client-reply """ @@ -724,19 +726,21 @@ def client_unblock( def client_pause(self, timeout: int, all: bool = True, **kwargs) -> ResponseT: """ - Suspend all the Redis clients for the specified amount of time - :param timeout: milliseconds to pause clients + Suspend all the Redis clients for the specified amount of time. + For more information see https://redis.io/commands/client-pause + + :param timeout: milliseconds to pause clients :param all: If true (default) all client commands are blocked. - otherwise, clients are only blocked if they attempt to execute - a write command. - For the WRITE mode, some commands have special behavior: - EVAL/EVALSHA: Will block client for all scripts. - PUBLISH: Will block client. - PFCOUNT: Will block client. - WAIT: Acknowledgments will be delayed, so this command will - appear blocked. + otherwise, clients are only blocked if they attempt to execute + a write command. + For the WRITE mode, some commands have special behavior: + EVAL/EVALSHA: Will block client for all scripts. + PUBLISH: Will block client. + PFCOUNT: Will block client. + WAIT: Acknowledgments will be delayed, so this command will + appear blocked. """ args = ["CLIENT PAUSE", str(timeout)] if not isinstance(timeout, int): @@ -1215,9 +1219,11 @@ def quit(self, **kwargs) -> ResponseT: def replicaof(self, *args, **kwargs) -> ResponseT: """ Update the replication settings of a redis replica, on the fly. + Examples of valid arguments include: - NO ONE (set no replication) - host port (set to the host and port of a redis server) + + NO ONE (set no replication) + host port (set to the host and port of a redis server) For more information see https://redis.io/commands/replicaof """ @@ -3603,27 +3609,37 @@ def xclaim( ) -> ResponseT: """ Changes the ownership of a pending message. + name: name of the stream. + groupname: name of the consumer group. + consumername: name of a consumer that claims the message. + min_idle_time: filter messages that were idle less than this amount of milliseconds + message_ids: non-empty list or tuple of message IDs to claim + idle: optional. Set the idle time (last time it was delivered) of the - message in ms + message in ms + time: optional integer. This is the same as idle but instead of a - relative amount of milliseconds, it sets the idle time to a specific - Unix time (in milliseconds). + relative amount of milliseconds, it sets the idle time to a specific + Unix time (in milliseconds). + retrycount: optional integer. set the retry counter to the specified - value. This counter is incremented every time a message is delivered - again. + value. This counter is incremented every time a message is delivered + again. + force: optional boolean, false by default. Creates the pending message - entry in the PEL even if certain specified IDs are not already in the - PEL assigned to a different client. + entry in the PEL even if certain specified IDs are not already in the + PEL assigned to a different client. + justid: optional boolean, false by default. Return just an array of IDs - of messages successfully claimed, without returning the actual message + of messages successfully claimed, without returning the actual message - For more information see https://redis.io/commands/xclaim + For more information see https://redis.io/commands/xclaim """ if not isinstance(min_idle_time, int) or min_idle_time < 0: raise DataError("XCLAIM min_idle_time must be a non negative integer") @@ -3875,11 +3891,15 @@ def xrange( ) -> ResponseT: """ Read stream values within an interval. + name: name of the stream. + start: first stream ID. defaults to '-', meaning the earliest available. + finish: last stream ID. defaults to '+', meaning the latest available. + count: if set, only return this many items, beginning with the earliest available. @@ -3902,10 +3922,13 @@ def xread( ) -> ResponseT: """ Block and monitor multiple streams for new data. + streams: a dict of stream names to stream IDs, where IDs indicate the last ID already seen. + count: if set, only return this many items, beginning with the earliest available. + block: number of milliseconds to wait, if nothing already present. For more information see https://redis.io/commands/xread @@ -3940,12 +3963,17 @@ def xreadgroup( ) -> ResponseT: """ Read from a stream via a consumer group. + groupname: name of the consumer group. + consumername: name of the requesting consumer. + streams: a dict of stream names to stream IDs, where IDs indicate the last ID already seen. + count: if set, only return this many items, beginning with the earliest available. + block: number of milliseconds to wait, if nothing already present. noack: do not add messages to the PEL @@ -3980,11 +4008,15 @@ def xrevrange( ) -> ResponseT: """ Read stream values within an interval, in reverse order. + name: name of the stream + start: first stream ID. defaults to '+', meaning the latest available. + finish: last stream ID. defaults to '-', meaning the earliest available. + count: if set, only return this many items, beginning with the latest available. @@ -5301,8 +5333,10 @@ def script_flush( self, sync_type: Union[Literal["SYNC"], Literal["ASYNC"]] = None ) -> ResponseT: """Flush all scripts from the script cache. + ``sync_type`` is by default SYNC (synchronous) but it can also be ASYNC. + For more information see https://redis.io/commands/script-flush """ @@ -5615,11 +5649,14 @@ def geosearch( area specified by a given shape. This command extends the GEORADIUS command, so in addition to searching within circular areas, it supports searching within rectangular areas. + This command should be used in place of the deprecated GEORADIUS and GEORADIUSBYMEMBER commands. + ``member`` Use the position of the given existing member in the sorted set. Can't be given with ``longitude`` and ``latitude``. + ``longitude`` and ``latitude`` Use the position given by this coordinates. Can't be given with ``member`` ``radius`` Similar to GEORADIUS, search inside circular @@ -5628,17 +5665,23 @@ def geosearch( ``height`` and ``width`` Search inside an axis-aligned rectangle, determined by the given height and width. Can't be given with ``radius`` + ``unit`` must be one of the following : m, km, mi, ft. `m` for meters (the default value), `km` for kilometers, `mi` for miles and `ft` for feet. + ``sort`` indicates to return the places in a sorted way, ASC for nearest to furthest and DESC for furthest to nearest. + ``count`` limit the results to the first count matching items. + ``any`` is set to True, the command will return as soon as enough matches are found. Can't be provided without ``count`` + ``withdist`` indicates to return the distances of each place. ``withcoord`` indicates to return the latitude and longitude of each place. + ``withhash`` indicates to return the geohash string of each place. For more information see https://redis.io/commands/geosearch diff --git a/tasks.py b/tasks.py index 5162566183..c60fa2791e 100644 --- a/tasks.py +++ b/tasks.py @@ -21,7 +21,7 @@ def devenv(c): def build_docs(c): """Generates the sphinx documentation.""" run("pip install -r docs/requirements.txt") - run("make html") + run("make -C docs html") @task From a49e65682c17d338ba4343154bc5e4b23e58068f Mon Sep 17 00:00:00 2001 From: Chayim Date: Wed, 2 Aug 2023 14:03:30 +0300 Subject: [PATCH 2/6] RESP3 connection examples (#2863) --- README.md | 12 +- docs/examples/asyncio_examples.ipynb | 210 ++++++++++++++++-------- docs/examples/connection_examples.ipynb | 65 ++++++-- 3 files changed, 203 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index e97119a888..e4e0debe03 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,8 @@ The table below higlights version compatibility of the most-recent library versi | Library version | Supported redis versions | |-----------------|-------------------| | 3.5.3 | <= 6.2 Family of releases | -| >= 4.1.0 | Version 5.0 to current | +| >= 4.5.0 | Version 5.0 to 7.0 | +| >= 5.0.0 | Versiond 5.0 to current | ## Usage @@ -63,6 +64,15 @@ b'bar' The above code connects to localhost on port 6379, sets a value in Redis, and retrieves it. All responses are returned as bytes in Python, to receive decoded strings, set *decode_responses=True*. For this, and more connection options, see [these examples](https://redis.readthedocs.io/en/stable/examples.html). + +#### RESP3 Support +To enable support for RESP3, ensure you have at least version 5.0 of the client, and change your connection object to include *protocol=3* + +``` python +>>> import redis +>>> r = redis.Redis(host='localhost', port=6379, db=0, protocol=3) +``` + ### Connection Pools By default, redis-py uses a connection pool to manage connections. Each instance of a Redis class receives its own connection pool. You can however define your own [redis.ConnectionPool](https://redis.readthedocs.io/en/stable/connections.html#connection-pools). diff --git a/docs/examples/asyncio_examples.ipynb b/docs/examples/asyncio_examples.ipynb index 855255c88d..7fdcc36bc5 100644 --- a/docs/examples/asyncio_examples.ipynb +++ b/docs/examples/asyncio_examples.ipynb @@ -21,6 +21,12 @@ { "cell_type": "code", "execution_count": 1, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", @@ -36,29 +42,29 @@ "connection = redis.Redis()\n", "print(f\"Ping successful: {await connection.ping()}\")\n", "await connection.close()" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] }, { "cell_type": "markdown", - "source": [ - "If you supply a custom `ConnectionPool` that is supplied to several `Redis` instances, you may want to disconnect the connection pool explicitly. Disconnecting the connection pool simply disconnects all connections hosted in the pool." - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "If you supply a custom `ConnectionPool` that is supplied to several `Redis` instances, you may want to disconnect the connection pool explicitly. Disconnecting the connection pool simply disconnects all connections hosted in the pool." + ] }, { "cell_type": "code", "execution_count": 2, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "import redis.asyncio as redis\n", @@ -67,16 +73,36 @@ "await connection.close()\n", "# Or: await connection.close(close_connection_pool=False)\n", "await connection.connection_pool.disconnect()" - ], + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By default, this library uses version 2 of the RESP protocol. To enable RESP version 3, you will want to set `protocol` to 3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import redis.asyncio as redis\n", + "\n", + "connection = redis.Redis(protocol=3)\n", + "await connection.close()\n", + "await connection.ping()" + ] + }, + { + "cell_type": "markdown", "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n" + "name": "#%% md\n" } - } - }, - { - "cell_type": "markdown", + }, "source": [ "## Transactions (Multi/Exec)\n", "\n", @@ -85,17 +111,17 @@ "The commands will not be reflected in Redis until execute() is called & awaited.\n", "\n", "Usually, when performing a bulk operation, taking advantage of a “transaction” (e.g., Multi/Exec) is to be desired, as it will also add a layer of atomicity to your bulk operation." - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } + ] }, { "cell_type": "code", "execution_count": 3, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "import redis.asyncio as redis\n", @@ -105,31 +131,31 @@ " ok1, ok2 = await (pipe.set(\"key1\", \"value1\").set(\"key2\", \"value2\").execute())\n", "assert ok1\n", "assert ok2" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] }, { "cell_type": "markdown", - "source": [ - "## Pub/Sub Mode\n", - "\n", - "Subscribing to specific channels:" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "## Pub/Sub Mode\n", + "\n", + "Subscribing to specific channels:" + ] }, { "cell_type": "code", "execution_count": 4, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", @@ -170,29 +196,29 @@ " await r.publish(\"channel:1\", STOPWORD)\n", "\n", " await future" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] }, { "cell_type": "markdown", - "source": [ - "Subscribing to channels matching a glob-style pattern:" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "Subscribing to channels matching a glob-style pattern:" + ] }, { "cell_type": "code", "execution_count": 5, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", @@ -234,16 +260,16 @@ " await r.publish(\"channel:1\", STOPWORD)\n", "\n", " await future" - ], + ] + }, + { + "cell_type": "markdown", "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n" + "name": "#%% md\n" } - } - }, - { - "cell_type": "markdown", + }, "source": [ "## Sentinel Client\n", "\n", @@ -252,17 +278,17 @@ "Calling aioredis.sentinel.Sentinel.master_for or aioredis.sentinel.Sentinel.slave_for methods will return Redis clients connected to specified services monitored by Sentinel.\n", "\n", "Sentinel client will detect failover and reconnect Redis clients automatically." - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } + ] }, { "cell_type": "code", "execution_count": null, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "import asyncio\n", @@ -277,13 +303,61 @@ "assert ok\n", "val = await r.get(\"key\")\n", "assert val == b\"value\"" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Connecting to Redis instances by specifying a URL scheme.\n", + "Parameters are passed to the following schems, as parameters to the url scheme.\n", + "\n", + "Three URL schemes are supported:\n", + "\n", + "- `redis://` creates a TCP socket connection. \n", + "- `rediss://` creates a SSL wrapped TCP socket connection. \n", + "- ``unix://``: creates a Unix Domain Socket connection.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "metadata": {}, + "output_type": "display_data" } - } + ], + "source": [ + "import redis.asyncio as redis\n", + "url_connection = redis.from_url(\"redis://localhost:6379?decode_responses=True\")\n", + "url_connection.ping()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To enable the RESP 3 protocol, append `protocol=3` to the URL." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import redis.asyncio as redis\n", + "\n", + "url_connection = redis.from_url(\"redis://localhost:6379?decode_responses=Trueprotocol=3\")\n", + "url_connection.ping()" + ] } ], "metadata": { @@ -307,4 +381,4 @@ }, "nbformat": 4, "nbformat_minor": 1 -} \ No newline at end of file +} diff --git a/docs/examples/connection_examples.ipynb b/docs/examples/connection_examples.ipynb index d15d964af7..e6d147c920 100644 --- a/docs/examples/connection_examples.ipynb +++ b/docs/examples/connection_examples.ipynb @@ -41,7 +41,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### by default Redis return binary responses, to decode them use decode_responses=True" + "### By default Redis return binary responses, to decode them use decode_responses=True" ] }, { @@ -67,6 +67,25 @@ "decoded_connection.ping()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "### by default this library uses the RESP 2 protocol. To eanble RESP3, set protocol=3." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "import redis\n", + "\n", + "r = redis.Redis(protocol=3)\n", + "rcon.ping()" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -99,14 +118,15 @@ }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## Connecting to a redis instance with username and password credential provider" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "import redis\n", @@ -114,19 +134,19 @@ "creds_provider = redis.UsernamePasswordCredentialProvider(\"username\", \"password\")\n", "user_connection = redis.Redis(host=\"localhost\", port=6379, credential_provider=creds_provider)\n", "user_connection.ping()" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## Connecting to a redis instance with standard credential provider" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "from typing import Tuple\n", @@ -158,19 +178,19 @@ "user_connection = redis.Redis(host=\"localhost\", port=6379,\n", " credential_provider=creds_provider)\n", "user_connection.ping()" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## Connecting to a redis instance first with an initial credential set and then calling the credential provider" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "from typing import Union\n", @@ -194,8 +214,7 @@ " return self.username, self.password\n", "\n", "cred_provider = InitCredsSetCredentialProvider(username=\"init_user\", password=\"init_pass\")" - ], - "metadata": {} + ] }, { "cell_type": "markdown", @@ -357,7 +376,23 @@ ], "source": [ "url_connection = redis.from_url(\"redis://localhost:6379?decode_responses=True&health_check_interval=2\")\n", - "\n", + "url_connection.ping()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Connecting to Redis instances by specifying a URL scheme and the RESP3 protocol.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url_connection = redis.from_url(\"redis://localhost:6379?decode_responses=True&health_check_interval=2&protocol=3\")\n", "url_connection.ping()" ] }, @@ -404,4 +439,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} From dc62e19231ae4ba4317ede15c3aa9de4f93cd328 Mon Sep 17 00:00:00 2001 From: Chayim Date: Wed, 2 Aug 2023 18:51:49 +0300 Subject: [PATCH 3/6] EOL for Python 3.7 (#2852) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index e4e0debe03..67912eb3ef 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ The Python interface to the Redis key-value store. --------------------------------------------- +**Note: ** redis-py 5.0 will be the last version of redis-py to support Python 3.7, as it has reached [end of life](https://devguide.python.org/versions/). redis-py 5.1 will support Python 3.8+. + +--------------------------------------------- + ## Installation Start a redis via docker: From 7d70c9123bd54ff42d3080c9596c81126ef7e4bc Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Wed, 2 Aug 2023 10:52:24 -0500 Subject: [PATCH 4/6] Fix a duplicate word in `CONTRIBUTING.md` (#2848) --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 90a538be46..1081f4cb46 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -179,6 +179,6 @@ you would like and how it should work. ## Code review process The core team regularly looks at pull requests. We will provide -feedback as as soon as possible. After receiving our feedback, please respond +feedback as soon as possible. After receiving our feedback, please respond within two weeks. After that time, we may close your PR if it isn't showing any activity. From 66bad8eda8a1513e111a0c0bf3c79b8b9536e33e Mon Sep 17 00:00:00 2001 From: dvora-h <67596500+dvora-h@users.noreply.github.com> Date: Thu, 3 Aug 2023 03:36:57 +0300 Subject: [PATCH 5/6] Add sync modules (except search) tests to cluster CI (#2850) * Add modules to cluster ci * remove async tests * fix protocol checking * fix tests * revert cluster docker change * skip json 2.6.0 tests * remove breakpoint * skip test_get_latest * skip json.mset * type hint * revert type hints * ilnters --------- Co-authored-by: Chayim --- redis/commands/bf/__init__.py | 12 +++--- redis/commands/helpers.py | 8 ++++ redis/commands/json/__init__.py | 13 ++---- redis/commands/timeseries/__init__.py | 4 +- tests/test_asyncio/test_bloom.py | 2 +- tests/test_asyncio/test_json.py | 1 + tests/test_bloom.py | 29 +------------ tests/test_graph.py | 22 ---------- tests/test_json.py | 60 +++------------------------ tests/test_timeseries.py | 36 ++-------------- 10 files changed, 32 insertions(+), 155 deletions(-) diff --git a/redis/commands/bf/__init__.py b/redis/commands/bf/__init__.py index bfa9456879..959358f8e8 100644 --- a/redis/commands/bf/__init__.py +++ b/redis/commands/bf/__init__.py @@ -1,6 +1,6 @@ from redis._parsers.helpers import bool_ok -from ..helpers import parse_to_list +from ..helpers import get_protocol_version, parse_to_list from .commands import * # noqa from .info import BFInfo, CFInfo, CMSInfo, TDigestInfo, TopKInfo @@ -108,7 +108,7 @@ def __init__(self, client, **kwargs): self.commandmixin = CMSCommands self.execute_command = client.execute_command - if self.client.connection_pool.connection_kwargs.get("protocol") in ["3", 3]: + if get_protocol_version(self.client) in ["3", 3]: _MODULE_CALLBACKS.update(_RESP3_MODULE_CALLBACKS) else: _MODULE_CALLBACKS.update(_RESP2_MODULE_CALLBACKS) @@ -139,7 +139,7 @@ def __init__(self, client, **kwargs): self.commandmixin = TOPKCommands self.execute_command = client.execute_command - if self.client.connection_pool.connection_kwargs.get("protocol") in ["3", 3]: + if get_protocol_version(self.client) in ["3", 3]: _MODULE_CALLBACKS.update(_RESP3_MODULE_CALLBACKS) else: _MODULE_CALLBACKS.update(_RESP2_MODULE_CALLBACKS) @@ -174,7 +174,7 @@ def __init__(self, client, **kwargs): self.commandmixin = CFCommands self.execute_command = client.execute_command - if self.client.connection_pool.connection_kwargs.get("protocol") in ["3", 3]: + if get_protocol_version(self.client) in ["3", 3]: _MODULE_CALLBACKS.update(_RESP3_MODULE_CALLBACKS) else: _MODULE_CALLBACKS.update(_RESP2_MODULE_CALLBACKS) @@ -210,7 +210,7 @@ def __init__(self, client, **kwargs): self.commandmixin = TDigestCommands self.execute_command = client.execute_command - if self.client.connection_pool.connection_kwargs.get("protocol") in ["3", 3]: + if get_protocol_version(self.client) in ["3", 3]: _MODULE_CALLBACKS.update(_RESP3_MODULE_CALLBACKS) else: _MODULE_CALLBACKS.update(_RESP2_MODULE_CALLBACKS) @@ -244,7 +244,7 @@ def __init__(self, client, **kwargs): self.commandmixin = BFCommands self.execute_command = client.execute_command - if self.client.connection_pool.connection_kwargs.get("protocol") in ["3", 3]: + if get_protocol_version(self.client) in ["3", 3]: _MODULE_CALLBACKS.update(_RESP3_MODULE_CALLBACKS) else: _MODULE_CALLBACKS.update(_RESP2_MODULE_CALLBACKS) diff --git a/redis/commands/helpers.py b/redis/commands/helpers.py index b65cd1a933..324d981d66 100644 --- a/redis/commands/helpers.py +++ b/redis/commands/helpers.py @@ -3,6 +3,7 @@ import string from typing import List, Tuple +import redis from redis.typing import KeysT, KeyT @@ -156,3 +157,10 @@ def stringify_param_value(value): return f'{{{",".join(f"{k}:{stringify_param_value(v)}" for k, v in value.items())}}}' # noqa else: return str(value) + + +def get_protocol_version(client): + if isinstance(client, redis.Redis) or isinstance(client, redis.asyncio.Redis): + return client.connection_pool.connection_kwargs.get("protocol") + elif isinstance(client, redis.cluster.AbstractRedisCluster): + return client.nodes_manager.connection_kwargs.get("protocol") diff --git a/redis/commands/json/__init__.py b/redis/commands/json/__init__.py index e895e6a2ba..01077e6b88 100644 --- a/redis/commands/json/__init__.py +++ b/redis/commands/json/__init__.py @@ -2,7 +2,7 @@ import redis -from ..helpers import nativestr +from ..helpers import get_protocol_version, nativestr from .commands import JSONCommands from .decoders import bulk_of_jsons, decode_list @@ -34,6 +34,7 @@ def __init__( self._MODULE_CALLBACKS = { "JSON.ARRPOP": self._decode, "JSON.DEBUG": self._decode, + "JSON.GET": self._decode, "JSON.MERGE": lambda r: r and nativestr(r) == "OK", "JSON.MGET": bulk_of_jsons(self._decode), "JSON.MSET": lambda r: r and nativestr(r) == "OK", @@ -61,19 +62,13 @@ def __init__( "JSON.TOGGLE": self._decode, } - _RESP3_MODULE_CALLBACKS = { - "JSON.GET": lambda response: [ - [self._decode(r) for r in res] for res in response - ] - if response - else response - } + _RESP3_MODULE_CALLBACKS = {} self.client = client self.execute_command = client.execute_command self.MODULE_VERSION = version - if self.client.connection_pool.connection_kwargs.get("protocol") in ["3", 3]: + if get_protocol_version(self.client) in ["3", 3]: self._MODULE_CALLBACKS.update(_RESP3_MODULE_CALLBACKS) else: self._MODULE_CALLBACKS.update(_RESP2_MODULE_CALLBACKS) diff --git a/redis/commands/timeseries/__init__.py b/redis/commands/timeseries/__init__.py index 498f5118f1..4188b93d70 100644 --- a/redis/commands/timeseries/__init__.py +++ b/redis/commands/timeseries/__init__.py @@ -1,7 +1,7 @@ import redis from redis._parsers.helpers import bool_ok -from ..helpers import parse_to_list +from ..helpers import get_protocol_version, parse_to_list from .commands import ( ALTER_CMD, CREATE_CMD, @@ -56,7 +56,7 @@ def __init__(self, client=None, **kwargs): self.client = client self.execute_command = client.execute_command - if self.client.connection_pool.connection_kwargs.get("protocol") in ["3", 3]: + if get_protocol_version(self.client) in ["3", 3]: self._MODULE_CALLBACKS.update(_RESP3_MODULE_CALLBACKS) else: self._MODULE_CALLBACKS.update(_RESP2_MODULE_CALLBACKS) diff --git a/tests/test_asyncio/test_bloom.py b/tests/test_asyncio/test_bloom.py index 0535ddfe02..d0a25e5625 100644 --- a/tests/test_asyncio/test_bloom.py +++ b/tests/test_asyncio/test_bloom.py @@ -365,7 +365,7 @@ async def test_tdigest_reset(decoded_r: redis.Redis): @pytest.mark.redismod -@pytest.mark.experimental +@pytest.mark.onlynoncluster async def test_tdigest_merge(decoded_r: redis.Redis): assert await decoded_r.tdigest().create("to-tDigest", 10) assert await decoded_r.tdigest().create("from-tDigest", 10) diff --git a/tests/test_asyncio/test_json.py b/tests/test_asyncio/test_json.py index 6f3e8c3251..ed651cd903 100644 --- a/tests/test_asyncio/test_json.py +++ b/tests/test_asyncio/test_json.py @@ -112,6 +112,7 @@ async def test_mgetshouldsucceed(decoded_r: redis.Redis): @pytest.mark.redismod +@pytest.mark.onlynoncluster @skip_ifmodversion_lt("2.6.0", "ReJSON") async def test_mset(decoded_r: redis.Redis): await decoded_r.json().mset( diff --git a/tests/test_bloom.py b/tests/test_bloom.py index a82fece470..464a946f54 100644 --- a/tests/test_bloom.py +++ b/tests/test_bloom.py @@ -24,7 +24,6 @@ def client(decoded_r): return decoded_r -@pytest.mark.redismod def test_create(client): """Test CREATE/RESERVE calls""" assert client.bf().create("bloom", 0.01, 1000) @@ -39,7 +38,6 @@ def test_create(client): assert client.topk().reserve("topk", 5, 100, 5, 0.9) -@pytest.mark.redismod def test_bf_reserve(client): """Testing BF.RESERVE""" assert client.bf().reserve("bloom", 0.01, 1000) @@ -54,13 +52,11 @@ def test_bf_reserve(client): assert client.topk().reserve("topk", 5, 100, 5, 0.9) -@pytest.mark.redismod @pytest.mark.experimental def test_tdigest_create(client): assert client.tdigest().create("tDigest", 100) -@pytest.mark.redismod def test_bf_add(client): assert client.bf().create("bloom", 0.01, 1000) assert 1 == client.bf().add("bloom", "foo") @@ -73,7 +69,6 @@ def test_bf_add(client): assert [1, 0] == intlist(client.bf().mexists("bloom", "foo", "noexist")) -@pytest.mark.redismod def test_bf_insert(client): assert client.bf().create("bloom", 0.01, 1000) assert [1] == intlist(client.bf().insert("bloom", ["foo"])) @@ -104,7 +99,6 @@ def test_bf_insert(client): ) -@pytest.mark.redismod def test_bf_scandump_and_loadchunk(client): # Store a filter client.bf().create("myBloom", "0.0001", "1000") @@ -156,7 +150,6 @@ def do_verify(): client.bf().create("myBloom", "0.0001", "10000000") -@pytest.mark.redismod def test_bf_info(client): expansion = 4 # Store a filter @@ -188,7 +181,6 @@ def test_bf_info(client): assert True -@pytest.mark.redismod def test_bf_card(client): # return 0 if the key does not exist assert client.bf().card("not_exist") == 0 @@ -203,7 +195,6 @@ def test_bf_card(client): client.bf().card("setKey") -@pytest.mark.redismod def test_cf_add_and_insert(client): assert client.cf().create("cuckoo", 1000) assert client.cf().add("cuckoo", "filter") @@ -229,7 +220,6 @@ def test_cf_add_and_insert(client): ) -@pytest.mark.redismod def test_cf_exists_and_del(client): assert client.cf().create("cuckoo", 1000) assert client.cf().add("cuckoo", "filter") @@ -242,7 +232,6 @@ def test_cf_exists_and_del(client): assert 0 == client.cf().count("cuckoo", "filter") -@pytest.mark.redismod def test_cms(client): assert client.cms().initbydim("dim", 1000, 5) assert client.cms().initbyprob("prob", 0.01, 0.01) @@ -258,7 +247,6 @@ def test_cms(client): assert 25 == info["count"] -@pytest.mark.redismod @pytest.mark.onlynoncluster def test_cms_merge(client): assert client.cms().initbydim("A", 1000, 5) @@ -276,7 +264,6 @@ def test_cms_merge(client): assert [16, 15, 21] == client.cms().query("C", "foo", "bar", "baz") -@pytest.mark.redismod def test_topk(client): # test list with empty buckets assert client.topk().reserve("topk", 3, 50, 4, 0.9) @@ -356,7 +343,6 @@ def test_topk(client): assert 0.9 == round(float(info["decay"]), 1) -@pytest.mark.redismod def test_topk_incrby(client): client.flushdb() assert client.topk().reserve("topk", 3, 10, 3, 1) @@ -370,7 +356,6 @@ def test_topk_incrby(client): ) -@pytest.mark.redismod @pytest.mark.experimental def test_tdigest_reset(client): assert client.tdigest().create("tDigest", 10) @@ -387,8 +372,7 @@ def test_tdigest_reset(client): ) -@pytest.mark.redismod -@pytest.mark.experimental +@pytest.mark.onlynoncluster def test_tdigest_merge(client): assert client.tdigest().create("to-tDigest", 10) assert client.tdigest().create("from-tDigest", 10) @@ -415,7 +399,6 @@ def test_tdigest_merge(client): assert 4.0 == client.tdigest().max("to-tDigest") -@pytest.mark.redismod @pytest.mark.experimental def test_tdigest_min_and_max(client): assert client.tdigest().create("tDigest", 100) @@ -426,7 +409,6 @@ def test_tdigest_min_and_max(client): assert 1 == client.tdigest().min("tDigest") -@pytest.mark.redismod @pytest.mark.experimental @skip_ifmodversion_lt("2.4.0", "bf") def test_tdigest_quantile(client): @@ -448,7 +430,6 @@ def test_tdigest_quantile(client): assert [3.0, 5.0] == client.tdigest().quantile("t-digest", 0.5, 0.8) -@pytest.mark.redismod @pytest.mark.experimental def test_tdigest_cdf(client): assert client.tdigest().create("tDigest", 100) @@ -460,7 +441,6 @@ def test_tdigest_cdf(client): assert [0.1, 0.9] == [round(x, 1) for x in res] -@pytest.mark.redismod @pytest.mark.experimental @skip_ifmodversion_lt("2.4.0", "bf") def test_tdigest_trimmed_mean(client): @@ -471,7 +451,6 @@ def test_tdigest_trimmed_mean(client): assert 4.5 == client.tdigest().trimmed_mean("tDigest", 0.4, 0.5) -@pytest.mark.redismod @pytest.mark.experimental def test_tdigest_rank(client): assert client.tdigest().create("t-digest", 500) @@ -482,7 +461,6 @@ def test_tdigest_rank(client): assert [-1, 20, 9] == client.tdigest().rank("t-digest", -20, 20, 9) -@pytest.mark.redismod @pytest.mark.experimental def test_tdigest_revrank(client): assert client.tdigest().create("t-digest", 500) @@ -492,7 +470,6 @@ def test_tdigest_revrank(client): assert [-1, 19, 9] == client.tdigest().revrank("t-digest", 21, 0, 10) -@pytest.mark.redismod @pytest.mark.experimental def test_tdigest_byrank(client): assert client.tdigest().create("t-digest", 500) @@ -504,7 +481,6 @@ def test_tdigest_byrank(client): client.tdigest().byrank("t-digest", -1)[0] -@pytest.mark.redismod @pytest.mark.experimental def test_tdigest_byrevrank(client): assert client.tdigest().create("t-digest", 500) @@ -516,8 +492,7 @@ def test_tdigest_byrevrank(client): client.tdigest().byrevrank("t-digest", -1)[0] -# @pytest.mark.redismod -# def test_pipeline(client): +# # def test_pipeline(client): # pipeline = client.bf().pipeline() # assert not client.bf().execute_command("get pipeline") # diff --git a/tests/test_graph.py b/tests/test_graph.py index 42f1d9e5df..6fa9977d98 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -30,14 +30,12 @@ def client(request): return r -@pytest.mark.redismod def test_bulk(client): with pytest.raises(NotImplementedError): client.graph().bulk() client.graph().bulk(foo="bar!") -@pytest.mark.redismod def test_graph_creation(client): graph = client.graph() @@ -82,7 +80,6 @@ def test_graph_creation(client): graph.delete() -@pytest.mark.redismod def test_array_functions(client): query = """CREATE (p:person{name:'a',age:32, array:[0,1,2]})""" client.graph().query(query) @@ -103,7 +100,6 @@ def test_array_functions(client): assert [a] == result.result_set[0][0] -@pytest.mark.redismod def test_path(client): node0 = Node(node_id=0, label="L1") node1 = Node(node_id=1, label="L1") @@ -123,7 +119,6 @@ def test_path(client): assert expected_results == result.result_set -@pytest.mark.redismod def test_param(client): params = [1, 2.3, "str", True, False, None, [0, 1, 2], r"\" RETURN 1337 //"] query = "RETURN $param" @@ -133,7 +128,6 @@ def test_param(client): assert expected_results == result.result_set -@pytest.mark.redismod def test_map(client): query = "RETURN {a:1, b:'str', c:NULL, d:[1,2,3], e:True, f:{x:1, y:2}}" @@ -150,7 +144,6 @@ def test_map(client): assert actual == expected -@pytest.mark.redismod def test_point(client): query = "RETURN point({latitude: 32.070794860, longitude: 34.820751118})" expected_lat = 32.070794860 @@ -167,7 +160,6 @@ def test_point(client): assert abs(actual["longitude"] - expected_lon) < 0.001 -@pytest.mark.redismod def test_index_response(client): result_set = client.graph().query("CREATE INDEX ON :person(age)") assert 1 == result_set.indices_created @@ -182,7 +174,6 @@ def test_index_response(client): client.graph().query("DROP INDEX ON :person(age)") -@pytest.mark.redismod def test_stringify_query_result(client): graph = client.graph() @@ -236,7 +227,6 @@ def test_stringify_query_result(client): graph.delete() -@pytest.mark.redismod def test_optional_match(client): # Build a graph of form (a)-[R]->(b) node0 = Node(node_id=0, label="L1", properties={"value": "a"}) @@ -261,7 +251,6 @@ def test_optional_match(client): graph.delete() -@pytest.mark.redismod def test_cached_execution(client): client.graph().query("CREATE ()") @@ -279,7 +268,6 @@ def test_cached_execution(client): assert cached_result.cached_execution -@pytest.mark.redismod def test_slowlog(client): create_query = """CREATE (:Rider {name:'Valentino Rossi'})-[:rides]->(:Team {name:'Yamaha'}), @@ -292,7 +280,6 @@ def test_slowlog(client): assert results[0][2] == create_query -@pytest.mark.redismod @pytest.mark.xfail(strict=False) def test_query_timeout(client): # Build a sample graph with 1000 nodes. @@ -307,7 +294,6 @@ def test_query_timeout(client): assert False is False -@pytest.mark.redismod def test_read_only_query(client): with pytest.raises(Exception): # Issue a write query, specifying read-only true, @@ -316,7 +302,6 @@ def test_read_only_query(client): assert False is False -@pytest.mark.redismod def test_profile(client): q = """UNWIND range(1, 3) AS x CREATE (p:Person {v:x})""" profile = client.graph().profile(q).result_set @@ -331,7 +316,6 @@ def test_profile(client): assert "Node By Label Scan | (p:Person) | Records produced: 3" in profile -@pytest.mark.redismod @skip_if_redis_enterprise() def test_config(client): config_name = "RESULTSET_SIZE" @@ -363,7 +347,6 @@ def test_config(client): client.graph().config("RESULTSET_SIZE", -100, set=True) -@pytest.mark.redismod @pytest.mark.onlynoncluster def test_list_keys(client): result = client.graph().list_keys() @@ -387,7 +370,6 @@ def test_list_keys(client): assert result == [] -@pytest.mark.redismod def test_multi_label(client): redis_graph = client.graph("g") @@ -413,7 +395,6 @@ def test_multi_label(client): assert True -@pytest.mark.redismod def test_cache_sync(client): pass return @@ -486,7 +467,6 @@ def test_cache_sync(client): assert A._relationship_types[1] == "R" -@pytest.mark.redismod def test_execution_plan(client): redis_graph = client.graph("execution_plan") create_query = """CREATE @@ -505,7 +485,6 @@ def test_execution_plan(client): redis_graph.delete() -@pytest.mark.redismod def test_explain(client): redis_graph = client.graph("execution_plan") # graph creation / population @@ -594,7 +573,6 @@ def test_explain(client): redis_graph.delete() -@pytest.mark.redismod def test_resultset_statistics(client): with patch.object(target=QueryResult, attribute="_get_stat") as mock_get_stats: result = client.graph().query("RETURN 1") diff --git a/tests/test_json.py b/tests/test_json.py index fb608ff425..be347f6677 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -14,7 +14,6 @@ def client(request): return r -@pytest.mark.redismod def test_json_setbinarykey(client): d = {"hello": "world", b"some": "value"} with pytest.raises(TypeError): @@ -22,7 +21,6 @@ def test_json_setbinarykey(client): assert client.json().set("somekey", Path.root_path(), d, decode_keys=True) -@pytest.mark.redismod def test_json_setgetdeleteforget(client): assert client.json().set("foo", Path.root_path(), "bar") assert_resp_response(client, client.json().get("foo"), "bar", [["bar"]]) @@ -32,13 +30,11 @@ def test_json_setgetdeleteforget(client): assert client.exists("foo") == 0 -@pytest.mark.redismod def test_jsonget(client): client.json().set("foo", Path.root_path(), "bar") assert_resp_response(client, client.json().get("foo"), "bar", [["bar"]]) -@pytest.mark.redismod def test_json_get_jset(client): assert client.json().set("foo", Path.root_path(), "bar") assert_resp_response(client, client.json().get("foo"), "bar", [["bar"]]) @@ -47,8 +43,7 @@ def test_json_get_jset(client): assert client.exists("foo") == 0 -@pytest.mark.redismod -@skip_ifmodversion_lt("2.6.0", "ReJSON") # todo: update after the release +@skip_ifmodversion_lt("2.06.00", "ReJSON") # todo: update after the release def test_json_merge(client): # Test with root path $ assert client.json().set( @@ -80,7 +75,6 @@ def test_json_merge(client): } -@pytest.mark.redismod def test_nonascii_setgetdelete(client): assert client.json().set("notascii", Path.root_path(), "hyvää-élève") res = "hyvää-élève" @@ -91,7 +85,6 @@ def test_nonascii_setgetdelete(client): assert client.exists("notascii") == 0 -@pytest.mark.redismod def test_jsonsetexistentialmodifiersshouldsucceed(client): obj = {"foo": "bar"} assert client.json().set("obj", Path.root_path(), obj) @@ -109,7 +102,6 @@ def test_jsonsetexistentialmodifiersshouldsucceed(client): client.json().set("obj", Path("foo"), "baz", nx=True, xx=True) -@pytest.mark.redismod def test_mgetshouldsucceed(client): client.json().set("1", Path.root_path(), 1) client.json().set("2", Path.root_path(), 2) @@ -118,8 +110,8 @@ def test_mgetshouldsucceed(client): assert client.json().mget([1, 2], Path.root_path()) == [1, 2] -@pytest.mark.redismod -@skip_ifmodversion_lt("2.6.0", "ReJSON") # todo: update after the release +@pytest.mark.onlynoncluster +@skip_ifmodversion_lt("2.06.00", "ReJSON") def test_mset(client): client.json().mset([("1", Path.root_path(), 1), ("2", Path.root_path(), 2)]) @@ -127,7 +119,6 @@ def test_mset(client): assert client.json().mget(["1", "2"], Path.root_path()) == [1, 2] -@pytest.mark.redismod @skip_ifmodversion_lt("99.99.99", "ReJSON") # todo: update after the release def test_clear(client): client.json().set("arr", Path.root_path(), [0, 1, 2, 3, 4]) @@ -135,7 +126,6 @@ def test_clear(client): assert_resp_response(client, client.json().get("arr"), [], [[[]]]) -@pytest.mark.redismod def test_type(client): client.json().set("1", Path.root_path(), 1) assert_resp_response( @@ -144,7 +134,6 @@ def test_type(client): assert_resp_response(client, client.json().type("1"), "integer", ["integer"]) -@pytest.mark.redismod def test_numincrby(client): client.json().set("num", Path.root_path(), 1) assert_resp_response( @@ -158,7 +147,6 @@ def test_numincrby(client): ) -@pytest.mark.redismod def test_nummultby(client): client.json().set("num", Path.root_path(), 1) @@ -174,7 +162,6 @@ def test_nummultby(client): ) -@pytest.mark.redismod @skip_ifmodversion_lt("99.99.99", "ReJSON") # todo: update after the release def test_toggle(client): client.json().set("bool", Path.root_path(), False) @@ -186,7 +173,6 @@ def test_toggle(client): client.json().toggle("num", Path.root_path()) -@pytest.mark.redismod def test_strappend(client): client.json().set("jsonkey", Path.root_path(), "foo") assert 6 == client.json().strappend("jsonkey", "bar") @@ -195,8 +181,7 @@ def test_strappend(client): ) -# @pytest.mark.redismod -# def test_debug(client): +# # def test_debug(client): # client.json().set("str", Path.root_path(), "foo") # assert 24 == client.json().debug("MEMORY", "str", Path.root_path()) # assert 24 == client.json().debug("MEMORY", "str") @@ -205,7 +190,6 @@ def test_strappend(client): # assert isinstance(client.json().debug("HELP"), list) -@pytest.mark.redismod def test_strlen(client): client.json().set("str", Path.root_path(), "foo") assert 3 == client.json().strlen("str", Path.root_path()) @@ -214,7 +198,6 @@ def test_strlen(client): assert 6 == client.json().strlen("str") -@pytest.mark.redismod def test_arrappend(client): client.json().set("arr", Path.root_path(), [1]) assert 2 == client.json().arrappend("arr", Path.root_path(), 2) @@ -222,7 +205,6 @@ def test_arrappend(client): assert 7 == client.json().arrappend("arr", Path.root_path(), *[5, 6, 7]) -@pytest.mark.redismod def test_arrindex(client): client.json().set("arr", Path.root_path(), [0, 1, 2, 3, 4]) assert 1 == client.json().arrindex("arr", Path.root_path(), 1) @@ -234,7 +216,6 @@ def test_arrindex(client): assert -1 == client.json().arrindex("arr", Path.root_path(), 4, start=1, stop=3) -@pytest.mark.redismod def test_arrinsert(client): client.json().set("arr", Path.root_path(), [0, 4]) assert 5 - -client.json().arrinsert("arr", Path.root_path(), 1, *[1, 2, 3]) @@ -248,7 +229,6 @@ def test_arrinsert(client): assert_resp_response(client, client.json().get("val2"), res, [[res]]) -@pytest.mark.redismod def test_arrlen(client): client.json().set("arr", Path.root_path(), [0, 1, 2, 3, 4]) assert 5 == client.json().arrlen("arr", Path.root_path()) @@ -256,7 +236,6 @@ def test_arrlen(client): assert client.json().arrlen("fakekey") is None -@pytest.mark.redismod def test_arrpop(client): client.json().set("arr", Path.root_path(), [0, 1, 2, 3, 4]) assert 4 == client.json().arrpop("arr", Path.root_path(), 4) @@ -274,7 +253,6 @@ def test_arrpop(client): assert client.json().arrpop("arr") is None -@pytest.mark.redismod def test_arrtrim(client): client.json().set("arr", Path.root_path(), [0, 1, 2, 3, 4]) assert 3 == client.json().arrtrim("arr", Path.root_path(), 1, 3) @@ -297,7 +275,6 @@ def test_arrtrim(client): assert 0 == client.json().arrtrim("arr", Path.root_path(), 9, 11) -@pytest.mark.redismod def test_resp(client): obj = {"foo": "bar", "baz": 1, "qaz": True} client.json().set("obj", Path.root_path(), obj) @@ -307,7 +284,6 @@ def test_resp(client): assert isinstance(client.json().resp("obj"), list) -@pytest.mark.redismod def test_objkeys(client): obj = {"foo": "bar", "baz": "qaz"} client.json().set("obj", Path.root_path(), obj) @@ -324,7 +300,6 @@ def test_objkeys(client): assert client.json().objkeys("fakekey") is None -@pytest.mark.redismod def test_objlen(client): obj = {"foo": "bar", "baz": "qaz"} client.json().set("obj", Path.root_path(), obj) @@ -334,7 +309,6 @@ def test_objlen(client): assert len(obj) == client.json().objlen("obj") -@pytest.mark.redismod def test_json_commands_in_pipeline(client): p = client.json().pipeline() p.set("foo", Path.root_path(), "bar") @@ -358,7 +332,6 @@ def test_json_commands_in_pipeline(client): assert client.get("foo") is None -@pytest.mark.redismod def test_json_delete_with_dollar(client): doc1 = {"a": 1, "nested": {"a": 2, "b": 3}} assert client.json().set("doc1", "$", doc1) @@ -410,7 +383,6 @@ def test_json_delete_with_dollar(client): client.json().delete("not_a_document", "..a") -@pytest.mark.redismod def test_json_forget_with_dollar(client): doc1 = {"a": 1, "nested": {"a": 2, "b": 3}} assert client.json().set("doc1", "$", doc1) @@ -462,7 +434,6 @@ def test_json_forget_with_dollar(client): client.json().forget("not_a_document", "..a") -@pytest.mark.redismod def test_json_mget_dollar(client): # Test mget with multi paths client.json().set( @@ -492,7 +463,6 @@ def test_json_mget_dollar(client): assert res == [None, None] -@pytest.mark.redismod def test_numby_commands_dollar(client): # Test NUMINCRBY @@ -537,7 +507,6 @@ def test_numby_commands_dollar(client): client.json().nummultby("doc1", ".b[0].a", 3) == 6 -@pytest.mark.redismod def test_strappend_dollar(client): client.json().set( @@ -569,7 +538,6 @@ def test_strappend_dollar(client): client.json().strappend("doc1", "piu") -@pytest.mark.redismod def test_strlen_dollar(client): # Test multi @@ -591,7 +559,6 @@ def test_strlen_dollar(client): client.json().strlen("non_existing_doc", "$..a") -@pytest.mark.redismod def test_arrappend_dollar(client): client.json().set( "doc1", @@ -666,7 +633,6 @@ def test_arrappend_dollar(client): client.json().arrappend("non_existing_doc", "$..a") -@pytest.mark.redismod def test_arrinsert_dollar(client): client.json().set( "doc1", @@ -705,7 +671,6 @@ def test_arrinsert_dollar(client): client.json().arrappend("non_existing_doc", "$..a") -@pytest.mark.redismod def test_arrlen_dollar(client): client.json().set( @@ -755,7 +720,6 @@ def test_arrlen_dollar(client): assert client.json().arrlen("non_existing_doc", "..a") is None -@pytest.mark.redismod def test_arrpop_dollar(client): client.json().set( "doc1", @@ -797,7 +761,6 @@ def test_arrpop_dollar(client): client.json().arrpop("non_existing_doc", "..a") -@pytest.mark.redismod def test_arrtrim_dollar(client): client.json().set( @@ -851,7 +814,6 @@ def test_arrtrim_dollar(client): client.json().arrtrim("non_existing_doc", "..a", 1, 1) -@pytest.mark.redismod def test_objkeys_dollar(client): client.json().set( "doc1", @@ -881,7 +843,6 @@ def test_objkeys_dollar(client): assert client.json().objkeys("doc1", "$..nowhere") == [] -@pytest.mark.redismod def test_objlen_dollar(client): client.json().set( "doc1", @@ -917,7 +878,6 @@ def test_objlen_dollar(client): client.json().objlen("doc1", ".nowhere") -@pytest.mark.redismod def load_types_data(nested_key_name): td = { "object": {}, @@ -937,7 +897,6 @@ def load_types_data(nested_key_name): return jdata, types -@pytest.mark.redismod def test_type_dollar(client): jdata, jtypes = load_types_data("a") client.json().set("doc1", "$", jdata) @@ -955,7 +914,6 @@ def test_type_dollar(client): ) -@pytest.mark.redismod def test_clear_dollar(client): client.json().set( "doc1", @@ -1006,7 +964,6 @@ def test_clear_dollar(client): client.json().clear("non_existing_doc", "$..a") -@pytest.mark.redismod def test_toggle_dollar(client): client.json().set( "doc1", @@ -1035,8 +992,7 @@ def test_toggle_dollar(client): client.json().toggle("non_existing_doc", "$..a") -# @pytest.mark.redismod -# def test_debug_dollar(client): +# # def test_debug_dollar(client): # # jdata, jtypes = load_types_data("a") # @@ -1058,7 +1014,6 @@ def test_toggle_dollar(client): # assert client.json().debug("MEMORY", "non_existing_doc", "$..a") == [] -@pytest.mark.redismod def test_resp_dollar(client): data = { @@ -1288,7 +1243,6 @@ def test_resp_dollar(client): client.json().resp("non_existing_doc", "$..a") -@pytest.mark.redismod def test_arrindex_dollar(client): client.json().set( @@ -1515,7 +1469,6 @@ def test_arrindex_dollar(client): assert client.json().arrindex("test_None", "..nested2_not_found.arr", "None") == 0 -@pytest.mark.redismod def test_decoders_and_unstring(): assert unstring("4") == 4 assert unstring("45.55") == 45.55 @@ -1526,7 +1479,6 @@ def test_decoders_and_unstring(): assert decode_list(["hello", b"world"]) == ["hello", "world"] -@pytest.mark.redismod def test_custom_decoder(client): import json @@ -1542,7 +1494,6 @@ def test_custom_decoder(client): assert not isinstance(cj.__decoder__, json.JSONDecoder) -@pytest.mark.redismod def test_set_file(client): import json import tempfile @@ -1561,7 +1512,6 @@ def test_set_file(client): client.json().set_file("test2", Path.root_path(), nojsonfile.name) -@pytest.mark.redismod def test_set_path(client): import json import tempfile diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py index 80490af4ef..4ab86cd56e 100644 --- a/tests/test_timeseries.py +++ b/tests/test_timeseries.py @@ -14,7 +14,6 @@ def client(decoded_r): return decoded_r -@pytest.mark.redismod def test_create(client): assert client.ts().create(1) assert client.ts().create(2, retention_msecs=5) @@ -32,7 +31,6 @@ def test_create(client): assert_resp_response(client, 128, info.get("chunk_size"), info.get("chunkSize")) -@pytest.mark.redismod @skip_ifmodversion_lt("1.4.0", "timeseries") def test_create_duplicate_policy(client): # Test for duplicate policy @@ -48,7 +46,6 @@ def test_create_duplicate_policy(client): ) -@pytest.mark.redismod def test_alter(client): assert client.ts().create(1) info = client.ts().info(1) @@ -69,7 +66,6 @@ def test_alter(client): ) -@pytest.mark.redismod @skip_ifmodversion_lt("1.4.0", "timeseries") def test_alter_diplicate_policy(client): assert client.ts().create(1) @@ -84,7 +80,6 @@ def test_alter_diplicate_policy(client): ) -@pytest.mark.redismod def test_add(client): assert 1 == client.ts().add(1, 1, 1) assert 2 == client.ts().add(2, 2, 3, retention_msecs=10) @@ -107,7 +102,6 @@ def test_add(client): assert_resp_response(client, 128, info.get("chunk_size"), info.get("chunkSize")) -@pytest.mark.redismod @skip_ifmodversion_lt("1.4.0", "timeseries") def test_add_duplicate_policy(client): @@ -145,13 +139,11 @@ def test_add_duplicate_policy(client): assert 5.0 == client.ts().get("time-serie-add-ooo-min")[1] -@pytest.mark.redismod def test_madd(client): client.ts().create("a") assert [1, 2, 3] == client.ts().madd([("a", 1, 5), ("a", 2, 10), ("a", 3, 15)]) -@pytest.mark.redismod def test_incrby_decrby(client): for _ in range(100): assert client.ts().incrby(1, 1) @@ -180,7 +172,6 @@ def test_incrby_decrby(client): assert_resp_response(client, 128, info.get("chunk_size"), info.get("chunkSize")) -@pytest.mark.redismod def test_create_and_delete_rule(client): # test rule creation time = 100 @@ -204,7 +195,6 @@ def test_create_and_delete_rule(client): assert not info["rules"] -@pytest.mark.redismod @skip_ifmodversion_lt("99.99.99", "timeseries") def test_del_range(client): try: @@ -219,7 +209,6 @@ def test_del_range(client): assert_resp_response(client, client.ts().range(1, 22, 22), [(22, 1.0)], [[22, 1.0]]) -@pytest.mark.redismod def test_range(client): for i in range(100): client.ts().add(1, i, i % 7) @@ -234,7 +223,6 @@ def test_range(client): assert 10 == len(client.ts().range(1, 0, 500, count=10)) -@pytest.mark.redismod @skip_ifmodversion_lt("99.99.99", "timeseries") def test_range_advanced(client): for i in range(100): @@ -263,7 +251,7 @@ def test_range_advanced(client): assert_resp_response(client, res, [(0, 2.55), (10, 3.0)], [[0, 2.55], [10, 3.0]]) -@pytest.mark.redismod +@pytest.mark.onlynoncluster @skip_ifmodversion_lt("1.8.0", "timeseries") def test_range_latest(client: redis.Redis): timeseries = client.ts() @@ -288,7 +276,6 @@ def test_range_latest(client: redis.Redis): ) -@pytest.mark.redismod @skip_ifmodversion_lt("1.8.0", "timeseries") def test_range_bucket_timestamp(client: redis.Redis): timeseries = client.ts() @@ -322,7 +309,6 @@ def test_range_bucket_timestamp(client: redis.Redis): ) -@pytest.mark.redismod @skip_ifmodversion_lt("1.8.0", "timeseries") def test_range_empty(client: redis.Redis): timeseries = client.ts() @@ -367,7 +353,6 @@ def test_range_empty(client: redis.Redis): assert_resp_response(client, res, resp2_expected, resp3_expected) -@pytest.mark.redismod @skip_ifmodversion_lt("99.99.99", "timeseries") def test_rev_range(client): for i in range(100): @@ -415,7 +400,7 @@ def test_rev_range(client): ) -@pytest.mark.redismod +@pytest.mark.onlynoncluster @skip_ifmodversion_lt("1.8.0", "timeseries") def test_revrange_latest(client: redis.Redis): timeseries = client.ts() @@ -434,7 +419,6 @@ def test_revrange_latest(client: redis.Redis): assert_resp_response(client, res, [(0, 4.0)], [[0, 4.0]]) -@pytest.mark.redismod @skip_ifmodversion_lt("1.8.0", "timeseries") def test_revrange_bucket_timestamp(client: redis.Redis): timeseries = client.ts() @@ -468,7 +452,6 @@ def test_revrange_bucket_timestamp(client: redis.Redis): ) -@pytest.mark.redismod @skip_ifmodversion_lt("1.8.0", "timeseries") def test_revrange_empty(client: redis.Redis): timeseries = client.ts() @@ -513,7 +496,6 @@ def test_revrange_empty(client: redis.Redis): assert_resp_response(client, res, resp2_expected, resp3_expected) -@pytest.mark.redismod @pytest.mark.onlynoncluster def test_mrange(client): client.ts().create(1, labels={"Test": "This", "team": "ny"}) @@ -562,7 +544,6 @@ def test_mrange(client): assert {"Test": "This", "team": "ny"} == res["1"][0] -@pytest.mark.redismod @pytest.mark.onlynoncluster @skip_ifmodversion_lt("99.99.99", "timeseries") def test_multi_range_advanced(client): @@ -676,7 +657,6 @@ def test_multi_range_advanced(client): assert [[0, 5.0], [5, 6.0]] == res["1"][2] -@pytest.mark.redismod @pytest.mark.onlynoncluster @skip_ifmodversion_lt("1.8.0", "timeseries") def test_mrange_latest(client: redis.Redis): @@ -706,7 +686,6 @@ def test_mrange_latest(client: redis.Redis): ) -@pytest.mark.redismod @pytest.mark.onlynoncluster @skip_ifmodversion_lt("99.99.99", "timeseries") def test_multi_reverse_range(client): @@ -825,7 +804,6 @@ def test_multi_reverse_range(client): assert [[1, 10.0], [0, 1.0]] == res["1"][2] -@pytest.mark.redismod @pytest.mark.onlynoncluster @skip_ifmodversion_lt("1.8.0", "timeseries") def test_mrevrange_latest(client: redis.Redis): @@ -855,7 +833,6 @@ def test_mrevrange_latest(client: redis.Redis): ) -@pytest.mark.redismod def test_get(client): name = "test" client.ts().create(name) @@ -866,7 +843,7 @@ def test_get(client): assert 4 == client.ts().get(name)[1] -@pytest.mark.redismod +@pytest.mark.onlynoncluster @skip_ifmodversion_lt("1.8.0", "timeseries") def test_get_latest(client: redis.Redis): timeseries = client.ts() @@ -883,7 +860,6 @@ def test_get_latest(client: redis.Redis): ) -@pytest.mark.redismod @pytest.mark.onlynoncluster def test_mget(client): client.ts().create(1, labels={"Test": "This"}) @@ -919,7 +895,6 @@ def test_mget(client): assert {"Taste": "That", "Test": "This"} == res["2"][0] -@pytest.mark.redismod @pytest.mark.onlynoncluster @skip_ifmodversion_lt("1.8.0", "timeseries") def test_mget_latest(client: redis.Redis): @@ -937,7 +912,6 @@ def test_mget_latest(client: redis.Redis): assert_resp_response(client, res, [{"t2": [{}, 10, 8.0]}], {"t2": [{}, [10, 8.0]]}) -@pytest.mark.redismod def test_info(client): client.ts().create(1, retention_msecs=5, labels={"currentLabel": "currentData"}) info = client.ts().info(1) @@ -947,7 +921,6 @@ def test_info(client): assert info["labels"]["currentLabel"] == "currentData" -@pytest.mark.redismod @skip_ifmodversion_lt("1.4.0", "timeseries") def testInfoDuplicatePolicy(client): client.ts().create(1, retention_msecs=5, labels={"currentLabel": "currentData"}) @@ -963,7 +936,6 @@ def testInfoDuplicatePolicy(client): ) -@pytest.mark.redismod @pytest.mark.onlynoncluster def test_query_index(client): client.ts().create(1, labels={"Test": "This"}) @@ -973,7 +945,6 @@ def test_query_index(client): assert_resp_response(client, client.ts().queryindex(["Taste=That"]), [2], {"2"}) -@pytest.mark.redismod def test_pipeline(client): pipeline = client.ts().pipeline() pipeline.create("with_pipeline") @@ -991,7 +962,6 @@ def test_pipeline(client): assert client.ts().get("with_pipeline")[1] == 99 * 1.1 -@pytest.mark.redismod def test_uncompressed(client): client.ts().create("compressed") client.ts().create("uncompressed", uncompressed=True) From da27f4bfb70faaf2c0f2e0d60a7b6d6d76740029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juraj=20P=C3=A1ll?= Date: Thu, 3 Aug 2023 02:38:56 +0200 Subject: [PATCH 6/6] Fix timeout retrying on Redis pipeline execution (#2812) Achieved by modifying Pipeline._disconnect_raise_reset Co-authored-by: dvora-h <67596500+dvora-h@users.noreply.github.com> --- redis/client.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/redis/client.py b/redis/client.py index 66e2c7b84f..a856ef84ad 100755 --- a/redis/client.py +++ b/redis/client.py @@ -1379,7 +1379,7 @@ def load_scripts(self): def _disconnect_raise_reset(self, conn, error): """ Close the connection, raise an exception if we were watching, - and raise an exception if retry_on_timeout is not set, + and raise an exception if TimeoutError is not part of retry_on_error, or the error is not a TimeoutError """ conn.disconnect() @@ -1390,11 +1390,13 @@ def _disconnect_raise_reset(self, conn, error): raise WatchError( "A ConnectionError occurred on while watching one or more keys" ) - # if retry_on_timeout is not set, or the error is not - # a TimeoutError, raise it - if not (conn.retry_on_timeout and isinstance(error, TimeoutError)): + # if TimeoutError is not part of retry_on_error, or the error + # is not a TimeoutError, raise it + if not ( + TimeoutError in conn.retry_on_error and isinstance(error, TimeoutError) + ): self.reset() - raise + raise error def execute(self, raise_on_error=True): """Execute all the commands in the current pipeline"""