Add support for ExceptionGroups - PEP-0654

[MaxwellDPS]

Hey Y'all!

With the introduction of ExceptionGroups PEP-0654 It would be awesome to be able to catch ExceptionGroups in a context like below.

class CeleryTaskError(Task):
    """
    Extended Task base class.
    """
    def on_failure(self, exc, task_id, args, kwargs, einfo):
        """
        Error handler.
        This is run by the worker when the task fails.
        Arguments:
            exc (Exception): The exception raised by the task.
            task_id (str): Unique id of the failed task.
            args (Tuple): Original arguments for the task that failed.
            kwargs (Dict): Original keyword arguments for the task that failed.
            einfo (~billiard.einfo.ExceptionInfo): Exception information.
        Returns:
            None: The return value of this handler is ignored.
        """
        panic(task_id, einfo)

@shared_task(base=CeleryTaskError)
def im_a_failure() -> None:
    """
    The Celery task that breaks
    """
    excs = []

    try:
        math = 0 / 0 
    except Exception as err:
        excs.append(err)

    try:
        im_not_real()
    except Exception as err:
        excs.append(err)

    exc = ExceptionGroup(
        "THIS WILL FAIL",
        excs
    )

    exc.exceptions

    raise exc

        

Trace

[2023-03-13 16:38:48,268: INFO/MainProcess] Task that_app.tasks.im_a_failure[59451401-0d5f-437f-b5f8-6ee274fa8064] received
[2023-03-13 16:38:48,298: WARNING/MainProcess] ----------!!!!!!!!!!!!!!--------------------
[2023-03-13 16:38:48,299: WARNING/MainProcess] --- Logging error ---
[2023-03-13 16:38:48,301: WARNING/MainProcess]   + Exception Group Traceback (most recent call last):
[2023-03-13 16:38:48,301: WARNING/MainProcess]   |   File "/usr/local/lib/python3.11/site-packages/celery/app/trace.py", line 451, in trace_task
                                                 |     R = retval = fun(*args, **kwargs)
                                                 |                  ^^^^^^^^^^^^^^^^^^^^
[2023-03-13 16:38:48,301: WARNING/MainProcess]   |   File "/usr/local/lib/python3.11/site-packages/celery/app/trace.py", line 734, in __protected_call__
                                                 |     return self.run(*args, **kwargs)
                                                 |            ^^^^^^^^^^^^^^^^^^^^^^^^^
[2023-03-13 16:38:48,301: WARNING/MainProcess]   |   File "/opt/yee/that_app/tasks.py", line 84, in im_a_failure
  |     raise exc
[2023-03-13 16:38:48,301: WARNING/MainProcess]   | ExceptionGroup: THIS WILL FAIL (2 sub-exceptions)
[2023-03-13 16:38:48,301: WARNING/MainProcess]   +-+---------------- 1 ----------------
[2023-03-13 16:38:48,301: WARNING/MainProcess]     | Traceback (most recent call last):
[2023-03-13 16:38:48,301: WARNING/MainProcess]     |   File "/opt/yee/that_app/tasks.py", line 68, in im_a_failure
                                                   |     math = 0 / 0
                                                   |           ~~^~~
[2023-03-13 16:38:48,301: WARNING/MainProcess]     | ZeroDivisionError: division by zero
[2023-03-13 16:38:48,301: WARNING/MainProcess]     +---------------- 2 ----------------
[2023-03-13 16:38:48,301: WARNING/MainProcess]     | Traceback (most recent call last):
[2023-03-13 16:38:48,301: WARNING/MainProcess]     |   File "/opt/yee/that_app/tasks.py", line 73, in im_a_failure
                                                   |     im_not_real()
                                                   |     ^^^^^^^^^^^
[2023-03-13 16:38:48,301: WARNING/MainProcess]     | NameError: name 'im_not_real' is not defined
[2023-03-13 16:38:48,301: WARNING/MainProcess]     +------------------------------------
[2023-03-13 16:38:48,301: WARNING/MainProcess] During handling of the above exception, another exception occurred:
[2023-03-13 16:38:48,301: WARNING/MainProcess] Traceback (most recent call last):
[2023-03-13 16:38:48,302: WARNING/MainProcess]   File "/usr/local/lib/python3.11/logging/__init__.py", line 1110, in emit
    msg = self.format(record)
          ^^^^^^^^^^^^^^^^^^^
[2023-03-13 16:38:48,302: WARNING/MainProcess]   File "/usr/local/lib/python3.11/logging/__init__.py", line 953, in format
    return fmt.format(record)
           ^^^^^^^^^^^^^^^^^^
[2023-03-13 16:38:48,302: WARNING/MainProcess]   File "/usr/local/lib/python3.11/site-packages/celery/utils/log.py", line 146, in format
    msg = super().format(record)
          ^^^^^^^^^^^^^^^^^^^^^^
[2023-03-13 16:38:48,302: WARNING/MainProcess]   File "/usr/local/lib/python3.11/logging/__init__.py", line 695, in format
    record.exc_text = self.formatException(record.exc_info)
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[2023-03-13 16:38:48,302: WARNING/MainProcess]   File "/usr/local/lib/python3.11/site-packages/celery/utils/log.py", line 142, in formatException
    r = super().formatException(ei)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^
[2023-03-13 16:38:48,302: WARNING/MainProcess]   File "/usr/local/lib/python3.11/logging/__init__.py", line 645, in formatException
    traceback.print_exception(ei[0], ei[1], tb, None, sio)
[2023-03-13 16:38:48,302: WARNING/MainProcess]   File "/usr/local/lib/python3.11/traceback.py", line 124, in print_exception
    te = TracebackException(type(value), value, tb, limit=limit, compact=True)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[2023-03-13 16:38:48,302: WARNING/MainProcess]   File "/usr/local/lib/python3.11/traceback.py", line 690, in __init__
    self.stack = StackSummary._extract_from_extended_frame_gen(
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[2023-03-13 16:38:48,302: WARNING/MainProcess]   File "/usr/local/lib/python3.11/traceback.py", line 416, in _extract_from_extended_frame_gen
    for f, (lineno, end_lineno, colno, end_colno) in frame_gen:
[2023-03-13 16:38:48,302: WARNING/MainProcess]   File "/usr/local/lib/python3.11/traceback.py", line 353, in _walk_tb_with_full_positions
    positions = _get_code_position(tb.tb_frame.f_code, tb.tb_lasti)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[2023-03-13 16:38:48,302: WARNING/MainProcess]   File "/usr/local/lib/python3.11/traceback.py", line 366, in _get_code_position
    positions_gen = code.co_positions()
                    ^^^^^^^^^^^^^^^^^
[2023-03-13 16:38:48,302: WARNING/MainProcess] AttributeError: '_Code' object has no attribute 'co_positions'
[2023-03-13 16:38:48,302: WARNING/MainProcess] Call stack:
[2023-03-13 16:38:48,303: WARNING/MainProcess]   File "/usr/local/lib/python3.11/site-packages/celery/concurrency/base.py", line 29, in apply_target
    ret = target(*args, **kwargs)
[2023-03-13 16:38:48,303: WARNING/MainProcess]   File "/usr/local/lib/python3.11/site-packages/celery/app/trace.py", line 649, in fast_trace_task
    R, I, T, Rstr = tasks[task].__trace__(
[2023-03-13 16:38:48,303: WARNING/MainProcess]   File "/usr/local/lib/python3.11/site-packages/celery/app/trace.py", line 468, in trace_task
    I, R, state, retval = on_error(task_request, exc, uuid)
[2023-03-13 16:38:48,303: WARNING/MainProcess]   File "/usr/local/lib/python3.11/site-packages/celery/app/trace.py", line 379, in on_error
    R = I.handle_error_state(
[2023-03-13 16:38:48,303: WARNING/MainProcess]   File "/usr/local/lib/python3.11/site-packages/celery/app/trace.py", line 178, in handle_error_state
    return {
[2023-03-13 16:38:48,303: WARNING/MainProcess]   File "/usr/local/lib/python3.11/site-packages/celery/app/trace.py", line 237, in handle_failure
    self._log_error(task, req, einfo)
[2023-03-13 16:38:48,303: WARNING/MainProcess]   File "/usr/local/lib/python3.11/site-packages/celery/app/trace.py", line 265, in _log_error
    logger.log(policy.severity, policy.format.strip(), context,

[auvipy]

isn't it python 3.11 only? can this back ported?

[moaddib666]

Hi, I've faced another error with co_positions while using celery.
I use the following requirements versions which were latest available at the moment of installation:

billiard==4.1.0
celery==5.3.4
django-celery-results==2.5.1

Pre-condition CELERY_ALWAYS_EAGER=True

Task must fail, logging must have been enabled.

        logger.log(policy.severity, policy.format.strip(), context,
                   exc_info=exc_info if policy.traceback else None,
                   extra={'data': context}) <--- "'_Object' object has no attribute 'co_positions'"

This caused by absence co_positions attribute on billiard _Object class in the billiard/einfo.py

Debug Details:

My intention is to add default co_positions with ((None, None, None, None),) to the empty _Object.

The created the PR with fix which was tested it locally and with help of existing unittests.