Bubble status up (#893)

* bubble status and log gitignored files

* assign value to enum instead

* allow google count token to fail and set return direction to negative

* fix test

* Patched /home/runner/work/patchwork/patchwork/patchwork/steps/PRPB/README.md

* Patched /home/runner/work/patchwork/patchwork/patchwork/steps/SimplifiedLLMOncePB/README.md

* add auto retry

* remove slots

* fix test and bump code2prompt

* improve bubbling error in retrying scope

* add error logging for retries

---------

Co-authored-by: patched.codes[bot] <298395+patched.codes[bot]@users.noreply.github.com>
Co-authored-by: Asankhaya Sharma <[email protected]>

patchwork/common/client/llm/google.py

@@ -58,7 +58,10 @@ def is_model_supported(self, model: str) -> bool:
  def is_prompt_supported(self, messages: Iterable[ChatCompletionMessageParam], model: str) -> int:
  system, chat = self.__openai_messages_to_google_messages(messages)
  gen_model = generativeai.GenerativeModel(model_name=model, system_instruction=system)
- token_count = gen_model.count_tokens(chat).total_tokens
+ try:
+ token_count = gen_model.count_tokens(chat).total_tokens
+ except Exception as e:
+ return -1
  model_limit = self.__get_model_limits(model)
  return model_limit - token_count
 

patchwork/common/utils/utils.py

@@ -1,16 +1,18 @@
 from __future__ import annotations
 
 import atexit
+import dataclasses
 import signal
 import tempfile
 from pathlib import Path
 
 import tiktoken
 from chardet.universaldetector import UniversalDetector
 from git import Head, Repo
-from typing_extensions import Callable
+from typing_extensions import Any, Callable
 
 from patchwork.common.utils.dependency import chromadb
+from patchwork.logger import logger
 from patchwork.managed_files import HOME_FOLDER
 
 _CLEANUP_FILES: set[Path] = set()
@@ -192,3 +194,19 @@ def is_container() -> bool:
 
 def exclude_none_dict(d: dict) -> dict:
  return {k: v for k, v in d.items() if v is not None}
+
+
+@dataclasses.dataclass
+class RetryData:
+ retry_limit: int
+ retry_count: int
+
+
+def retry(callback: Callable[[RetryData], Any], retry_limit=3) -> Any:
+ for i in range(retry_limit):
+ try:
+ return callback(RetryData(retry_limit=retry_limit, retry_count=i))
+ except Exception as e:
+ logger.error(f"Retry {i + 1} failed with error: {e}")
+ if i == retry_limit - 1:
+ raise e

patchwork/step.py

@@ -1,23 +1,31 @@
 import abc
-from enum import Flag, auto
+from enum import Enum
 
-from typing_extensions import Any, Dict, List, Union, is_typeddict
+from typing_extensions import Any, Dict, List, Optional, Union, is_typeddict
 
 from patchwork.logger import logger
 
 DataPoint = Dict[str, Union[str, int, float, bool, "OneOrMore"]]
 OneOrMoreDataPoint = Union[DataPoint, List[DataPoint]]
 
 
-class StepStatus(Flag):
- COMPLETED = auto()
- FAILED = auto()
- SKIPPED = auto()
- WARNING = auto()
+class StepStatus(Enum):
+ COMPLETED = (1, logger.info)
+ SKIPPED = (2, logger.warning)
+ WARNING = (3, logger.warning)
+ FAILED = (4, logger.error)
+
+ def __init__(self, priority: int, logger_func):
+ self.priority = priority
+ self._logger = logger_func
 
  def __str__(self):
  return self.name.lower()
 
+ @classmethod
+ def values(cls):
+ return list(StepStatus.__members__.values())
+
 
 class Step(abc.ABC):
  def __init__(self, inputs: DataPoint):
@@ -66,33 +74,37 @@ def __managed_run(self, *args, **kwargs) -> Any:
  except Exception as e:
  exc = e
 
- is_fail = self.__status == StepStatus.FAILED or exc is not None
  if self.__status_msg is not None:
- message_logger = logger.error if is_fail else logger.info
- message_logger(f"Step {self.__step_name} message: {self.__status_msg}")
+ self.__status._logger(f"Step {self.__step_name} message: {self.__status_msg}")
 
  if exc is not None:
  logger.error(f"Step {self.__step_name} failed")
  raise exc
 
- if is_fail:
+ if self.__status == StepStatus.FAILED:
  raise ValueError(f"Step {self.__step_name} failed")
 
  logger.info(f"Run {self.__status} {self.__step_name}")
  return output
 
- def set_status(self, status: StepStatus, msg: str = None):
- if status not in StepStatus:
+ def set_status(self, status: StepStatus, msg: Optional[str] = None):
+ if status not in StepStatus.values():
  raise ValueError(f"Invalid status: {status}")
- self.__status = status
- self.__status_msg = msg
+
+ if status.priority >= self.__status.priority:
+ self.__status = status
+ self.__status_msg = msg
 
  @property
  def status(self) -> StepStatus:
  return self.__status
 
+ @property
+ def status_message(self) -> Optional[str]:
+ return self.__status_msg
+
  @abc.abstractmethod
- def run(self) -> OneOrMoreDataPoint:
+ def run(self) -> DataPoint:
  """
  Runs the step.
  :return: a dictionary of outputs

patchwork/steps/CommitChanges/CommitChanges.py

@@ -144,8 +144,10 @@ def __get_repo_tracked_modified_files(self, repo: Repo) -> set[Path]:
  repo_changed_files = set()
  for item in repo.index.diff(None):
  repo_changed_file = item.a_path
- if any(fnmatch.fnmatch(repo_changed_file, ignored_grok) for ignored_grok in ignored_groks):
- continue
+ for ignored_grok in ignored_groks:
+ if fnmatch.fnmatch(repo_changed_file, ignored_grok):
+ logger.warn(f'Ignoring file: {repo_changed_file} because of "{ignored_grok}" in .gitignore file.')
+ continue
  repo_changed_files.add(repo_dir_path / repo_changed_file)
 
  return repo_changed_files

patchwork/steps/CreatePR/CreatePR.py

@@ -83,6 +83,7 @@ def run(self) -> dict:
  )
  return dict()
 
+ self.set_status(StepStatus.WARNING, "PR creation is disabled. Skipping PR creation.")
  logger.warning(f"PR creation is disabled. Skipping PR creation.")
  return dict()
 

patchwork/steps/PR/PR.py

@@ -19,16 +19,24 @@ def __init__(self, inputs):
  return
 
  def run(self):
- commit_changes_output = CommitChanges(self.inputs).run()
- prepare_pr_output = PreparePR(self.inputs).run()
- create_pr_outputs = CreatePR(
+ commit_changes = CommitChanges(self.inputs)
+ commit_changes_output = commit_changes.run()
+ self.set_status(commit_changes.status, commit_changes.status_message)
+
+ prepare_pr = PreparePR(self.inputs)
+ prepare_pr_output = prepare_pr.run()
+ self.set_status(prepare_pr.status, prepare_pr.status_message)
+
+ create_pr = CreatePR(
  dict(
  base_branch=commit_changes_output.get("base_branch"),
  target_branch=commit_changes_output.get("target_branch"),
  pr_body=prepare_pr_output.get("pr_body"),
  **self.inputs,
  )
- ).run()
+ )
+ create_pr_outputs = create_pr.run()
+ self.set_status(create_pr.status, create_pr.status_message)
 
  return exclude_none_dict(
  dict(

patchwork/steps/PRPB/PRPB.py

@@ -31,5 +31,5 @@ def __init__(self, inputs):
  def run(self):
  pr = PR({**self.inputs, "modified_code_files": self.modified_files})
  pr_outputs = pr.run()
-
+ self.set_status(pr.status, pr.status_message)
  return pr_outputs

patchwork/steps/PRPB/README.md

@@ -0,0 +1,115 @@
+# PRPB Module Documentation
+
+## Overview
+
+The `PRPB` module in the `patchwork.steps` package is designed to handle preparation and creation of Pull Requests (PRs) by extending the capabilities of the `Step` class. It processes the inputs to manage file modifications and initiates the PR creation process. This module includes several components distributed across different Python files.
+
+## Files
+
+### `patchwork/steps/PRPB/PRPB.py`
+
+#### Contents
+
+The primary class in this file is `PRPB`, which manages the operation of processing modified files and initiating a Pull Request.
+
+#### Inputs
+
+- `inputs`: An instance of `PRPBInputs`, which is a detailed dictionary allowing various configurations required for the PR process. This includes paths, commit messages, branch configurations, and API keys for GitHub or GitLab.
+
+#### Outputs
+
+- The `run` method returns an instance of `PRPBOutputs`, which includes details such as base and target branches, PR body, and PR URL.
+
+#### Code Summary
+
+```python
+class PRPB(Step, input_class=PRPBInputs, output_class=PRPBOutputs):
+ def __init__(self, inputs):
+ super().__init__(inputs)
+ # Mapping input keys to internal keys
+ key_map = dict(path=inputs["path_key"])
+ if inputs.get("title_key") is not None:
+ key_map["commit_message"] = inputs["comment_title_key"]
+ if inputs.get("message_key") is not None:
+ key_map["patch_message"] = inputs["comment_message_key"]
+
+ # Processing the modified files input
+ self.modified_files = []
+ input_modified_files = inputs.get("modified_files")
+ if isinstance(input_modified_files, list):
+ for modified_file in input_modified_files:
+ converted_modified_file = {key: modified_file.get(mapped_key) for key, mapped_key in key_map.items()}
+ if converted_modified_file.get("path") is None:
+ continue
+ self.modified_files.append(converted_modified_file)
+ elif isinstance(input_modified_files, dict):
+ converted_modified_file = {key: input_modified_files.get(mapped_key) for key, mapped_key in key_map.items()}
+ self.modified_files.append(converted_modified_file)
+ elif isinstance(input_modified_files, str):
+ converted_modified_file = {"path": input_modified_files}
+ self.modified_files.append(converted_modified_file)
+ self.inputs = inputs
+
+ def run(self):
+ pr = PR({**self.inputs, "modified_code_files": self.modified_files})
+ pr_outputs = pr.run()
+ self.set_status(pr.status, pr.status_message)
+ return pr_outputs
+```
+
+### `patchwork/steps/PRPB/typed.py`
+
+#### Contents
+
+This file defines the structured types for the inputs and outputs used in the `PRPB` class.
+
+#### Inputs
+
+- `PRPBInputs`: This is a `TypedDict` that includes required fields such as `modified_files` and `path_key`, along with optional keys for commit messages, branch configurations, and SCM details.
+
+#### Outputs
+
+- `PRPBOutputs`: This `TypedDict` includes fields that capture the results of the PR operation, such as branch names, PR body, and PR URL.
+
+#### Code Summary
+
+```python
+class __PRPBInputsRequired(TypedDict):
+ modified_files: List[Dict]
+ path_key: str
+
+class PRPBInputs(__PRPBInputsRequired, total=False):
+ comment_title_key: str
+ comment_message_key: str
+ disable_branch: Annotated[bool, StepTypeConfig(is_config=True)]
+ force_branch_creation: Annotated[bool, StepTypeConfig(is_config=True)]
+ branch_prefix: Annotated[str, StepTypeConfig(is_config=True)]
+ branch_suffix: Annotated[str, StepTypeConfig(is_config=True)]
+ pr_header: Annotated[str, StepTypeConfig(is_config=True)]
+ pr_title: Annotated[str, StepTypeConfig(is_config=True)]
+ force_pr_creation: Annotated[bool, StepTypeConfig(is_config=True)]
+ disable_pr: Annotated[bool, StepTypeConfig(is_config=True)]
+ scm_url: Annotated[str, StepTypeConfig(is_config=True)]
+ gitlab_api_key: Annotated[str, StepTypeConfig(is_config=True)]
+ github_api_key: Annotated[str, StepTypeConfig(is_config=True)]
+
+class PRPBOutputs(TypedDict):
+ base_branch: str
+ target_branch: str
+ pr_body: str
+ pr_url: str
+```
+
+### `patchwork/steps/PRPB/__init__.py`
+
+#### Contents
+
+This file is currently an empty initializer for the `PRPB` module, ensuring that the parent directory is treated as a package.
+
+#### Code Summary
+
+```python
+
+```
+
+This documentation provides a comprehensive overview of the `PRPB` module, describing its purpose, key functions, inputs, and outputs, and summarizes the code and its components to help users understand how to utilize this module effectively.