Add full TTS to vision mode

README.md

@@ -79,18 +79,15 @@ Currently, `use_tts` enables either a mixed mode where image processing is still
 but TTS is used for everything text-related. This results in a small improvement in performance and a major improvement
 in accuracy. Or a full mode where only TTS is used.
 
-**The following is currently supported using use_tts=mixed:**
+**IT IS NOT POSSIBLE TO DETECT GREATER AFFIXES USING TTS!!! YOU NEED TO SPECIFY THEM USING THEIR VALUES!**
+
+**The following is currently supported using any form of tts:**
 
 - Full item detection for all wearable items, e.g. armor, weapons, and accessories. Both in `vision_mode` and
 `loot_filter`.
 - Basic item detection for all? other items, e.g. only type + rarity
 
-**The following is currently supported using use_tts=mixed:**
-
-- Full item detection for all wearable items, e.g. armor, weapons, and accessories. Only in `loot_filter`.
-- For everything else, mixed mode is used
-
-We might also discontinue the pure image processing mode in the future, as TTS is easier to maintain.
+We might also discontinue the pure image processing mode and even mixed mode in the future, as TTS is easier to maintain.
 
 ### Configs
 

src/__init__.py

@@ -2,4 +2,4 @@
 
 TP = concurrent.futures.ThreadPoolExecutor()
 
-__version__ = "5.9.0alpha2"
+__version__ = "5.9.0alpha3"

src/item/descr/read_descr_tts.py

@@ -277,7 +277,7 @@ def read_descr_mixed(img_item_descr: np.ndarray) -> Item | None:
  return _add_affixes_from_tts_mixed(tts_section, item, inherent_affix_bullets, affix_bullets)
 
 
-def read_description() -> Item | None:
+def read_descr() -> Item | None:
  tts_section = copy.copy(src.tts.LAST_ITEM)
  if not tts_section:
  return None

src/item/filter.py

@@ -33,16 +33,16 @@
 
 
 @dataclass
-class _MatchedFilter:
+class MatchedFilter:
  profile: str
  matched_affixes: list[Affix] = field(default_factory=list)
  did_match_aspect: bool = False
 
 
 @dataclass
-class _FilterResult:
+class FilterResult:
  keep: bool
- matched: list[_MatchedFilter]
+ matched: list[MatchedFilter]
  unique_aspect_in_profile = False
  all_unique_filters_are_aspects = False
 
@@ -78,10 +78,10 @@ def __new__(cls):
  cls._instance = super().__new__(cls)
  return cls._instance
 
- def _check_affixes(self, item: Item) -> _FilterResult:
- res = _FilterResult(False, [])
+ def _check_affixes(self, item: Item) -> FilterResult:
+ res = FilterResult(False, [])
  if not self.affix_filters:
- return _FilterResult(True, [])
+ return FilterResult(True, [])
  non_tempered_affixes = [affix for affix in item.affixes if affix.type != AffixType.tempered]
  for profile_name, profile_filter in self.affix_filters.items():
  for filter_item in profile_filter:
@@ -113,27 +113,27 @@ def _check_affixes(self, item: Item) -> _FilterResult:
  all_matches = matched_affixes + matched_inherents
  LOGGER.info(f"Matched {profile_name}.Affixes.{filter_name}: {[x.name for x in all_matches]}")
  res.keep = True
- res.matched.append(_MatchedFilter(f"{profile_name}.{filter_name}", all_matches))
+ res.matched.append(MatchedFilter(f"{profile_name}.{filter_name}", all_matches))
  return res
 
  @staticmethod
- def _check_aspect(item: Item) -> _FilterResult:
- res = _FilterResult(False, [])
+ def _check_aspect(item: Item) -> FilterResult:
+ res = FilterResult(False, [])
  if IniConfigLoader().general.keep_aspects == AspectFilterType.none or (
  IniConfigLoader().general.keep_aspects == AspectFilterType.upgrade and not item.codex_upgrade
  ):
  return res
  LOGGER.info("Matched Aspects that updates codex")
  res.keep = True
- res.matched.append(_MatchedFilter("Aspects", did_match_aspect=True))
+ res.matched.append(MatchedFilter("Aspects", did_match_aspect=True))
  return res
 
- def _check_sigil(self, item: Item) -> _FilterResult:
- res = _FilterResult(False, [])
+ def _check_sigil(self, item: Item) -> FilterResult:
+ res = FilterResult(False, [])
  if not self.sigil_filters.items():
  LOGGER.info("Matched Sigils")
  res.keep = True
- res.matched.append(_MatchedFilter("default"))
+ res.matched.append(MatchedFilter("default"))
  for profile_name, profile_filter in self.sigil_filters.items():
  # check item power
  if not self._match_item_power(max_power=profile_filter.maxTier, min_power=profile_filter.minTier, item_power=item.power):
@@ -164,15 +164,15 @@ def _check_sigil(self, item: Item) -> _FilterResult:
  continue
  LOGGER.info(f"Matched {profile_name}.Sigils")
  res.keep = True
- res.matched.append(_MatchedFilter(f"{profile_name}"))
+ res.matched.append(MatchedFilter(f"{profile_name}"))
  return res
 
- def _check_unique_item(self, item: Item) -> _FilterResult:
- res = _FilterResult(False, [])
+ def _check_unique_item(self, item: Item) -> FilterResult:
+ res = FilterResult(False, [])
  all_filters_are_aspect = True
  if not self.unique_filters:
  keep = IniConfigLoader().general.handle_uniques != UnfilteredUniquesType.junk or item.rarity == ItemRarity.Mythic
- return _FilterResult(keep, [])
+ return FilterResult(keep, [])
  for profile_name, profile_filter in self.unique_filters.items():
  for filter_item in profile_filter:
  if not filter_item.aspect:
@@ -202,7 +202,7 @@ def _check_unique_item(self, item: Item) -> _FilterResult:
  matched_full_name = f"{profile_name}.{item.aspect.name}"
  if filter_item.profileAlias:
  matched_full_name = f"{filter_item.profileAlias}.{item.aspect.name}"
- res.matched.append(_MatchedFilter(matched_full_name, did_match_aspect=True))
+ res.matched.append(MatchedFilter(matched_full_name, did_match_aspect=True))
  res.all_unique_filters_are_aspects = all_filters_are_aspect
 
  # Always keep mythics no matter what
@@ -348,11 +348,11 @@ def load_files(self):
  sys.exit(1)
  self.last_loaded = time.time()
 
- def should_keep(self, item: Item) -> _FilterResult:
+ def should_keep(self, item: Item) -> FilterResult:
  if not self.files_loaded or self._did_files_change():
  self.load_files()
 
- res = _FilterResult(False, [])
+ res = FilterResult(False, [])
 
  if item.item_type is None or item.power is None:
  return res

src/overlay.py

@@ -1,38 +1,32 @@
 import ctypes
 import logging
 import threading
+import time
 import tkinter as tk
 import typing
 
+import src.item.descr.read_descr_tts
+import src.logger
 import src.scripts.loot_filter
 import src.scripts.loot_filter_tts
+import src.scripts.vision_mode
+import src.scripts.vision_mode_tts
+import src.tts
 from src.cam import Cam
 from src.config.loader import IniConfigLoader
 from src.config.models import ItemRefreshType, UseTTSType
-from src.config.ui import ResManager
 from src.loot_mover import move_items_to_inventory, move_items_to_stash
-from src.scripts.vision_mode import vision_mode
+from src.ui.char_inventory import CharInventory
+from src.ui.chest import Chest
+from src.utils.custom_mouse import mouse
 from src.utils.process_handler import kill_thread
-from src.utils.window import WindowSpec, move_window_to_foreground
+from src.utils.window import screenshot
 
 LOGGER = logging.getLogger(__name__)
 
 LOCK = threading.Lock()
 
 
-class TextLogHandler(logging.Handler):
- def __init__(self, text):
- logging.Handler.__init__(self)
- self.text = text
- self.text.tag_configure("wrapindent", lmargin2=60)
-
- def emit(self, record):
- log_entry = self.format(record)
- padded_text = " " * 1 + log_entry + " \n" * 1
- self.text.insert(tk.END, padded_text, "wrapindent")
- self.text.yview(tk.END) # Auto-scroll to the end
-
-
 class Overlay:
  def __init__(self):
  self.loot_interaction_thread = None
@@ -48,7 +42,7 @@ def __init__(self):
  self.screen_width = Cam().window_roi["width"]
  self.screen_height = Cam().window_roi["height"]
  self.initial_height = int(Cam().window_roi["height"] * 0.03)
- self.initial_width = int(self.screen_width * 0.068)
+ self.initial_width = int(self.screen_width * 0.047)
  self.maximized_height = int(self.initial_height * 3.4)
  self.maximized_width = int(self.initial_width * 5)
 
@@ -62,51 +56,21 @@ def __init__(self):
  self.root.bind("<Enter>", self.show_canvas)
  self.root.bind("<Leave>", self.hide_canvas)
 
- self.toggle_button = tk.Button(self.root, text="max", bg="#222222", fg="#555555", borderwidth=0, command=self.toggle_size)
- self.canvas.create_window(int(self.initial_width * 0.19), self.initial_height // 2, window=self.toggle_button)
-
  self.start_scripts_button = tk.Button(self.root, text="vision", bg="#222222", fg="#555555", borderwidth=0, command=self.run_scripts)
 
  if not IniConfigLoader().advanced_options.vision_mode_only:
  self.filter_button = tk.Button(self.root, text="filter", bg="#222222", fg="#555555", borderwidth=0, command=self.filter_items)
- self.canvas.create_window(int(self.initial_width * 0.48), self.initial_height // 2, window=self.filter_button)
+ self.canvas.create_window(int(self.initial_width * 0.24), self.initial_height // 2, window=self.filter_button)
 
  self.start_scripts_button = tk.Button(self.root, text="vision", bg="#222222", fg="#555555", borderwidth=0, command=self.run_scripts)
- self.canvas.create_window(int(self.initial_width * 0.81), self.initial_height // 2, window=self.start_scripts_button)
-
- font_size = 8
- window_height = ResManager().pos.window_dimensions[1]
- if window_height == 1440:
- font_size = 9
- elif window_height > 1440:
- font_size = 10
- self.terminal_text = tk.Text(
- self.canvas,
- bg="black",
- fg="white",
- highlightcolor="white",
- highlightthickness=0,
- selectbackground="#222222",
- borderwidth=0,
- font=("Courier New", font_size),
- wrap="word",
- )
- self.terminal_text.place(
- relx=0, rely=0, relwidth=1, relheight=1 - (self.initial_height / self.maximized_height), y=self.initial_height
- )
+ self.canvas.create_window(int(self.initial_width * 0.73), self.initial_height // 2, window=self.start_scripts_button)
 
  if IniConfigLoader().general.hidden_transparency == 0:
  self.root.update()
  hwnd = self.root.winfo_id()
  style = ctypes.windll.user32.GetWindowLongW(hwnd, -20)
  ctypes.windll.user32.SetWindowLongW(hwnd, -20, style | 0x80000 | 0x20)
 
- # Setup the listbox logger handler
- textlog_handler = TextLogHandler(self.terminal_text)
- log_level = LOGGER.root.handlers[0].level if LOGGER.root.handlers else 0
- textlog_handler.setLevel(log_level)
- LOGGER.root.addHandler(textlog_handler)
-
  if IniConfigLoader().general.run_vision_mode_on_startup:
  self.run_scripts()
 
@@ -125,32 +89,11 @@ def hide_canvas(self, _):
  self.root.after_cancel(self.hide_id)
  self.hide_id = self.root.after(3000, lambda: self.root.attributes("-alpha", IniConfigLoader().general.hidden_transparency))
 
- def toggle_size(self):
- if not self.is_minimized:
- self.canvas.config(height=self.initial_height, width=self.initial_width)
- self.root.geometry(
- f"{self.initial_width}x{self.initial_height}+{self.screen_width // 2 - self.initial_width // 2 + self.screen_off_x}+{self.screen_height - self.initial_height + self.screen_off_y}"
- )
- else:
- self.canvas.config(height=self.maximized_height, width=self.maximized_width)
- self.root.geometry(
- f"{self.maximized_width}x{self.maximized_height}+{self.screen_width // 2 - self.maximized_width // 2 + self.screen_off_x}+{self.screen_height - self.maximized_height + self.screen_off_y}"
- )
- self.is_minimized = not self.is_minimized
- if self.is_minimized:
- self.hide_canvas(None)
- self.toggle_button.config(text="max")
- else:
- self.show_canvas(None)
- self.toggle_button.config(text="min")
- win_spec = WindowSpec(IniConfigLoader().advanced_options.process_name)
- move_window_to_foreground(win_spec)
-
  def filter_items(self, force_refresh=ItemRefreshType.no_refresh):
- if IniConfigLoader().general.use_tts == UseTTSType.full:
- self._start_or_stop_loot_interaction_thread(src.scripts.loot_filter_tts.run_loot_filter, (force_refresh,))
+ if IniConfigLoader().general.use_tts in [UseTTSType.full, UseTTSType.mixed]:
+ self._start_or_stop_loot_interaction_thread(run_loot_filter, (force_refresh, True))
  else:
- self._start_or_stop_loot_interaction_thread(src.scripts.loot_filter.run_loot_filter, (force_refresh,))
+ self._start_or_stop_loot_interaction_thread(run_loot_filter, (force_refresh, False))
 
  def move_items_to_inventory(self):
  self._start_or_stop_loot_interaction_thread(move_items_to_inventory)
@@ -167,8 +110,6 @@ def _start_or_stop_loot_interaction_thread(self, loot_interaction_method: typing
  self.loot_interaction_thread = None
  self.filter_button.config(fg="#555555")
  else:
- if self.is_minimized:
- self.toggle_size()
  self.loot_interaction_thread = threading.Thread(
  target=self._wrapper_run_loot_interaction_method, args=(loot_interaction_method, method_args), daemon=True
  )
@@ -196,8 +137,6 @@ def _wrapper_run_loot_interaction_method(self, loot_interaction_method: typing.C
  if did_stop_scripts:
  self.run_scripts()
  finally:
- if not self.is_minimized:
- self.toggle_size()
  self.loot_interaction_thread = None
  self.filter_button.config(fg="#555555")
 
@@ -216,7 +155,10 @@ def run_scripts(self):
  return
  for name in IniConfigLoader().advanced_options.scripts:
  if name == "vision_mode":
- vision_mode_thread = threading.Thread(target=vision_mode, daemon=True)
+ if IniConfigLoader().general.use_tts == UseTTSType.full:
+ vision_mode_thread = threading.Thread(target=src.scripts.vision_mode_tts.VisionMode().start, daemon=True)
+ else:
+ vision_mode_thread = threading.Thread(target=src.scripts.vision_mode.vision_mode, daemon=True)
  vision_mode_thread.start()
  self.script_threads.append(vision_mode_thread)
  self.start_scripts_button.config(fg="#006600")
@@ -227,3 +169,29 @@ def run_scripts(self):
 
  def run(self):
  self.root.mainloop()
+
+
+def run_loot_filter(force_refresh: ItemRefreshType = ItemRefreshType.no_refresh, tts: bool = False):
+ LOGGER.info("Run Loot filter")
+ mouse.move(*Cam().abs_window_to_monitor((0, 0)))
+ check_items = src.scripts.loot_filter_tts.check_items if tts else src.scripts.loot_filter.check_items
+
+ inv = CharInventory()
+ chest = Chest()
+
+ if chest.is_open():
+ for i in IniConfigLoader().general.check_chest_tabs:
+ chest.switch_to_tab(i)
+ time.sleep(0.3)
+ check_items(chest, force_refresh)
+ mouse.move(*Cam().abs_window_to_monitor((0, 0)))
+ time.sleep(0.3)
+ check_items(inv, force_refresh)
+ else:
+ if not inv.open():
+ screenshot("inventory_not_open", img=Cam().grab())
+ LOGGER.error("Inventory did not open up")
+ return
+ check_items(inv, force_refresh)
+ mouse.move(*Cam().abs_window_to_monitor((0, 0)))
+ LOGGER.info("Loot Filter done")

src/scripts/common.py

@@ -2,13 +2,23 @@
 import time
 
 import keyboard
+import numpy as np
 
 from src.cam import Cam
+from src.config.ui import ResManager
 from src.utils.custom_mouse import mouse
+from src.utils.image_operations import crop
+from src.utils.ocr.read import image_to_text
 
 LOGGER = logging.getLogger(__name__)
 
 
+def is_vendor_open(img: np.ndarray):
+ cropped = crop(img, ResManager().roi.vendor_text)
+ res = image_to_text(cropped, do_pre_proc=False)
+ return res.text.strip().lower() == "vendor"
+
+
 def mark_as_junk():
  keyboard.send("space")
  time.sleep(0.13)
@@ -22,6 +32,14 @@ def mark_as_favorite():
  time.sleep(0.13)
 
 
+def reset_canvas(root, canvas):
+ canvas.delete("all")
+ canvas.config(height=0, width=0)
+ root.geometry("0x0+0+0")
+ root.update_idletasks()
+ root.update()
+
+
 def reset_item_status(occupied, inv):
  for item_slot in occupied:
  if item_slot.is_fav: