Merge pull request #331 from Critical-Impact/anidb-authentication

AniDB Authentication and Tag Builder

config/config.yml.template

@@ -89,3 +89,6 @@ mal:
  token_type:
  expires_in:
  refresh_token:
+anidb:
+ username: ###### - optional
+ password: ######

modules/anidb.py

@@ -1,25 +1,47 @@
-import logging, requests
+import logging, requests,time
 from lxml import html
 from modules import util
 from modules.util import Failed
 from retrying import retry
 
 logger = logging.getLogger("Plex Meta Manager")
 
-builders = ["anidb_id", "anidb_relation", "anidb_popular"]
+builders = ["anidb_id", "anidb_relation", "anidb_popular", 'anidb_tag']
 
 class AniDB:
- def __init__(self, config):
+ def __init__(self, params, config):
  self.config = config
+
+ # Create a session so if we login we can continue to use the same session
+ self.anidb_session = requests.Session()
+
  self.urls = {
  "anime": "https://anidb.net/anime",
  "popular": "https://anidb.net/latest/anime/popular/?h=1",
- "relation": "/relation/graph"
+ "relation": "/relation/graph",
+ "anidb_tag": "https://anidb.net/tag",
+ "login": "https://anidb.net/perl-bin/animedb.pl"
  }
 
+ if params and "username" in params and "password" in params:
+ result = str(self._login(params["username"], params["password"]).content)
+
+ # Login response does not use proper status codes so we have to check the content of the document
+ if "Wrong username/password" in result:
+ raise Failed("AniDB Error: Login failed")
+
  @retry(stop_max_attempt_number=6, wait_fixed=10000)
  def _request(self, url, language):
- return html.fromstring(requests.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content)
+ return html.fromstring(self.anidb_session.get(url, headers={"Accept-Language": language, "User-Agent": "Mozilla/5.0 x64"}).content)
+
+ def _login(self, username, password):
+ data = {
+ "show": "main",
+ "xuser": username,
+ "xpass": password,
+ "xdoautologin": "on"
+ }
+ return self.anidb_session.post(self.urls["login"], data, headers={"Accept-Language": "en-US,en;q=0.5", "User-Agent": "Mozilla/5.0 x64"})
 
  def _popular(self, language):
  response = self._request(self.urls["popular"], language)
@@ -47,12 +69,36 @@ def validate_anidb_list(self, anidb_list, language):
  return anidb_values
  raise Failed(f"AniDB Error: No valid AniDB IDs in {anidb_list}")
 
+ def _tag(self, tag, limit, language):
+ anidb_ids = []
+ next_page = True
+ current_url = self.urls["anidb_tag"] + "/" + str(tag)
+ while next_page:
+ logger.debug(f"Sending request to {current_url}")
+ response = self._request(current_url, language)
+ int_list = util.get_int_list(response.xpath("//td[@class='name main anime']/a/@href"), "AniDB ID")
+ anidb_ids.extend(int_list)
+ next_page_list = response.xpath("//li[@class='next']/a/@href")
+ logger.debug(f"next page list {next_page_list}")
+ if len(next_page_list) != 0 and len(anidb_ids) <= limit:
+ logger.debug(f"Loading next anidb page")
+ time.sleep(2)# Sleep as we are paging through anidb and don't want the ban hammer
+ current_url = "https://anidb.net" + next_page_list[0]
+ else:
+ logger.debug(f"Got to last page")
+ next_page = False
+ anidb_ids = anidb_ids[:limit]
+ return anidb_ids
+
  def get_items(self, method, data, language):
  pretty = util.pretty_names[method] if method in util.pretty_names else method
  anidb_ids = []
  if method == "anidb_popular":
  logger.info(f"Processing {pretty}: {data} Anime")
  anidb_ids.extend(self._popular(language)[:data])
+ elif method == "anidb_tag":
+ anidb_ids = self._tag(data["tag"], data["limit"], language)
+ logger.info(f"Processing {pretty}: {data['limit'] if data['limit'] > 0 else 'All'} Anime from the Tag ID: {data['tag']}")
  else:
  logger.info(f"Processing {pretty}: {data}")
  if method == "anidb_id": anidb_ids.append(data)
@@ -63,4 +109,4 @@ def get_items(self, method, data, language):
  logger.debug(f"{len(anidb_ids)} AniDB IDs Found: {anidb_ids}")
  logger.debug(f"{len(movie_ids)} TMDb IDs Found: {movie_ids}")
  logger.debug(f"{len(show_ids)} TVDb IDs Found: {show_ids}")
- return movie_ids, show_ids
+ return movie_ids, show_ids

modules/config.py

@@ -108,6 +108,7 @@ def replace_attr(all_data, attr, par):
  if "omdb" in new_config: new_config["omdb"] = new_config.pop("omdb")
  if "trakt" in new_config: new_config["trakt"] = new_config.pop("trakt")
  if "mal" in new_config: new_config["mal"] = new_config.pop("mal")
+ if "anidb" in new_config: new_config["anidb"] = new_config.pop("anidb")
  yaml.round_trip_dump(new_config, open(self.config_path, "w", encoding="utf-8"), indent=ind, block_seq_indent=bsi)
  self.data = new_config
  except yaml.scanner.ScannerError as e:
@@ -270,9 +271,29 @@ def check_for_attribute(data, attribute, parent=None, test_list=None, default=No
  else:
  logger.warning("mal attribute not found")
 
+ util.separator()
+
+ self.AniDB = None
+ anidb_username = check_for_attribute(self.data, "username", parent="anidb", throw=False, default=False)
+ anidb_password = check_for_attribute(self.data, "username", parent="anidb", throw=False, default=False)
+ if "anidb" in self.data and anidb_username and anidb_password:
+ logger.info("Connecting to AniDB...")
+ self.anidb = {}
+ try:
+ self.anidb["username"] = check_for_attribute(self.data, "username", parent="anidb", throw=True)
+ self.anidb["password"] = check_for_attribute(self.data, "password", parent="anidb", throw=True)
+ self.AniDB = AniDB(self.anidb, self)
+ except Failed as e:
+ logger.error(e)
+ logger.info(f"My Anime List Connection {'Failed' if self.MyAnimeList is None else 'Successful'}")
+ else:
+ logger.info("Using guest authentication for AniDB")
+ self.AniDB = AniDB(None, self)
+
+ util.separator()
+
  self.TVDb = TVDb(self)
  self.IMDb = IMDb(self)
- self.AniDB = AniDB(self)
  self.Convert = Convert(self)
  self.AniList = AniList(self)
  self.Letterboxd = Letterboxd(self)