Merge pull request #1029 from openzim/ted_topics

Enhance maint script for TED recipes

dispatcher/backend/maint-scripts/create_ted_topics_recipes.py

@@ -18,7 +18,7 @@
 TED_INDEX_NAME = "coyote_models_acme_videos_alias_38ce41d1f97ca56a38068f613af166da"
 
 
-def get_ted_topics():
+def get_ted_topics() -> set:
  data = [
  {
  "indexName": TED_INDEX_NAME,
@@ -45,11 +45,15 @@ def get_ted_topics():
  req.raise_for_status()
  topics = json.loads(req.content)["results"][0]["facets"]["tags"].keys()
  logger.info(f"{len(topics)} topics found")
- return topics
+ return set(topics)
+
+
+def get_clean_ted_topic_name(ted_topic_name):
+ return ted_topic_name.replace(" ", "-").replace("'", "")
 
 
 def create_recipe(ted_topic_name: str, access_token):
- clean_ted_topic_name = ted_topic_name.replace(" ", "-").replace("'", "")
+ clean_ted_topic_name = get_clean_ted_topic_name(ted_topic_name)
  schedule_name = f"ted_topic_{clean_ted_topic_name}"
  response = requests.get(
  get_url(f"/schedules/{schedule_name}"),
@@ -119,19 +123,68 @@ def create_recipe(ted_topic_name: str, access_token):
  response.raise_for_status()
 
 
+def get_existing_recipes_topics(access_token: str) -> set:
+
+ skip = 0
+ per_page = 200
+
+ topics = set()
+ while True:
+ response = requests.get(
+ get_url(f"/schedules/?limit={per_page}&skip={skip}&tag=ted-by-topic"),
+ headers=get_token_headers(access_token),
+ )
+ response.raise_for_status()
+
+ schedules = response.json()
+ topics.update({schedule["name"][10:] for schedule in schedules["items"]})
+
+ if (
+ schedules["meta"]["limit"] + schedules["meta"]["skip"]
+ < schedules["meta"]["count"]
+ ):
+ skip += per_page
+ else:
+ break
+
+ return topics
+
+
 def main(zf_username, zf_password):
  """Creates recipes for TED by topics"""
 
  access_token, refresh_token = get_token(zf_username, zf_password)
 
- ted_topics = get_ted_topics()
- logger.debug(",".join((ted_topics)))
- logger.debug(",".join(sorted(ted_topics)))
- for topic in ted_topics:
- if topic != "street art":
- continue
+ existing_recipes_topics = get_existing_recipes_topics(access_token=access_token)
+ logger.debug(",".join(sorted(existing_recipes_topics)))
+ exisiting_online_topics = get_ted_topics()
+ logger.debug(",".join(sorted(exisiting_online_topics)))
+
+ recipes_to_create = [
+ topic
+ for topic in exisiting_online_topics
+ if get_clean_ted_topic_name(topic) not in existing_recipes_topics
+ ]
+ logger.info(f"*** Recipes to create: {','.join(sorted(recipes_to_create))}")
+ recipes_to_delete = [
+ topic
+ for topic in existing_recipes_topics
+ if not any(
+ [
+ 1 if get_clean_ted_topic_name(topic2) == topic else 0
+ for topic2 in exisiting_online_topics
+ ]
+ )
+ and topic != "all"
+ ]
+
+ for topic in recipes_to_create:
  create_recipe(ted_topic_name=topic, access_token=access_token)
 
+ logger.info(
+ f"*** Recipes to delete manually: {','.join(sorted(recipes_to_delete))}"
+ )
+
 
 if __name__ == "__main__":
  if len(sys.argv) != 3: