feat(hbi_utils & locust): Add timeout mechanism when waiting for Locust workers, Display clearer host count in exception message when not all hosts are read - display only the newly added host count for the error message to make more sense

opl/args.py

@@ -318,6 +318,13 @@ def add_locust_opts(parser):
  default=int(os.getenv("LOCUST_STOP_TIMEOUT", 10)),
  help="Locust stop timeout (also use env variable LOCUST_STOP_TIMEOUT)",
  )
+ parser.add_argument(
+ "--locust-wait-for-worker-timeout",
+ dest="worker_wait_timeout",
+ type=int,
+ default=int(os.getenv("LOCUST_WAIT_FOR_WORKER_TIMEOUT", 120)),
+ help="Locust timeout [s] for waiting until worker pods are ready. (also use env variable LOCUST_WAIT_FOR_WORKER_TIMEOUT)",
+ )
 
  # Our test specific parameters
  parser.add_argument(

opl/hbi_utils.py

@@ -141,7 +141,7 @@ def verify(args, previous_records, status_data, inventory, collect_info):
  attempt += 1
  if attempt > attempts_max:
  raise Exception(
- f"After {attempt} attempts, we only have {existing_ids} out of {args.count}"
+ f"After {attempt} attempts, we only have {existing_ids-previous_records} out of {args.count}"
  )
 
  # If there were no new hosts now, wait a bit

opl/locust.py

@@ -93,6 +93,7 @@ def run_locust(args, status_data, test_set, new_stats=False, summary_only=False)
 
  env.runner.spawn_rate = args.hatch_rate
 
+ time_spent_waiting = 0
  while len(env.runner.clients.ready) < args.expect_workers:
  logging.info(
  "Waiting for worker to become ready, %s of %s - %s",
@@ -101,6 +102,11 @@ def run_locust(args, status_data, test_set, new_stats=False, summary_only=False)
  ",".join([i.state for i in env.runner.clients.values()]),
  )
  time.sleep(1)
+ time_spent_waiting += 1
+ if time_spent_waiting >= args.worker_wait_timeout:
+ raise TimeoutError(
+ f"Timed out waiting for Locust workers to get ready: {len(env.runner.clients.ready)} out of {args.expect_workers}"
+ )
 
  # Start the test
  logging.info("Starting master Locust runer")