From da2e32242eda19a15c787261a7c3a4dcebc631ed Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 5 Jun 2023 06:12:10 -0500 Subject: [PATCH] remove time from logging format to match with current sdwui logging indentation. bandaid for remainder handling --- scripts/extension.py | 2 +- scripts/spartan/World.py | 28 ++++++++++++---------------- scripts/spartan/shared.py | 2 +- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/scripts/extension.py b/scripts/extension.py index 53e9d07..240f7d5 100644 --- a/scripts/extension.py +++ b/scripts/extension.py @@ -388,7 +388,7 @@ class Script(scripts.Script): payload_temp['batch_size'] = job.batch_size payload_temp['subseed'] += prior_images payload_temp['seed'] += prior_images if payload_temp['subseed_strength'] == 0 else 0 - logger.debug(f"{job.worker.uuid}' job given starting seed is {payload_temp['seed']} with {prior_images} coming before it") + logger.debug(f"'{job.worker.uuid}' job's given starting seed is {payload_temp['seed']} with {prior_images} coming before it") if job.worker.loaded_model != name or job.worker.loaded_vae != vae: sync = True diff --git a/scripts/spartan/World.py b/scripts/spartan/World.py index 086a07a..2bf3ad9 100644 --- a/scripts/spartan/World.py +++ b/scripts/spartan/World.py @@ -114,11 +114,7 @@ class World: """the amount of images/total images requested that a worker would compute if conditions were perfect and each worker generated at the same speed""" - quotient, remainder = divmod(self.total_batch_size, self.get_world_size()) - chosen = quotient if quotient > remainder else remainder - per_worker_batch_size = math.ceil(chosen) - - return per_worker_batch_size + return self.total_batch_size // self.get_world_size() def get_world_size(self) -> int: """ @@ -452,21 +448,21 @@ class World: # *if that hasn't already been filled by complementary fill or the requirement that master's batch size be >= 1 remainder_images = self.total_batch_size - self.get_current_output_size() if remainder_images >= 1: - logger.debug(f"The requested number of images({self.total_batch_size}) was not cleanly divisible by the number of realtime nodes({len(self.realtime_jobs())}) and complementary jobs did not provide this missing image.") + logger.debug(f"The requested number of images({self.total_batch_size}) was not cleanly divisible by the number of realtime nodes({len(self.realtime_jobs())}) resulting in {remainder_images} that will be redistributed") - # Gets the fastest job that has been assigned the least amount of images - laziest_realtime_job = None - for job in self.realtime_jobs(): - if laziest_realtime_job is None: - laziest_realtime_job = job - elif laziest_realtime_job.batch_size > job.batch_size: - laziest_realtime_job = job - - laziest_realtime_job.batch_size += remainder_images + realtime_jobs = self.realtime_jobs() + realtime_jobs.sort(key=lambda x: x.batch_size) + # round-robin distribute the remaining images + while remainder_images >= 1: + for job in realtime_jobs: + if remainder_images < 1: + break + job.batch_size += 1 + remainder_images -= 1 logger.info("Job distribution:") iterations = payload['n_iter'] - logger.info(f"{iterations} iteration(s)") + logger.info(f"{self.total_batch_size} * {iterations} iteration(s): {self.total_batch_size * iterations} images") for job in self.jobs: logger.info(f"worker '{job.worker.uuid}' - {job.batch_size * iterations} images") print() diff --git a/scripts/spartan/shared.py b/scripts/spartan/shared.py index fa397ab..9c4303f 100644 --- a/scripts/spartan/shared.py +++ b/scripts/spartan/shared.py @@ -4,7 +4,7 @@ from modules.shared import cmd_opts log_level = 'DEBUG' if cmd_opts.distributed_debug else 'INFO' logger = logging.getLogger("rich") -logger.addHandler(RichHandler(rich_tracebacks=True, markup=True)) +logger.addHandler(RichHandler(rich_tracebacks=True, markup=True, show_time=False)) logger.setLevel(log_level) warmup_samples = 2 # number of samples to do before recording a valid benchmark sample