fix state inconsistency at startup, connection error handling
parent
b2118c8cae
commit
3e1a895c80
|
|
@ -641,6 +641,7 @@ class Worker:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def load_options(self, model, vae=None):
|
def load_options(self, model, vae=None):
|
||||||
|
failure_msg = f"failed to load options for worker '{self.label}'"
|
||||||
if self.master:
|
if self.master:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
@ -654,17 +655,25 @@ class Worker:
|
||||||
if vae is not None:
|
if vae is not None:
|
||||||
payload['sd_vae'] = vae
|
payload['sd_vae'] = vae
|
||||||
|
|
||||||
self.set_state(State.WORKING)
|
state_cache = self.state
|
||||||
|
self.set_state(State.WORKING, expect_cycle=True) # may already be WORKING if called by worker.request()
|
||||||
start = time.time()
|
start = time.time()
|
||||||
response = self.session.post(
|
try:
|
||||||
self.full_url("options"),
|
response = self.session.post(
|
||||||
json=payload
|
self.full_url("options"),
|
||||||
)
|
json=payload
|
||||||
|
)
|
||||||
|
except requests.exceptions.RequestException:
|
||||||
|
self.set_state(State.UNAVAILABLE)
|
||||||
|
logger.debug(f"{failure_msg} (connection error... OOM?)")
|
||||||
|
return
|
||||||
|
|
||||||
elapsed = time.time() - start
|
elapsed = time.time() - start
|
||||||
self.set_state(State.IDLE)
|
if state_cache != State.WORKING: # see above comment, this lets caller determine when worker is IDLE
|
||||||
|
self.set_state(State.IDLE)
|
||||||
|
|
||||||
if response.status_code != 200:
|
if response.status_code != 200:
|
||||||
logger.debug(f"failed to load options for worker '{self.label}'")
|
logger.debug(failure_msg)
|
||||||
else:
|
else:
|
||||||
logger.debug(f"worker '{self.label}' loaded weights in {elapsed:.2f}s")
|
logger.debug(f"worker '{self.label}' loaded weights in {elapsed:.2f}s")
|
||||||
self.loaded_model = model_name
|
self.loaded_model = model_name
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue