1580158
/
stable-diffusion-aws-extension
mirror of https://github.com/awslabs/stable-diffusion-aws-extension.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update tests

pull/718/head
Jingyi 2024-04-27 08:54:59 +08:00
parent 97daa90e8e
commit 1cb05aa9b2
10 changed files with 86 additions and 769 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
test/local_test.sh
View File

@ -11,4 +11,4 @@ if [ -n "$1" ]; then
exit 0
fi
pytest ./ --exitfirst -rA --log-cli-level=INFO --json-report --json-report-summary --json-report-file=detailed_report.json --html=report.html --self-contained-html --continue-on-collection-errors
pytest ./ --tb=short --exitfirst -qrA --log-cli-level=INFO --json-report --json-report-summary --json-report-file=detailed_report.json --html=report.html --self-contained-html --continue-on-collection-errors

2
test/local_test_k.sh
View File

@ -11,4 +11,4 @@ if [ -z "$1" ]; then
exit 1
fi
pytest ./ -k "$1" --exitfirst -rA --log-cli-level=INFO --json-report --json-report-summary --json-report-file=detailed_report.json --html=report.html --self-contained-html --continue-on-collection-errors
pytest ./ -k "$1" --tb=short --exitfirst -qrA --log-cli-level=INFO --json-report --json-report-summary --json-report-file=detailed_report.json --html=report.html --self-contained-html --continue-on-collection-errors

176
test/test_06_api_inference/test_06_comfy_create_and_get.py
View File

@ -1,12 +1,15 @@
import json
import logging
import time
import uuid
import config as config
from utils.api import Api
from utils.helper import get_endpoint_comfy_async
logger = logging.getLogger(__name__)
id = "0354bca4-7e04-4c35-86e5-5fab18879376"
prompt_id = str(uuid.uuid4())
class TestTxt2ImgReCreateAndGetComfyE2E:
@ -14,173 +17,28 @@ class TestTxt2ImgReCreateAndGetComfyE2E:
def setup_class(self):
self.api = Api(config)
self.api.feat_oas_schema()
self.ep_name = get_endpoint_comfy_async(self.api)
@classmethod
def teardown_class(self):
pass
def test_0_comfy_txt2img_async_create(self):
headers = {
"x-api-key": config.api_key,
}
with open('./data/api_params/comfy_workflow.json', 'r') as f:
headers = {
"x-api-key": config.api_key,
}
workflow = json.load(f)
workflow['prompt_id'] = prompt_id
workflow['endpoint_name'] = self.ep_name
payload = json.dumps({
"need_sync": True,
"prompt": {
"4": {
"inputs": {
"ckpt_name": "sdXL_v10VAEFix.safetensors"
},
"class_type": "CheckpointLoaderSimple"
},
"5": {
"inputs": {
"width": 1024,
"height": 1024,
"batch_size": 1
},
"class_type": "EmptyLatentImage"
},
"6": {
"inputs": {
"text": "evening sunset scenery blue sky nature, glass bottle with a galaxy in it",
"clip": [
"4",
1
]
},
"class_type": "CLIPTextEncode"
},
"7": {
"inputs": {
"text": "text, watermark",
"clip": [
"4",
1
]
},
"class_type": "CLIPTextEncode"
},
"10": {
"inputs": {
"add_noise": "enable",
"noise_seed": 721897303308196,
"steps": 25,
"cfg": 8,
"sampler_name": "euler",
"scheduler": "normal",
"start_at_step": 0,
"end_at_step": 20,
"return_with_leftover_noise": "enable",
"model": [
"4",
0
],
"positive": [
"6",
0
],
"negative": [
"7",
0
],
"latent_image": [
"5",
0
]
},
"class_type": "KSamplerAdvanced"
},
"11": {
"inputs": {
"add_noise": "disable",
"noise_seed": 0,
"steps": 25,
"cfg": 8,
"sampler_name": "euler",
"scheduler": "normal",
"start_at_step": 20,
"end_at_step": 10000,
"return_with_leftover_noise": "disable",
"model": [
"12",
0
],
"positive": [
"15",
0
],
"negative": [
"16",
0
],
"latent_image": [
"10",
0
]
},
"class_type": "KSamplerAdvanced"
},
"12": {
"inputs": {
"ckpt_name": "sdXL_v10VAEFix.safetensors"
},
"class_type": "CheckpointLoaderSimple"
},
"15": {
"inputs": {
"text": "evening sunset scenery blue sky nature, glass bottle with a galaxy in it",
"clip": [
"12",
1
]
},
"class_type": "CLIPTextEncode"
},
"16": {
"inputs": {
"text": "text, watermark",
"clip": [
"12",
1
]
},
"class_type": "CLIPTextEncode"
},
"17": {
"inputs": {
"samples": [
"11",
0
],
"vae": [
"12",
2
]
},
"class_type": "VAEDecode"
},
"19": {
"inputs": {
"filename_prefix": "ComfyUI",
"images": [
"17",
0
]
},
"class_type": "SaveImage"
}
},
"prompt_id": id,
"endpoint_name": config.comfy_async_ep_name
})
resp = self.api.create_execute(headers=headers, data=json.loads(payload))
assert resp.json()["statusCode"] == 201, resp.dumps()
logger.info(f"resp: {resp.json()['data']['prompt_id']}")
resp = self.api.create_execute(headers=headers, data=workflow)
assert resp.status_code in [200, 201], resp.dumps()
assert resp.json()['data']['prompt_id'] == prompt_id, resp.dumps()
def test_2_comfy_txt2img_get(self):
headers = {
"x-api-key": config.api_key,
}
self.api.get_execute_job(headers=headers, prompt_id=id)
time.sleep(15)
self.api.get_execute_job(headers=headers, prompt_id=prompt_id)

170
test/test_06_api_inference/test_07_comfy_query_and_delete.py
View File

@ -4,8 +4,10 @@ import uuid
import config as config
from utils.api import Api
from utils.helper import get_endpoint_comfy_async
logger = logging.getLogger(__name__)
prompt_id = str(uuid.uuid4())
class TestTxt2ImgReQueryAndDeleteComfyE2E:
@ -13,6 +15,7 @@ class TestTxt2ImgReQueryAndDeleteComfyE2E:
def setup_class(self):
self.api = Api(config)
self.api.feat_oas_schema()
self.ep_name = get_endpoint_comfy_async(self.api)
@classmethod
def teardown_class(self):
@ -24,164 +27,17 @@ class TestTxt2ImgReQueryAndDeleteComfyE2E:
self.comfy_txt2img_async_create()
def comfy_txt2img_async_create(self):
headers = {
"x-api-key": config.api_key,
}
with open('./data/api_params/comfy_workflow.json', 'r') as f:
headers = {
"x-api-key": config.api_key,
}
workflow = json.load(f)
workflow['prompt_id'] = prompt_id
workflow['endpoint_name'] = self.ep_name
payload = json.dumps({
"need_sync": True,
"prompt": {
"4": {
"inputs": {
"ckpt_name": "sdXL_v10VAEFix.safetensors"
},
"class_type": "CheckpointLoaderSimple"
},
"5": {
"inputs": {
"width": 1024,
"height": 1024,
"batch_size": 1
},
"class_type": "EmptyLatentImage"
},
"6": {
"inputs": {
"text": "evening sunset scenery blue sky nature, glass bottle with a galaxy in it",
"clip": [
"4",
1
]
},
"class_type": "CLIPTextEncode"
},
"7": {
"inputs": {
"text": "text, watermark",
"clip": [
"4",
1
]
},
"class_type": "CLIPTextEncode"
},
"10": {
"inputs": {
"add_noise": "enable",
"noise_seed": 721897303308196,
"steps": 25,
"cfg": 8,
"sampler_name": "euler",
"scheduler": "normal",
"start_at_step": 0,
"end_at_step": 20,
"return_with_leftover_noise": "enable",
"model": [
"4",
0
],
"positive": [
"6",
0
],
"negative": [
"7",
0
],
"latent_image": [
"5",
0
]
},
"class_type": "KSamplerAdvanced"
},
"11": {
"inputs": {
"add_noise": "disable",
"noise_seed": 0,
"steps": 25,
"cfg": 8,
"sampler_name": "euler",
"scheduler": "normal",
"start_at_step": 20,
"end_at_step": 10000,
"return_with_leftover_noise": "disable",
"model": [
"12",
0
],
"positive": [
"15",
0
],
"negative": [
"16",
0
],
"latent_image": [
"10",
0
]
},
"class_type": "KSamplerAdvanced"
},
"12": {
"inputs": {
"ckpt_name": "sdXL_v10VAEFix.safetensors"
},
"class_type": "CheckpointLoaderSimple"
},
"15": {
"inputs": {
"text": "evening sunset scenery blue sky nature, glass bottle with a galaxy in it",
"clip": [
"12",
1
]
},
"class_type": "CLIPTextEncode"
},
"16": {
"inputs": {
"text": "text, watermark",
"clip": [
"12",
1
]
},
"class_type": "CLIPTextEncode"
},
"17": {
"inputs": {
"samples": [
"11",
0
],
"vae": [
"12",
2
]
},
"class_type": "VAEDecode"
},
"19": {
"inputs": {
"filename_prefix": "ComfyUI",
"images": [
"17",
0
]
},
"class_type": "SaveImage"
}
},
"prompt_id": str(uuid.uuid4()),
"endpoint_name": config.comfy_async_ep_name
})
resp = self.api.create_execute(headers=headers, data=json.loads(payload))
assert resp.json()["statusCode"] == 201
logger.info(f"execute created: {resp.json()['data']['prompt_id']}")
resp = self.api.create_execute(headers=headers, data=workflow)
assert resp.status_code in [200, 201], resp.dumps()
assert resp.json()['data']['prompt_id'] == prompt_id, resp.dumps()
def test_2_comfy_txt2img_list(self):
last_evaluated_key = None

4
test/test_06_api_inference/test_08_comfy_inference_txt2img_async.py
View File

@ -51,8 +51,8 @@ class TestTxt2ImgInferenceAsyncAfterComfyE2E:
resp = self.api.prepare(data=data, headers=headers)
assert resp.status_code == 200, resp.dumps()
logger.info(resp.json())
logger.info(f"prepare {prepare_id} wait 50s for endpoint sync files...")
time.sleep(50)
logger.info(f"prepare {prepare_id} wait 20s for endpoint sync files...")
time.sleep(20)
def test_1_comfy_txt2img_async_create(self):
comfy_execute_create(1, self.api, self.ep_name)

4
test/test_06_api_inference/test_09_comfy_inference_txt2img_real_time.py
View File

@ -50,8 +50,8 @@ class TestTxt2ImgInferenceRealtimeAfterComfyE2E:
resp = self.api.prepare(data=data, headers=headers)
assert resp.status_code == 200, resp.dumps()
logger.info(resp.json())
logger.info(f"prepare {prepare_id} wait 50s for endpoint sync files...")
time.sleep(50)
logger.info(f"prepare {prepare_id} wait 20s for endpoint sync files...")
time.sleep(20)
def test_1_comfy_txt2img_real_time_create(self):
comfy_execute_create(1, self.api, self.ep_name)

15
test/test_08_api_inference_re/test_00_comfy_sync_files.py
View File

@ -2,6 +2,7 @@ from __future__ import print_function
import logging
import os
import time
import uuid
import config as config
@ -15,7 +16,7 @@ endpoint_name = f"comfy-async-{config.endpoint_name}"
id = str(uuid.uuid4())
class TestComfyReSyncFiles:
class TestReComfySyncFiles:
def setup_class(self):
self.api = Api(config)
@ -25,7 +26,7 @@ class TestComfyReSyncFiles:
pass
def test_0_download_file(self):
local_path = f"./data/comfy/checkpoints/v1-5-pruned-emaonly.ckpt"
local_path = f"./data/comfy/models/checkpoints/v1-5-pruned-emaonly.ckpt"
wget_file(
local_path,
'https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt'
@ -41,8 +42,16 @@ class TestComfyReSyncFiles:
os.system(f"./s5cmd sync {local} {target}")
def test_2_sync_files(self):
headers = {
"x-api-key": config.api_key,
"username": config.username
}
data = {"endpoint_name": f"{endpoint_name}",
"need_reboot": True,
"prepare_id": id,
"prepare_type": "models"}
self.api.prepare(data=data)
resp = self.api.prepare(data=data, headers=headers)
assert resp.status_code == 200, resp.dumps()
logger.info(resp.json())
logger.info(f"wait 30s for endpoint sync files...")
time.sleep(30)

178
test/test_08_api_inference_re/test_01_re_comfy_create_and_get.py
View File

@ -1,186 +1,44 @@
import json
import logging
import time
import uuid
import config as config
from utils.api import Api
from utils.helper import get_endpoint_comfy_async
logger = logging.getLogger(__name__)
id = "0354bca4-7e04-4c35-86e5-5fab18879376"
prompt_id = str(uuid.uuid4())
class TestTxt2ImgCreateAndGetComfyE2E:
class TestReTxt2ImgReCreateAndGetComfyE2E:
def setup_class(self):
self.api = Api(config)
self.api.feat_oas_schema()
self.ep_name = get_endpoint_comfy_async(self.api)
@classmethod
def teardown_class(self):
pass
def test_0_comfy_txt2img_async_create(self):
headers = {
"x-api-key": config.api_key,
}
with open('./data/api_params/comfy_workflow.json', 'r') as f:
headers = {
"x-api-key": config.api_key,
}
workflow = json.load(f)
workflow['prompt_id'] = prompt_id
workflow['endpoint_name'] = self.ep_name
payload = json.dumps({
"need_sync": True,
"prompt": {
"4": {
"inputs": {
"ckpt_name": "sdXL_v10VAEFix.safetensors"
},
"class_type": "CheckpointLoaderSimple"
},
"5": {
"inputs": {
"width": 1024,
"height": 1024,
"batch_size": 1
},
"class_type": "EmptyLatentImage"
},
"6": {
"inputs": {
"text": "evening sunset scenery blue sky nature, glass bottle with a galaxy in it",
"clip": [
"4",
1
]
},
"class_type": "CLIPTextEncode"
},
"7": {
"inputs": {
"text": "text, watermark",
"clip": [
"4",
1
]
},
"class_type": "CLIPTextEncode"
},
"10": {
"inputs": {
"add_noise": "enable",
"noise_seed": 721897303308196,
"steps": 25,
"cfg": 8,
"sampler_name": "euler",
"scheduler": "normal",
"start_at_step": 0,
"end_at_step": 20,
"return_with_leftover_noise": "enable",
"model": [
"4",
0
],
"positive": [
"6",
0
],
"negative": [
"7",
0
],
"latent_image": [
"5",
0
]
},
"class_type": "KSamplerAdvanced"
},
"11": {
"inputs": {
"add_noise": "disable",
"noise_seed": 0,
"steps": 25,
"cfg": 8,
"sampler_name": "euler",
"scheduler": "normal",
"start_at_step": 20,
"end_at_step": 10000,
"return_with_leftover_noise": "disable",
"model": [
"12",
0
],
"positive": [
"15",
0
],
"negative": [
"16",
0
],
"latent_image": [
"10",
0
]
},
"class_type": "KSamplerAdvanced"
},
"12": {
"inputs": {
"ckpt_name": "sdXL_v10VAEFix.safetensors"
},
"class_type": "CheckpointLoaderSimple"
},
"15": {
"inputs": {
"text": "evening sunset scenery blue sky nature, glass bottle with a galaxy in it",
"clip": [
"12",
1
]
},
"class_type": "CLIPTextEncode"
},
"16": {
"inputs": {
"text": "text, watermark",
"clip": [
"12",
1
]
},
"class_type": "CLIPTextEncode"
},
"17": {
"inputs": {
"samples": [
"11",
0
],
"vae": [
"12",
2
]
},
"class_type": "VAEDecode"
},
"19": {
"inputs": {
"filename_prefix": "ComfyUI",
"images": [
"17",
0
]
},
"class_type": "SaveImage"
}
},
"prompt_id": id,
"endpoint_name": config.comfy_async_ep_name
})
resp = self.api.create_execute(headers=headers, data=json.loads(payload))
assert resp.json()["statusCode"] == 201, resp.dumps()
logger.info(f"resp: {resp.json()['data']['prompt_id']}")
resp = self.api.create_execute(headers=headers, data=workflow)
assert resp.status_code in [200, 201], resp.dumps()
assert resp.json()['data']['prompt_id'] == prompt_id, resp.dumps()
def test_2_comfy_txt2img_get(self):
headers = {
"x-api-key": config.api_key,
}
self.api.get_execute_job(headers=headers, prompt_id=id)
time.sleep(15)
self.api.get_execute_job(headers=headers, prompt_id=prompt_id)

184
test/test_08_api_inference_re/test_02_re_comfy_query_and_delete.py
View File

@ -4,186 +4,42 @@ import uuid
import config as config
from utils.api import Api
from utils.helper import get_endpoint_comfy_async
logger = logging.getLogger(__name__)
prompt_id = str(uuid.uuid4())
class TestTxt2ImgQueryAndDeleteComfyE2E:
class TestReTxt2ImgReQueryAndDeleteComfyE2E:
def setup_class(self):
self.api = Api(config)
self.api.feat_oas_schema()
self.ep_name = get_endpoint_comfy_async(self.api)
@classmethod
def teardown_class(self):
pass
def test_1_comfy_re_txt2img_async_batch_create(self):
def test_1_comfy_txt2img_async_batch_create(self):
count = 20
for i in range(count):
self.comfy_re_txt2img_async_create()
self.comfy_txt2img_async_create()
def comfy_re_txt2img_async_create(self):
headers = {
"x-api-key": config.api_key,
}
def comfy_txt2img_async_create(self):
with open('./data/api_params/comfy_workflow.json', 'r') as f:
headers = {
"x-api-key": config.api_key,
}
workflow = json.load(f)
workflow['prompt_id'] = prompt_id
workflow['endpoint_name'] = self.ep_name
payload = json.dumps({
"need_sync": True,
"prompt": {
"4": {
"inputs": {
"ckpt_name": "sdXL_v10VAEFix.safetensors"
},
"class_type": "CheckpointLoaderSimple"
},
"5": {
"inputs": {
"width": 1024,
"height": 1024,
"batch_size": 1
},
"class_type": "EmptyLatentImage"
},
"6": {
"inputs": {
"text": "evening sunset scenery blue sky nature, glass bottle with a galaxy in it",
"clip": [
"4",
1
]
},
"class_type": "CLIPTextEncode"
},
"7": {
"inputs": {
"text": "text, watermark",
"clip": [
"4",
1
]
},
"class_type": "CLIPTextEncode"
},
"10": {
"inputs": {
"add_noise": "enable",
"noise_seed": 721897303308196,
"steps": 25,
"cfg": 8,
"sampler_name": "euler",
"scheduler": "normal",
"start_at_step": 0,
"end_at_step": 20,
"return_with_leftover_noise": "enable",
"model": [
"4",
0
],
"positive": [
"6",
0
],
"negative": [
"7",
0
],
"latent_image": [
"5",
0
]
},
"class_type": "KSamplerAdvanced"
},
"11": {
"inputs": {
"add_noise": "disable",
"noise_seed": 0,
"steps": 25,
"cfg": 8,
"sampler_name": "euler",
"scheduler": "normal",
"start_at_step": 20,
"end_at_step": 10000,
"return_with_leftover_noise": "disable",
"model": [
"12",
0
],
"positive": [
"15",
0
],
"negative": [
"16",
0
],
"latent_image": [
"10",
0
]
},
"class_type": "KSamplerAdvanced"
},
"12": {
"inputs": {
"ckpt_name": "sdXL_v10VAEFix.safetensors"
},
"class_type": "CheckpointLoaderSimple"
},
"15": {
"inputs": {
"text": "evening sunset scenery blue sky nature, glass bottle with a galaxy in it",
"clip": [
"12",
1
]
},
"class_type": "CLIPTextEncode"
},
"16": {
"inputs": {
"text": "text, watermark",
"clip": [
"12",
1
]
},
"class_type": "CLIPTextEncode"
},
"17": {
"inputs": {
"samples": [
"11",
0
],
"vae": [
"12",
2
]
},
"class_type": "VAEDecode"
},
"19": {
"inputs": {
"filename_prefix": "ComfyUI",
"images": [
"17",
0
]
},
"class_type": "SaveImage"
}
},
"prompt_id": str(uuid.uuid4()),
"endpoint_name": config.comfy_async_ep_name
})
resp = self.api.create_execute(headers=headers, data=workflow)
assert resp.status_code in [200, 201], resp.dumps()
assert resp.json()['data']['prompt_id'] == prompt_id, resp.dumps()
resp = self.api.create_execute(headers=headers, data=json.loads(payload))
assert resp.json()["statusCode"] == 201
logger.info(f"execute created: {resp.json()['data']['prompt_id']}")
def test_2_comfy_re_txt2img_list(self):
def test_2_comfy_txt2img_list(self):
last_evaluated_key = None
while True:
resp = self.executes_list(exclusive_start_key=last_evaluated_key)
@ -202,7 +58,7 @@ class TestTxt2ImgQueryAndDeleteComfyE2E:
params={"exclusive_start_key": exclusive_start_key, "limit": 20})
return resp
def test_4_comfy_re_txt2img_clean(self):
def test_4_comfy_txt2img_clean(self):
last_evaluated_key = None
while True:
resp = self.executes_list(exclusive_start_key=last_evaluated_key)
@ -225,7 +81,7 @@ class TestTxt2ImgQueryAndDeleteComfyE2E:
if not last_evaluated_key:
break
def test_5_comfy_re_txt2img_check_clean(self):
def test_5_comfy_txt2img_check_clean(self):
resp = self.executes_list()
executes = resp.json()['data']['executes']
assert len(executes) == 0, resp.dumps()

120
test/test_10_local_only/test_00_comfy_endpoint_create.py
View File

@ -1,120 +0,0 @@
from __future__ import print_function
import logging
import time
from datetime import datetime
from datetime import timedelta
import config as config
from utils.api import Api
logger = logging.getLogger(__name__)
class TestComfyEndpointCreateE2E:
def setup_class(self):
self.api = Api(config)
self.api.feat_oas_schema()
@classmethod
def teardown_class(self):
pass
def test_1_clean_all_endpoints(self):
headers = {
"x-api-key": config.api_key,
"username": config.username
}
resp = self.api.list_endpoints(headers=headers)
endpoints = resp.json()['data']['endpoints']
for endpoint in endpoints:
endpoint_name = endpoint['endpoint_name']
while True:
data = {
"endpoint_name_list": [
endpoint_name
],
}
resp = self.api.delete_endpoints(headers=headers, data=data)
time.sleep(5)
if resp.status_code == 400:
logger.info(resp.json()['message'])
continue
else:
break
def test_3_create_confy_endpoint_async(self):
headers = {
"x-api-key": config.api_key,
"username": config.username
}
data = {
"endpoint_name": 'mutil-gpus',
"service_type": "comfy",
"endpoint_type": "Async",
"instance_type": 'ml.g5.12xlarge',
"initial_instance_count": 1,
"autoscaling_enabled": False,
"assign_to_roles": [config.role_comfy_async],
"creator": config.username
}
resp = self.api.create_endpoint(headers=headers, data=data)
assert 'data' in resp.json(), resp.dumps()
assert resp.json()["data"]["endpoint_status"] == "Creating", resp.dumps()
def test_1_list_endpoints_status(self):
headers = {
"x-api-key": config.api_key,
"username": config.username
}
params = {
"username": config.username
}
resp = self.api.list_endpoints(headers=headers, params=params)
assert resp.status_code == 200, resp.dumps()
endpoints = resp.json()['data']["endpoints"]
assert len(endpoints) >= 0
timeout = datetime.now() + timedelta(minutes=40)
while datetime.now() < timeout:
result = self.endpoints_wait_for_in_service()
if result:
break
time.sleep(50)
else:
raise Exception("Create Endpoint timed out after 40 minutes.")
def endpoints_wait_for_in_service(self):
headers = {
"x-api-key": config.api_key,
"username": config.username
}
params = {
"username": config.username
}
resp = self.api.list_endpoints(headers=headers, params=params)
assert resp.status_code == 200, resp.dumps()
for endpoint in resp.json()['data']["endpoints"]:
endpoint_name = endpoint["endpoint_name"]
if endpoint["endpoint_status"] == "Failed":
raise Exception(f"{endpoint_name} is {endpoint['endpoint_status']}")
if endpoint["endpoint_status"] != "InService":
logger.info(f"{endpoint_name} is {endpoint['endpoint_status']}")
return False
else:
return True
return False