168 lines
6.4 KiB
Bash
Executable File
168 lines
6.4 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
chmod +x env.properties
|
|
|
|
source env.properties
|
|
|
|
echo "----------------------------------------------------------------"
|
|
printenv
|
|
|
|
aws cloudformation delete-stack --stack-name "comfy-stack"
|
|
aws cloudformation delete-stack --stack-name "webui-stack"
|
|
|
|
if [ -z "$ACCOUNT_ID" ]; then
|
|
echo "ACCOUNT_ID is not set"
|
|
exit 1
|
|
fi
|
|
|
|
cd stable-diffusion-aws-extension/test || exit
|
|
|
|
ls -la
|
|
|
|
echo "----------------------------------------------------------------"
|
|
properties=("Account: $ACCOUNT_ID")
|
|
properties+=("Version/Branch: $CODE_BRANCH")
|
|
properties+=("Region: $AWS_DEFAULT_REGION")
|
|
properties+=("Deploy Method: $DEPLOY_STACK")
|
|
|
|
if [ -n "$DEPLOY_DURATION_TIME" ]; then
|
|
DEPLOY_DURATION_TIME=$(printf "%dm%ds\n" $(($DEPLOY_DURATION_TIME/60)) $(($DEPLOY_DURATION_TIME%60)))
|
|
properties+=("Deploy Duration: $DEPLOY_DURATION_TIME")
|
|
fi
|
|
|
|
|
|
if [ "$CODEBUILD_BUILD_SUCCEEDING" -eq 0 ]; then
|
|
result="Failed"
|
|
CASE_PASSED_RESULT="Failed"
|
|
else
|
|
result="Passed"
|
|
CASE_PASSED_RESULT="Passed"
|
|
fi
|
|
|
|
properties+=("Result: ${result}")
|
|
|
|
if [ -f "detailed_report.json" ]; then
|
|
CASE_TOTAL=$(cat detailed_report.json | jq -r '.summary.total')
|
|
CASE_PASSED=$(cat detailed_report.json | jq -r '.summary.passed')
|
|
properties+=("Total Cases: ${CASE_TOTAL}")
|
|
properties+=("Passed Cases: ${CASE_PASSED}")
|
|
CASE_PASSED_RESULT="Passed $CASE_PASSED Cases"
|
|
CASE_SKIPPED=$(cat detailed_report.json | jq -r '.summary.skipped')
|
|
if [ -n "$CASE_SKIPPED" ]; then
|
|
properties+=("Skipped Cases: ${CASE_SKIPPED}")
|
|
fi
|
|
fi
|
|
|
|
if [ "$CODEBUILD_BUILD_SUCCEEDING" -eq 0 ]; then
|
|
CASE_PASSED_RESULT="Failed"
|
|
fi
|
|
|
|
if [ -n "$API_TEST_STARTED_TIME" ]; then
|
|
CURRENT_TIME=$(date +%s)
|
|
API_TEST_DURATION_TIME=$(( $CURRENT_TIME - $API_TEST_STARTED_TIME ))
|
|
API_TEST_DURATION_TIME=$(printf "%dm%ds\n" $(($API_TEST_DURATION_TIME/60)) $(($API_TEST_DURATION_TIME%60)))
|
|
properties+=("Test Duration: ${API_TEST_DURATION_TIME}")
|
|
fi
|
|
|
|
if [ "$result" = "Passed" ]; then
|
|
properties+=("G5 Instance: OK")
|
|
properties+=("G4 Instance: OK")
|
|
properties+=("train Task: OK")
|
|
properties+=("Lora Task: OK")
|
|
properties+=("txt2img Task: OK")
|
|
properties+=("img2img Task: OK")
|
|
properties+=("rembg Task: OK")
|
|
properties+=("extra-single-image Task: OK")
|
|
properties+=("train_instance_type: ${TRAIN_INSTANCE_TYPE}")
|
|
|
|
if [ "$CLEAN_RESOURCES" = "yes" ]; then
|
|
aws s3 rb "s3://$API_BUCKET" --force
|
|
aws s3 rb "s3://sagemaker-$AWS_DEFAULT_REGION-$ACCOUNT_ID" --force
|
|
|
|
aws dynamodb delete-table --table-name "CheckpointTable" | jq
|
|
aws dynamodb delete-table --table-name "DatasetInfoTable" | jq
|
|
aws dynamodb delete-table --table-name "DatasetItemTable" | jq
|
|
aws dynamodb delete-table --table-name "MultiUserTable" | jq
|
|
aws dynamodb delete-table --table-name "SDEndpointDeploymentJobTable" | jq
|
|
aws dynamodb delete-table --table-name "SDInferenceJobTable" | jq
|
|
aws dynamodb delete-table --table-name "TrainingTable" | jq
|
|
|
|
aws sns delete-topic --topic-arn "arn:aws:sns:$AWS_DEFAULT_REGION:$ACCOUNT_ID:failureCreateModel" | jq
|
|
aws sns delete-topic --topic-arn "arn:aws:sns:$AWS_DEFAULT_REGION:$ACCOUNT_ID:ReceiveSageMakerInferenceError" | jq
|
|
aws sns delete-topic --topic-arn "arn:aws:sns:$AWS_DEFAULT_REGION:$ACCOUNT_ID:ReceiveSageMakerInferenceSuccess" | jq
|
|
aws sns delete-topic --topic-arn "arn:aws:sns:$AWS_DEFAULT_REGION:$ACCOUNT_ID:sde-api-test-result" | jq
|
|
aws sns delete-topic --topic-arn "arn:aws:sns:$AWS_DEFAULT_REGION:$ACCOUNT_ID:StableDiffusionSnsUserTopic" | jq
|
|
aws sns delete-topic --topic-arn "arn:aws:sns:$AWS_DEFAULT_REGION:$ACCOUNT_ID:successCreateModel" | jq
|
|
fi
|
|
|
|
fi
|
|
|
|
if [ -f "/tmp/txt2img_sla_report.json" ]; then
|
|
txt2img_sla_report=$(cat /tmp/txt2img_sla_report.json)
|
|
|
|
sla_model_id=$(echo "$txt2img_sla_report" | jq -r '.model_id')
|
|
sla_instance_type=$(echo "$txt2img_sla_report" | jq -r '.instance_type')
|
|
sla_instance_count=$(echo "$txt2img_sla_report" | jq -r '.instance_count')
|
|
sla_count=$(echo "$txt2img_sla_report" | jq -r '.count')
|
|
sla_succeed=$(echo "$txt2img_sla_report" | jq -r '.succeed')
|
|
sla_failed=$(echo "$txt2img_sla_report" | jq -r '.failed')
|
|
sla_success_rate=$(echo "$txt2img_sla_report" | jq -r '.success_rate')
|
|
sla_max_duration=$(echo "$txt2img_sla_report" | jq -r '.max_duration')
|
|
sla_min_duration=$(echo "$txt2img_sla_report" | jq -r '.min_duration')
|
|
sla_avg_duration=$(echo "$txt2img_sla_report" | jq -r '.avg_duration')
|
|
|
|
properties+=("\\n[Inference SLA]")
|
|
properties+=("model_id: ${sla_model_id}")
|
|
properties+=("instance_type: ${sla_instance_type}")
|
|
properties+=("instance_count: ${sla_instance_count}")
|
|
properties+=("count: ${sla_count}")
|
|
properties+=("succeed: ${sla_succeed}")
|
|
properties+=("failed: ${sla_failed}")
|
|
properties+=("success_rate: ${sla_success_rate}")
|
|
properties+=("max_duration_seconds: ${sla_max_duration}")
|
|
properties+=("min_duration_seconds: ${sla_min_duration}")
|
|
properties+=("avg_duration_seconds: ${sla_avg_duration}")
|
|
|
|
failed_list=$(echo "$txt2img_sla_report" | jq -r '.failed_list')
|
|
properties+=("${failed_list}")
|
|
fi
|
|
|
|
properties+=("SNS_ARN: ${SNS_ARN}")
|
|
|
|
if [ -f "report-${CODEBUILD_BUILD_NUMBER}.html" ]; then
|
|
report_file="report-${CODEBUILD_BUILD_NUMBER}.html"
|
|
aws s3 cp "$report_file" "s3://$API_BUCKET/test_report/"
|
|
properties+=("Report: s3://$API_BUCKET/test_report/$report_file")
|
|
fi
|
|
|
|
properties+=("CodeBuildUrl: ${CODEBUILD_BUILD_URL}")
|
|
|
|
message=""
|
|
for property in "${properties[@]}"; do
|
|
message="${message}${property}\\n\\n"
|
|
done
|
|
|
|
properties+=("CLEAN_RESOURCES: ${CLEAN_RESOURCES}")
|
|
properties+=("PYTHON_311_VERSION: ${PYTHON_311_VERSION}")
|
|
properties+=("PYTHON_PIP_VERSION: ${PYTHON_PIP_VERSION}")
|
|
properties+=("CODEBUILD_BUILD_IMAGE: ${CODEBUILD_BUILD_IMAGE}")
|
|
|
|
if [ -n "$SNS_ARN" ]; then
|
|
unset AWS_PROFILE
|
|
echo -e "$message"
|
|
aws sns publish \
|
|
--region "$SNS_REGION" \
|
|
--topic-arn "$SNS_ARN" \
|
|
--message-structure json \
|
|
--subject "SD&Comfy $CODE_BRANCH $CASE_PASSED_RESULT $AWS_DEFAULT_REGION" \
|
|
--message-attributes '{"key": {"DataType": "String", "StringValue": "value"}}' \
|
|
--message "{\"default\": \"$message\"}"
|
|
fi
|
|
|
|
#if [ "$result" = "Passed" ]; then
|
|
# echo "----------------------------------------------------------------"
|
|
# echo "Delete log groups"
|
|
# echo "----------------------------------------------------------------"
|
|
# aws logs describe-log-groups | jq -r '.logGroups[].logGroupName' | grep 'Extension-for-Stable' | xargs -I {} aws logs delete-log-group --log-group-name {}
|
|
#fi
|