90 lines
3.7 KiB
Python
90 lines
3.7 KiB
Python
import logging
|
||
import os
|
||
|
||
import boto3
|
||
from boto3.dynamodb.conditions import Key
|
||
from aws_lambda_powertools import Tracer
|
||
|
||
from common.ddb_service.client import DynamoDbUtilsService
|
||
from common.response import ok, no_content
|
||
from libs.utils import response_error
|
||
|
||
tracer = Tracer()
|
||
logger = logging.getLogger(__name__)
|
||
logger.setLevel(os.environ.get('LOG_LEVEL') or logging.ERROR)
|
||
|
||
region = os.environ.get('AWS_REGION')
|
||
bucket_name = os.environ.get('S3_BUCKET_NAME')
|
||
|
||
inference_monitor_table = os.environ.get('INSTANCE_MONITOR_TABLE')
|
||
sync_table = os.environ.get('SYNC_TABLE')
|
||
config_table = os.environ.get('CONFIG_TABLE')
|
||
|
||
ddb_service = DynamoDbUtilsService(logger=logger)
|
||
|
||
|
||
dynamodb = boto3.resource('dynamodb')
|
||
sync_table = dynamodb.Table(sync_table)
|
||
|
||
|
||
def get_last_ddb_sync_record(endpoint_name):
|
||
sync_response = sync_table.query(
|
||
KeyConditionExpression=Key('endpoint_name').eq(endpoint_name),
|
||
Limit=1,
|
||
ScanIndexForward=False
|
||
)
|
||
latest_sync_record = sync_response['Items'][0] if ('Items' in sync_response
|
||
and len(sync_response['Items']) > 0) else None
|
||
if latest_sync_record:
|
||
logger.info(f"latest_sync_record is：{latest_sync_record}")
|
||
return latest_sync_record
|
||
|
||
logger.info("no latest_sync_record found")
|
||
return None
|
||
|
||
|
||
def check_sync_and_instance_from_ddb(endpoint_name):
|
||
sync_record = get_last_ddb_sync_record(endpoint_name)
|
||
logger.debug(f'sync_record is : {sync_record}')
|
||
if sync_record is None or len(sync_record) == 0:
|
||
logger.info("No sync record for check_sync_and_instance_from_ddb return False")
|
||
return False
|
||
instance_count = int(sync_record['instance_count'])
|
||
instance_monitor_records_resp = ddb_service.query_items(inference_monitor_table,
|
||
key_values={"endpoint_name": endpoint_name})
|
||
logger.info(instance_monitor_records_resp)
|
||
if (instance_monitor_records_resp is None):
|
||
logger.info(f"No instance record for check_sync_and_instance_from_ddb return False")
|
||
logger.debug(f" {instance_monitor_records_resp}")
|
||
return False
|
||
if len(instance_monitor_records_resp) < instance_count:
|
||
logger.info(f"No enough instance record for check_sync_and_instance_from_ddb return False")
|
||
logger.debug(f"false {instance_monitor_records_resp} {sync_record}")
|
||
return False
|
||
list_sync_instance = []
|
||
for item in instance_monitor_records_resp:
|
||
logger.debug(f"instance_monitor_records_resp :{item} {str(item['last_sync_request_id'].get('S'))} {sync_record['request_id']}")
|
||
if str(item['last_sync_request_id'].get('S')) == sync_record['request_id']:
|
||
list_sync_instance.append(item)
|
||
if len(list_sync_instance) < instance_count:
|
||
logger.info(f"{sync_record['request_id']} No enough instance record {list_sync_instance} for check_sync_and_instance_from_ddb return False")
|
||
# logger.debug(f"false {instance_monitor_records_resp} {sync_record}")
|
||
return False
|
||
logger.info(f"check_sync_and_instance_from_ddb return True")
|
||
return True
|
||
|
||
|
||
@tracer.capture_lambda_handler
|
||
def handler(event, ctx):
|
||
try:
|
||
logger.info(f"get prepare start... Received event: {event}")
|
||
logger.info(f"Received ctx: {ctx}")
|
||
if 'pathParameters' not in event or not event['pathParameters'] or not event['pathParameters']['id']:
|
||
return no_content()
|
||
endpoint_name = event['pathParameters']['id']
|
||
sync_success = check_sync_and_instance_from_ddb(endpoint_name)
|
||
logger.info(f"get prepare end... response: {sync_success}")
|
||
return ok(data={"prepareSuccess": sync_success})
|
||
except Exception as e:
|
||
return response_error(e)
|