Skip to content

Commit a354340

Browse files
authored
Merge pull request #138 from terhunej/zetl-updates
Updated zetl scripts
2 parents b42579c + 6f1e908 commit a354340

File tree

3 files changed

+151
-21
lines changed

3 files changed

+151
-21
lines changed

static/files/dynamodb-opensearch-zetl/OpenSearchPipeline/bedrock_query.py

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -67,21 +67,23 @@ def product_recommend(input_text, language, region, opensearch_host, model_id):
6767
llm_prompt = 'Human: 你现在是一个导购客服,需要帮助客户推荐商品,根据商品的描述信息,给客户推荐具体的商品名称和编号. 客户的问题如下: ' + input_text + ',你必须基于以下商品信息进行推荐.适当的时候如果客户问题不清晰,可以反问一些关键信息.' + str(es_res) + ' Assistant:'
6868

6969
llm_request_body = json.dumps({
70-
"prompt": llm_prompt,
71-
"max_tokens_to_sample": 4000,
70+
"anthropic_version": "bedrock-2023-05-31",
71+
"messages": [
72+
{"role": "user", "content": llm_prompt}
73+
],
74+
"max_tokens": 4000,
7275
"temperature": 0.1,
7376
"top_p": 0.9,
7477
})
7578

76-
modelId = 'anthropic.claude-v2:1'
79+
modelId = 'anthropic.claude-haiku-4-5-20251001-v1:0'
7780
accept = 'application/json'
7881
contentType = 'application/json'
7982

8083
response = brt.invoke_model(body=llm_request_body, modelId=modelId, accept=accept, contentType=contentType)
81-
8284
response_body = json.loads(response.get('body').read())
83-
84-
llm_result = response_body.get('completion')
85+
#llm_result = response_body.get('completion')
86+
llm_result = response_body.get('content', [{}])[0].get('text', '')
8587
return llm_result,es_response
8688

8789
def reviews_analytis(input_text, language, region, opensearch_host, model_id):
@@ -135,21 +137,23 @@ def reviews_analytis(input_text, language, region, opensearch_host, model_id):
135137
llm_prompt = 'Human: 你现在是一个导购客服,需要帮助客户分析商品的评价,根据商品过去的评论信息,给客户做评论总结,主要关注商品的评分,评论内容的情绪表达. 客户的问题如下: ' + input_text + ',你必须基于以下商品评价信息进行总结.适当的时候如果客户问题不清晰,可以反问一些关键信息.' + str(es_res) + ' Assistant:'
136138

137139
llm_request_body = json.dumps({
138-
"prompt": llm_prompt,
139-
"max_tokens_to_sample": 4000,
140+
"anthropic_version": "bedrock-2023-05-31",
141+
"messages": [
142+
{"role": "user", "content": llm_prompt}
143+
],
144+
"max_tokens": 4000,
140145
"temperature": 0.1,
141146
"top_p": 0.9,
142147
})
143148

144-
modelId = 'anthropic.claude-v2:1'
149+
modelId = 'anthropic.claude-haiku-4-5-20251001-v1:0'
145150
accept = 'application/json'
146151
contentType = 'application/json'
147152

148153
response = brt.invoke_model(body=llm_request_body, modelId=modelId, accept=accept, contentType=contentType)
149-
150154
response_body = json.loads(response.get('body').read())
151155

152-
llm_result = response_body.get('completion')
156+
llm_result = response_body.get('content', [{}])[0].get('text', '')
153157
return llm_result,es_response
154158

155159

static/files/dynamodb-opensearch-zetl/OpenSearchPipeline/credentials.sh

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
aws cloudformation describe-stacks --stack-name dynamodb-opensearch-setup --query "Stacks[0].Outputs" --output table > CloudFormation-Outputs.txt
2+
export OPENSEARCH_ENDPOINT=`aws cloudformation describe-stacks --stack-name dynamodb-opensearch-setup --query "Stacks[0].Outputs[?OutputKey=='OSDomainEndpoint'].OutputValue" --output text`
13
TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
24
INSTANCE_ROLE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/iam/security-credentials/)
35
RESULTS=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/iam/security-credentials/${INSTANCE_ROLE})
@@ -18,4 +20,5 @@ echo "METADATA_AWS_ACCESS_KEY_ID: $AccessKeyId"
1820
echo "METADATA_AWS_SECRET_ACCESS_KEY: $SecretAccessKey"
1921
echo "METADATA_AWS_SESSION_TOKEN: $Token"
2022
echo "METADATA_AWS_REGION: $Region"
21-
echo "METADATA_AWS_ROLE: $Role"
23+
echo "METADATA_AWS_ROLE: $Role"
24+
echo "OPENSEARCH_ENDPOINT: $OPENSEARCH_ENDPOINT"

static/files/dynamodb-opensearch-zetl/dynamodb-opensearch-setup.yaml

Lines changed: 132 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,106 @@ Resources:
2121
PasswordLength: 16
2222
ExcludeCharacters: "\"'@/\\"
2323

24+
SecretPlaintextLambdaRole:
25+
Type: AWS::IAM::Role
26+
Properties:
27+
AssumeRolePolicyDocument:
28+
Version: 2012-10-17
29+
Statement:
30+
- Effect: Allow
31+
Principal:
32+
Service: !Sub lambda.${AWS::URLSuffix}
33+
Action: sts:AssumeRole
34+
ManagedPolicyArns:
35+
- !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
36+
Policies:
37+
- PolicyName: AwsSecretsManager
38+
PolicyDocument:
39+
Version: 2012-10-17
40+
Statement:
41+
- Effect: Allow
42+
Action:
43+
- secretsmanager:GetSecretValue
44+
Resource:
45+
- !Ref OpenSearchSecret
46+
47+
SecretPlaintextLambda:
48+
Type: AWS::Lambda::Function
49+
Metadata:
50+
cfn_nag:
51+
rules_to_suppress:
52+
- id: W58
53+
reason: Warning incorrectly reported. The role associated with the Lambda function has the AWSLambdaBasicExecutionRole managed policy attached, which includes permission to write CloudWatch Logs. See https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AWSLambdaBasicExecutionRole.html
54+
- id: W89
55+
reason: CloudFormation custom function does not need the scaffolding of a VPC, to do so would add unnecessary complexity
56+
- id: W92
57+
reason: CloudFormation custom function does not need reserved concurrent executions, to do so would add unnecessary complexity
58+
Properties:
59+
Description: Return the value of the secret
60+
Handler: index.lambda_handler
61+
Runtime: python3.13
62+
MemorySize: 128
63+
Timeout: 10
64+
Architectures:
65+
- arm64
66+
Role: !GetAtt SecretPlaintextLambdaRole.Arn
67+
Code:
68+
ZipFile: |
69+
import boto3
70+
import json
71+
import cfnresponse
72+
import logging
73+
74+
logger = logging.getLogger()
75+
logger.setLevel(logging.INFO)
76+
77+
def is_valid_json(json_string):
78+
logger.debug(f'Calling is_valid_jason:{json_string}')
79+
try:
80+
json.loads(json_string)
81+
logger.info('Secret is in json format')
82+
return True
83+
except json.JSONDecodeError:
84+
logger.info('Secret is in string format')
85+
return False
86+
87+
def lambda_handler(event, context):
88+
logger.debug(f'event: {event}')
89+
logger.debug(f'context: {context}')
90+
try:
91+
if event['RequestType'] == 'Delete':
92+
cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}, reason='No action to take')
93+
else:
94+
resource_properties = event['ResourceProperties']
95+
secret_name = resource_properties['SecretArn']
96+
secrets_mgr = boto3.client('secretsmanager')
97+
98+
logger.info('Getting secret from %s', secret_name)
99+
100+
secret = secrets_mgr.get_secret_value(SecretId = secret_name)
101+
logger.debug(f'secret: {secret}')
102+
secret_value = secret['SecretString']
103+
104+
responseData = {}
105+
if is_valid_json(secret_value):
106+
responseData = secret_value
107+
else:
108+
responseData = {'secret': secret_value}
109+
logger.debug(f'responseData: {responseData}')
110+
logger.debug(f'type(responseData): {type(responseData)}')
111+
cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData=json.loads(responseData), reason='OK', noEcho=True)
112+
except Exception as e:
113+
logger.error(e)
114+
cfnresponse.send(event, context, cfnresponse.FAILED, responseData={}, reason=str(e))
115+
116+
SecretPlaintext:
117+
Type: Custom::SecretPlaintextLambda
118+
Properties:
119+
ServiceToken: !GetAtt SecretPlaintextLambda.Arn
120+
ServiceTimeout: 15
121+
SecretArn: !Ref OpenSearchSecret
122+
123+
24124
ProductDetailsTable:
25125
Type: AWS::DynamoDB::Table
26126
Properties:
@@ -63,7 +163,7 @@ Resources:
63163
Statement:
64164
- Effect: Allow
65165
Principal:
66-
AWS: !ImportValue Cloud9RoleArn
166+
AWS: !ImportValue CodeInstanceRoleArn
67167
Action: 'es:*'
68168
Resource:
69169
!Join
@@ -88,17 +188,40 @@ Resources:
88188
MasterUserPassword:
89189
Fn::Sub: "{{resolve:secretsmanager:${OpenSearchSecret}::password}}"
90190

191+
PipelineBucket:
192+
Type: AWS::S3::Bucket
193+
Metadata:
194+
cfn_nag:
195+
rules_to_suppress:
196+
- id: W35
197+
reason: Access logs aren't needed for this bucket
198+
DeletionPolicy: Delete
199+
Properties:
200+
AccessControl: Private
201+
BucketEncryption:
202+
ServerSideEncryptionConfiguration:
203+
- ServerSideEncryptionByDefault:
204+
SSEAlgorithm: AES256
205+
PublicAccessBlockConfiguration:
206+
BlockPublicAcls: true
207+
BlockPublicPolicy: true
208+
IgnorePublicAcls: true
209+
RestrictPublicBuckets: true
210+
91211
Outputs:
92212

93-
SecretConsoleLink:
94-
Description: URL to the secret in AWS Secrets Manager console
95-
Value: !Sub "https://${AWS::Region}.console.aws.amazon.com/secretsmanager/secret?name=${OpenSearchSecret}&region=${AWS::Region}"
96-
Cloud9IdeUrl:
97-
Description: URL to launch the Cloud9 IDE
98-
Value: !ImportValue Cloud9IdeUrl
213+
VSCodeUrl:
214+
Description: URL to launch the VSCode IDE
215+
Value: !ImportValue VSCodeUrl
216+
VSCodePassword:
217+
Description: VSCode Server Password (stored in AWS Secrets Manager)
218+
Value: !ImportValue VSCodePassword
99219
OSDashboardsURL:
100220
Description: URL to the OpenSearch Dashboards
101221
Value: !Sub "https://${OpenSearchServiceDomain.DomainEndpoint}/_dashboards/"
222+
OpenSearchPassword:
223+
Description: OpenSearch Password (stored in AWS Secrets Manager)
224+
Value: !GetAtt SecretPlaintext.password
102225
OSDomainEndpoint:
103226
Description: The endpoint of the OpenSearch domain.
104227
Value: !Sub "https://${OpenSearchServiceDomain.DomainEndpoint}"
@@ -112,7 +235,7 @@ Outputs:
112235
Value: !GetAtt ProductDetailsTable.Arn
113236
Role:
114237
Description: "ARN of the Role used to provide access"
115-
Value: !ImportValue Cloud9RoleArn
238+
Value: !ImportValue CodeInstanceRoleArn
116239
S3Bucket:
117240
Description: "Name of the S3 Bucket"
118-
Value: !ImportValue Cloud9LogBucket
241+
Value: !Ref PipelineBucket

0 commit comments

Comments
 (0)