从负载均衡的 EC2 Web 服务器向排队的 EC2 工作人员发送消息
Send message from load balanced EC2 web servers to queued EC2 workers
我有一个我一直在使用的相当简单的 CloudFormation 模板，它最初只包含一个负载平衡的 EC2 Web 服务器集群。但是，现在我希望这些 Web 服务器能够将消息发送到 SQS 消息队列，然后将工作交给一些 EC2 工作服务器。
这是我的设计：
这是我当前的 JSON 模板：
{
"AWSTemplateFormatVersion": "2010-09-09",
"Description": "Stack for MySite",
"Parameters": {
"KeyName": {
"Description": "Key Pair name",
"Type": "AWS::EC2::KeyPair::KeyName",
"Default": "mykey"
},
"SiteID": {
"Description": "A unique identifier for the site.",
"Type": "String",
"AllowedPattern": "[A-Za-z0-9\-]+",
"ConstraintDescription": "Only letters, digits or dash allowed."
},
"SiteTitle": {
"Description": "The title of the site.",
"Type": "String",
"Default": "MySite"
},
"AdminUsername": {
"Description": "A username for admin.",
"Type": "String",
"Default": "admin"
},
"AdminPassword": {
"Description": "A password for admin.",
"Type": "String",
"NoEcho": "true"
},
"AdminEMail": {
"Description": "The email address of the administrator.",
"Type": "String"
}
},
"Mappings": {
"EC2RegionMap": {
"ap-northeast-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-cbf90ecb"},
"ap-southeast-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-68d8e93a"},
"ap-southeast-2": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-fd9cecc7"},
"eu-central-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-a8221fb5"},
"eu-west-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-a10897d6"},
"sa-east-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-b52890a8"},
"us-east-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-1ecae776"},
"us-west-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-d114f295"},
"us-west-2": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-e7527ed7"}
}
},
"Resources": {
"VPC": {
"Type": "AWS::EC2::VPC",
"Properties": {
"CidrBlock": "172.31.0.0/16",
"EnableDnsHostnames": "true"
}
},
"InternetGateway": {
"Type": "AWS::EC2::InternetGateway",
"Properties": {
}
},
"VPCGatewayAttachment": {
"Type": "AWS::EC2::VPCGatewayAttachment",
"Properties": {
"VpcId": {"Ref": "VPC"},
"InternetGatewayId": {"Ref": "InternetGateway"}
}
},
"SubnetA": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"AvailabilityZone": {"Fn::Select": ["0", {"Fn::GetAZs": ""}]},
"CidrBlock": "172.31.38.0/24",
"VpcId": {"Ref": "VPC"}
}
},
"SubnetB": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"AvailabilityZone": {"Fn::Select": ["1", {"Fn::GetAZs": ""}]},
"CidrBlock": "172.31.37.0/24",
"VpcId": {"Ref": "VPC"}
}
},
"WebServerRouteTable": {
"Type": "AWS::EC2::RouteTable",
"Properties": {
"VpcId": {"Ref": "VPC"}
}
},
"RouteTableAssociationA": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"SubnetId": {"Ref": "SubnetA"},
"RouteTableId": {"Ref": "WebServerRouteTable"}
}
},
"RouteTableAssociationB": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"SubnetId": {"Ref": "SubnetB"},
"RouteTableId": {"Ref": "WebServerRouteTable"}
}
},
"RoutePublicNATToInternet": {
"Type": "AWS::EC2::Route",
"Properties": {
"RouteTableId": {"Ref": "WebServerRouteTable"},
"DestinationCidrBlock": "0.0.0.0/0",
"GatewayId": {"Ref": "InternetGateway"}
},
"DependsOn": "VPCGatewayAttachment"
},
"NetworkAcl": {
"Type": "AWS::EC2::NetworkAcl",
"Properties": {
"VpcId": {"Ref": "VPC"}
}
},
"SubnetNetworkAclAssociationA": {
"Type": "AWS::EC2::SubnetNetworkAclAssociation",
"Properties": {
"SubnetId": {"Ref": "SubnetA"},
"NetworkAclId": {"Ref": "NetworkAcl"}
}
},
"SubnetNetworkAclAssociationB": {
"Type": "AWS::EC2::SubnetNetworkAclAssociation",
"Properties": {
"SubnetId": {"Ref": "SubnetB"},
"NetworkAclId": {"Ref": "NetworkAcl"}
}
},
"NetworkAclEntryIngress": {
"Type": "AWS::EC2::NetworkAclEntry",
"Properties": {
"NetworkAclId": {"Ref": "NetworkAcl"},
"RuleNumber": "100",
"Protocol": "-1",
"RuleAction": "allow",
"Egress": "false",
"CidrBlock": "0.0.0.0/0"
}
},
"NetworkAclEntryEgress": {
"Type": "AWS::EC2::NetworkAclEntry",
"Properties": {
"NetworkAclId": {"Ref": "NetworkAcl"},
"RuleNumber": "100",
"Protocol": "-1",
"RuleAction": "allow",
"Egress": "true",
"CidrBlock": "0.0.0.0/0"
}
},
"LoadBalancer": {
"Type": "AWS::ElasticLoadBalancing::LoadBalancer",
"Properties": {
"Subnets": [{"Ref": "SubnetA"}, {"Ref": "SubnetB"}],
"LoadBalancerName": {"Ref": "SiteID"},
"Listeners": [{
"InstancePort": "80",
"InstanceProtocol": "HTTP",
"LoadBalancerPort": "80",
"Protocol": "HTTP"
}],
"HealthCheck": {
"HealthyThreshold": "2",
"Interval": "5",
"Target": "TCP:80",
"Timeout": "3",
"UnhealthyThreshold": "2"
},
"SecurityGroups": [{"Ref": "LoadBalancerSecurityGroup"}],
"Scheme": "internet-facing",
"CrossZone": "true"
},
"DependsOn": "VPCGatewayAttachment"
},
"LoadBalancerSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "mysite-elb-sg",
"VpcId": {"Ref": "VPC"},
"SecurityGroupIngress": [{
"CidrIp": "0.0.0.0/0",
"FromPort": 80,
"IpProtocol": "tcp",
"ToPort": 80
}]
}
},
"WebServerSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "mysite-web-server-sg",
"VpcId": {"Ref": "VPC"},
"SecurityGroupIngress": [{
"CidrIp": "0.0.0.0/0",
"FromPort": 22,
"IpProtocol": "tcp",
"ToPort": 22
}, {
"FromPort": 80,
"IpProtocol": "tcp",
"SourceSecurityGroupId": {"Ref": "LoadBalancerSecurityGroup"},
"ToPort": 80
}]
}
},
"DatabaseSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "mysite-db-sg",
"VpcId": {"Ref": "VPC"},
"SecurityGroupIngress": [{
"IpProtocol": "tcp",
"FromPort": "3306",
"ToPort": "3306",
"SourceSecurityGroupId": {"Ref": "WebServerSecurityGroup"}
}]
}
},
"Database": {
"Type": "AWS::RDS::DBInstance",
"Properties": {
"AllocatedStorage": "5",
"BackupRetentionPeriod": "0",
"DBInstanceClass": "db.t2.micro",
"DBInstanceIdentifier": {"Ref": "SiteID"},
"DBName": "quantsketch",
"Engine": "MySQL",
"MasterUsername": "mysite",
"MasterUserPassword": "mysite",
"VPCSecurityGroups": [{"Fn::GetAtt": ["DatabaseSecurityGroup", "GroupId"]}],
"DBSubnetGroupName": {"Ref": "DBSubnetGroup"}
},
"DependsOn": "VPCGatewayAttachment"
},
"DBSubnetGroup" : {
"Type" : "AWS::RDS::DBSubnetGroup",
"Properties" : {
"DBSubnetGroupDescription" : "DB subnet group",
"SubnetIds": [{"Ref": "SubnetA"}, {"Ref": "SubnetB"}]
}
},
"S3Bucket": {
"Type": "AWS::S3::Bucket",
"Properties": {
"BucketName": {"Ref": "SiteID"},
"WebsiteConfiguration": {
"IndexDocument": "index.html"
}
}
},
"WebServerLaunchConfiguration": {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Metadata": {
"AWS::CloudFormation::Init": {
"config": {
"packages": {
"yum": {
"php": [],
"php-mysql": [],
"mysql": [],
"httpd": []
}
},
"sources": {
"/var/www/html": "https://wordpress.org/wordpress-4.2.4.tar.gz"
},
"files": {
"/tmp/config": {
"content": {"Fn::Join": ["", [
"#!/bin/bash -ex\n",
]]},
"mode": "000500",
"owner": "root",
"group": "root"
}
},
"commands": {
"01_config": {
"command": "/tmp/config",
"cwd": "/var/www/html/wordpress"
}
},
"services": {
"sysvinit": {
"httpd": {
"enabled": "true",
"ensureRunning": "true"
}
}
}
}
}
},
"Properties": {
"ImageId": {"Fn::FindInMap": ["EC2RegionMap", {"Ref": "AWS::Region"}, "AmazonLinuxAMIHVMEBSBacked64bit"]},
"InstanceType": "t2.micro",
"SecurityGroups": [{"Ref": "WebServerSecurityGroup"}],
"KeyName": {"Ref": "KeyName"},
"AssociatePublicIpAddress": true,
"UserData": {"Fn::Base64": {"Fn::Join": ["", [
"#!/bin/bash -ex\n",
"yum update -y aws-cfn-bootstrap\n",
"/opt/aws/bin/cfn-init -v --stack ", {"Ref": "AWS::StackName"}, " --resource WebServerLaunchConfiguration --region ", {"Ref": "AWS::Region"}, "\n",
"/opt/aws/bin/cfn-signal -e $? --stack ", {"Ref": "AWS::StackName"}, " --resource WebServerAutoScalingGroup --region ", {"Ref": "AWS::Region"}, "\n"
]]}}
}
},
"WebServerAutoScalingGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"LoadBalancerNames": [{"Ref": "LoadBalancer"}],
"LaunchConfigurationName": {"Ref": "WebServerLaunchConfiguration"},
"MinSize": "2",
"MaxSize": "4",
"DesiredCapacity": "2",
"Cooldown": "60",
"HealthCheckGracePeriod": "120",
"HealthCheckType": "ELB",
"VPCZoneIdentifier": [{"Ref": "SubnetA"}, {"Ref": "SubnetB"}],
"Tags": [{
"PropagateAtLaunch": true,
"Value": "quantsketch",
"Key": "Name"
}]
},
"CreationPolicy": {
"ResourceSignal": {
"Timeout": "PT10M"
}
},
"DependsOn": "VPCGatewayAttachment"
},
"WebServerScalingUpPolicy": {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"AutoScalingGroupName": {"Ref": "WebServerAutoScalingGroup"},
"Cooldown": "60",
"ScalingAdjustment": "1"
}
},
"WebServerCPUHighAlarm": {
"Type": "AWS::CloudWatch::Alarm",
"Properties": {
"EvaluationPeriods": "1",
"Statistic": "Average",
"Threshold": "80",
"AlarmDescription": "Alarm if CPU load is high.",
"Period": "60",
"AlarmActions": [{"Ref": "WebServerScalingUpPolicy"}],
"Namespace": "AWS/EC2",
"Dimensions": [{
"Name": "AutoScalingGroupName",
"Value": {"Ref": "WebServerAutoScalingGroup"}
}],
"ComparisonOperator": "GreaterThanThreshold",
"MetricName": "CPUUtilization"
}
},
"WebServerScalingDownPolicy": {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"AutoScalingGroupName": {"Ref": "WebServerAutoScalingGroup"},
"Cooldown": "60",
"ScalingAdjustment": "-1"
}
},
"WebServerCPULowAlarm": {
"Type": "AWS::CloudWatch::Alarm",
"Properties": {
"EvaluationPeriods": "1",
"Statistic": "Average",
"Threshold": "25",
"AlarmDescription": "Alarm if CPU load is low.",
"Period": "60",
"AlarmActions": [{"Ref": "WebServerScalingDownPolicy"}],
"Namespace": "AWS/EC2",
"Dimensions": [{
"Name": "AutoScalingGroupName",
"Value": {"Ref": "WebServerAutoScalingGroup"}
}],
"ComparisonOperator": "LessThanThreshold",
"MetricName": "CPUUtilization"
}
}
},
"Outputs": {
"URL": {
"Value": {"Fn::Join": ["", ["http://", {"Fn::GetAtt": ["LoadBalancer", "DNSName"]}, "/quantsketch"]]},
"Description": "QuantSketch URL"
}
}
}
遗憾的是，我无法找到解决此堆栈架构的任何示例模板，尽管我认为这是一个常见问题。
如何连接这两个集群，以便 Web 服务器可以将工作卸载到工作服务器上？任何包含示例、教程或其他内容的建议都会大有帮助。
您通常可以通过在您的负载平衡和工作机器上的应用程序 运行 中调用 AWS APIs/SDKs 来发送和接收来自 SQS 的消息来实现这一点。
选项 A：直接 Send/Receive 通过 SQS
- Web 服务器准备您的消息并调用 SQS SendMessage 将其发送到队列。
- Worker 调用 SQS ReceiveMessage 从队列中接收一批最多 10 条消息，并处理它们。
- 如果您想减少对 SQS 的调用，工作人员调用 SQS DeleteMessage after processing, so that the message isn't retried. Could also call SQS DeleteMessageBatch。
选项 B：Pub/Sub 通过 SNS 发布者到 SQS 订阅者
另一种选择是先通过 SNS 发布您的消息，然后通过 SQS 订阅它。这为您提供了额外的好处，即能够订阅不同的应用程序（甚至人类，通过 SMS/email）到消息，如果您有这样的用例。
- Web 服务器准备您的消息并调用 SNS Publish 将您的消息发布给给定 SNS 主题的所有订阅者。
- SNS 向给定 SNS 主题的所有订阅者发布消息；您的队列就是这样的订阅者，因此消息最终出现在您的队列中。
- Worker 调用 SQS ReceiveMessage 从队列中接收一批最多 10 条消息，并处理它们。
- 如果您想减少对 SQS 的调用，工作人员调用 SQS DeleteMessage after processing, so that the message isn't retried. Could also call SQS DeleteMessageBatch。
讨论
您的 Web 服务器和工作堆栈都需要能够在您的 VPC 之外进行通信，以便访问 SNS and/or SQS 服务。您可以通过使用 NAT Gateways 允许您的实例与全球互联网通信来完成此操作。
您需要提前配置 SNS/SQS 资源。您可以通过 CloudFormation 模板或通过 SNS/SQS API 再次执行此操作。
您没有提到您的应用程序使用的是哪种语言；您将需要阅读您将使用的与我在此答案中链接到的 API 相匹配的特定 SDK 的文档。
要授予您的应用程序访问您的 AWS 账户中这些资源的权限，我建议创建一个 IAM instance profile，它至少有权读取和写入此 SQS 队列，如果需要，还可以选择该 SNS 主题你选择 pub/sub 路线。如果您没有明确向您的应用程序提供凭据，则凭据链将进入 IAM 实例配置文件并继承它。您可以创建这些 IAM 资源作为 CloudFormation 模板的一部分。
还有一些最后的哲学笔记：
不要把这个问题想成连接你的集群；在这两种情况下，他们只会知道 SNS/SQS。这里的目标是您的网络服务器不了解您的工作人员，反之亦然。
我认为您对这个问题的思考过于宽泛，这就是为什么您一直难以找到相关资源的原因。将您的查询分解为更小的工作单元，例如 "How can I send messages to SQS?" 或 "How can I receive messages from SQS?"，我认为您的运气会更好。
进一步阅读
- AWS Documentation - Sending Amazon SNS Messages to Amazon SQS Queues
- AWS Documentation - NAT Instances - 如果 NAT 网关对于您的解决方案来说过于昂贵，您可以阅读此处了解如何设置执行相同功能的 EC2 实例。
- AWS Documentation - Tutorial: Sending and Receiving Messages in Amazon SQS - 使用 Javascript SDK 进行发送和接收
- AWS Documentation - SQS Long Polling - 通过延长消息接收等待时间来减少来自 SQS SDK 的空响应的有用策略。
我有一个我一直在使用的相当简单的 CloudFormation 模板，它最初只包含一个负载平衡的 EC2 Web 服务器集群。但是，现在我希望这些 Web 服务器能够将消息发送到 SQS 消息队列，然后将工作交给一些 EC2 工作服务器。
这是我的设计：
这是我当前的 JSON 模板：
{
"AWSTemplateFormatVersion": "2010-09-09",
"Description": "Stack for MySite",
"Parameters": {
"KeyName": {
"Description": "Key Pair name",
"Type": "AWS::EC2::KeyPair::KeyName",
"Default": "mykey"
},
"SiteID": {
"Description": "A unique identifier for the site.",
"Type": "String",
"AllowedPattern": "[A-Za-z0-9\-]+",
"ConstraintDescription": "Only letters, digits or dash allowed."
},
"SiteTitle": {
"Description": "The title of the site.",
"Type": "String",
"Default": "MySite"
},
"AdminUsername": {
"Description": "A username for admin.",
"Type": "String",
"Default": "admin"
},
"AdminPassword": {
"Description": "A password for admin.",
"Type": "String",
"NoEcho": "true"
},
"AdminEMail": {
"Description": "The email address of the administrator.",
"Type": "String"
}
},
"Mappings": {
"EC2RegionMap": {
"ap-northeast-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-cbf90ecb"},
"ap-southeast-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-68d8e93a"},
"ap-southeast-2": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-fd9cecc7"},
"eu-central-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-a8221fb5"},
"eu-west-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-a10897d6"},
"sa-east-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-b52890a8"},
"us-east-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-1ecae776"},
"us-west-1": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-d114f295"},
"us-west-2": {"AmazonLinuxAMIHVMEBSBacked64bit": "ami-e7527ed7"}
}
},
"Resources": {
"VPC": {
"Type": "AWS::EC2::VPC",
"Properties": {
"CidrBlock": "172.31.0.0/16",
"EnableDnsHostnames": "true"
}
},
"InternetGateway": {
"Type": "AWS::EC2::InternetGateway",
"Properties": {
}
},
"VPCGatewayAttachment": {
"Type": "AWS::EC2::VPCGatewayAttachment",
"Properties": {
"VpcId": {"Ref": "VPC"},
"InternetGatewayId": {"Ref": "InternetGateway"}
}
},
"SubnetA": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"AvailabilityZone": {"Fn::Select": ["0", {"Fn::GetAZs": ""}]},
"CidrBlock": "172.31.38.0/24",
"VpcId": {"Ref": "VPC"}
}
},
"SubnetB": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"AvailabilityZone": {"Fn::Select": ["1", {"Fn::GetAZs": ""}]},
"CidrBlock": "172.31.37.0/24",
"VpcId": {"Ref": "VPC"}
}
},
"WebServerRouteTable": {
"Type": "AWS::EC2::RouteTable",
"Properties": {
"VpcId": {"Ref": "VPC"}
}
},
"RouteTableAssociationA": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"SubnetId": {"Ref": "SubnetA"},
"RouteTableId": {"Ref": "WebServerRouteTable"}
}
},
"RouteTableAssociationB": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"SubnetId": {"Ref": "SubnetB"},
"RouteTableId": {"Ref": "WebServerRouteTable"}
}
},
"RoutePublicNATToInternet": {
"Type": "AWS::EC2::Route",
"Properties": {
"RouteTableId": {"Ref": "WebServerRouteTable"},
"DestinationCidrBlock": "0.0.0.0/0",
"GatewayId": {"Ref": "InternetGateway"}
},
"DependsOn": "VPCGatewayAttachment"
},
"NetworkAcl": {
"Type": "AWS::EC2::NetworkAcl",
"Properties": {
"VpcId": {"Ref": "VPC"}
}
},
"SubnetNetworkAclAssociationA": {
"Type": "AWS::EC2::SubnetNetworkAclAssociation",
"Properties": {
"SubnetId": {"Ref": "SubnetA"},
"NetworkAclId": {"Ref": "NetworkAcl"}
}
},
"SubnetNetworkAclAssociationB": {
"Type": "AWS::EC2::SubnetNetworkAclAssociation",
"Properties": {
"SubnetId": {"Ref": "SubnetB"},
"NetworkAclId": {"Ref": "NetworkAcl"}
}
},
"NetworkAclEntryIngress": {
"Type": "AWS::EC2::NetworkAclEntry",
"Properties": {
"NetworkAclId": {"Ref": "NetworkAcl"},
"RuleNumber": "100",
"Protocol": "-1",
"RuleAction": "allow",
"Egress": "false",
"CidrBlock": "0.0.0.0/0"
}
},
"NetworkAclEntryEgress": {
"Type": "AWS::EC2::NetworkAclEntry",
"Properties": {
"NetworkAclId": {"Ref": "NetworkAcl"},
"RuleNumber": "100",
"Protocol": "-1",
"RuleAction": "allow",
"Egress": "true",
"CidrBlock": "0.0.0.0/0"
}
},
"LoadBalancer": {
"Type": "AWS::ElasticLoadBalancing::LoadBalancer",
"Properties": {
"Subnets": [{"Ref": "SubnetA"}, {"Ref": "SubnetB"}],
"LoadBalancerName": {"Ref": "SiteID"},
"Listeners": [{
"InstancePort": "80",
"InstanceProtocol": "HTTP",
"LoadBalancerPort": "80",
"Protocol": "HTTP"
}],
"HealthCheck": {
"HealthyThreshold": "2",
"Interval": "5",
"Target": "TCP:80",
"Timeout": "3",
"UnhealthyThreshold": "2"
},
"SecurityGroups": [{"Ref": "LoadBalancerSecurityGroup"}],
"Scheme": "internet-facing",
"CrossZone": "true"
},
"DependsOn": "VPCGatewayAttachment"
},
"LoadBalancerSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "mysite-elb-sg",
"VpcId": {"Ref": "VPC"},
"SecurityGroupIngress": [{
"CidrIp": "0.0.0.0/0",
"FromPort": 80,
"IpProtocol": "tcp",
"ToPort": 80
}]
}
},
"WebServerSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "mysite-web-server-sg",
"VpcId": {"Ref": "VPC"},
"SecurityGroupIngress": [{
"CidrIp": "0.0.0.0/0",
"FromPort": 22,
"IpProtocol": "tcp",
"ToPort": 22
}, {
"FromPort": 80,
"IpProtocol": "tcp",
"SourceSecurityGroupId": {"Ref": "LoadBalancerSecurityGroup"},
"ToPort": 80
}]
}
},
"DatabaseSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "mysite-db-sg",
"VpcId": {"Ref": "VPC"},
"SecurityGroupIngress": [{
"IpProtocol": "tcp",
"FromPort": "3306",
"ToPort": "3306",
"SourceSecurityGroupId": {"Ref": "WebServerSecurityGroup"}
}]
}
},
"Database": {
"Type": "AWS::RDS::DBInstance",
"Properties": {
"AllocatedStorage": "5",
"BackupRetentionPeriod": "0",
"DBInstanceClass": "db.t2.micro",
"DBInstanceIdentifier": {"Ref": "SiteID"},
"DBName": "quantsketch",
"Engine": "MySQL",
"MasterUsername": "mysite",
"MasterUserPassword": "mysite",
"VPCSecurityGroups": [{"Fn::GetAtt": ["DatabaseSecurityGroup", "GroupId"]}],
"DBSubnetGroupName": {"Ref": "DBSubnetGroup"}
},
"DependsOn": "VPCGatewayAttachment"
},
"DBSubnetGroup" : {
"Type" : "AWS::RDS::DBSubnetGroup",
"Properties" : {
"DBSubnetGroupDescription" : "DB subnet group",
"SubnetIds": [{"Ref": "SubnetA"}, {"Ref": "SubnetB"}]
}
},
"S3Bucket": {
"Type": "AWS::S3::Bucket",
"Properties": {
"BucketName": {"Ref": "SiteID"},
"WebsiteConfiguration": {
"IndexDocument": "index.html"
}
}
},
"WebServerLaunchConfiguration": {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Metadata": {
"AWS::CloudFormation::Init": {
"config": {
"packages": {
"yum": {
"php": [],
"php-mysql": [],
"mysql": [],
"httpd": []
}
},
"sources": {
"/var/www/html": "https://wordpress.org/wordpress-4.2.4.tar.gz"
},
"files": {
"/tmp/config": {
"content": {"Fn::Join": ["", [
"#!/bin/bash -ex\n",
]]},
"mode": "000500",
"owner": "root",
"group": "root"
}
},
"commands": {
"01_config": {
"command": "/tmp/config",
"cwd": "/var/www/html/wordpress"
}
},
"services": {
"sysvinit": {
"httpd": {
"enabled": "true",
"ensureRunning": "true"
}
}
}
}
}
},
"Properties": {
"ImageId": {"Fn::FindInMap": ["EC2RegionMap", {"Ref": "AWS::Region"}, "AmazonLinuxAMIHVMEBSBacked64bit"]},
"InstanceType": "t2.micro",
"SecurityGroups": [{"Ref": "WebServerSecurityGroup"}],
"KeyName": {"Ref": "KeyName"},
"AssociatePublicIpAddress": true,
"UserData": {"Fn::Base64": {"Fn::Join": ["", [
"#!/bin/bash -ex\n",
"yum update -y aws-cfn-bootstrap\n",
"/opt/aws/bin/cfn-init -v --stack ", {"Ref": "AWS::StackName"}, " --resource WebServerLaunchConfiguration --region ", {"Ref": "AWS::Region"}, "\n",
"/opt/aws/bin/cfn-signal -e $? --stack ", {"Ref": "AWS::StackName"}, " --resource WebServerAutoScalingGroup --region ", {"Ref": "AWS::Region"}, "\n"
]]}}
}
},
"WebServerAutoScalingGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"LoadBalancerNames": [{"Ref": "LoadBalancer"}],
"LaunchConfigurationName": {"Ref": "WebServerLaunchConfiguration"},
"MinSize": "2",
"MaxSize": "4",
"DesiredCapacity": "2",
"Cooldown": "60",
"HealthCheckGracePeriod": "120",
"HealthCheckType": "ELB",
"VPCZoneIdentifier": [{"Ref": "SubnetA"}, {"Ref": "SubnetB"}],
"Tags": [{
"PropagateAtLaunch": true,
"Value": "quantsketch",
"Key": "Name"
}]
},
"CreationPolicy": {
"ResourceSignal": {
"Timeout": "PT10M"
}
},
"DependsOn": "VPCGatewayAttachment"
},
"WebServerScalingUpPolicy": {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"AutoScalingGroupName": {"Ref": "WebServerAutoScalingGroup"},
"Cooldown": "60",
"ScalingAdjustment": "1"
}
},
"WebServerCPUHighAlarm": {
"Type": "AWS::CloudWatch::Alarm",
"Properties": {
"EvaluationPeriods": "1",
"Statistic": "Average",
"Threshold": "80",
"AlarmDescription": "Alarm if CPU load is high.",
"Period": "60",
"AlarmActions": [{"Ref": "WebServerScalingUpPolicy"}],
"Namespace": "AWS/EC2",
"Dimensions": [{
"Name": "AutoScalingGroupName",
"Value": {"Ref": "WebServerAutoScalingGroup"}
}],
"ComparisonOperator": "GreaterThanThreshold",
"MetricName": "CPUUtilization"
}
},
"WebServerScalingDownPolicy": {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Properties": {
"AdjustmentType": "ChangeInCapacity",
"AutoScalingGroupName": {"Ref": "WebServerAutoScalingGroup"},
"Cooldown": "60",
"ScalingAdjustment": "-1"
}
},
"WebServerCPULowAlarm": {
"Type": "AWS::CloudWatch::Alarm",
"Properties": {
"EvaluationPeriods": "1",
"Statistic": "Average",
"Threshold": "25",
"AlarmDescription": "Alarm if CPU load is low.",
"Period": "60",
"AlarmActions": [{"Ref": "WebServerScalingDownPolicy"}],
"Namespace": "AWS/EC2",
"Dimensions": [{
"Name": "AutoScalingGroupName",
"Value": {"Ref": "WebServerAutoScalingGroup"}
}],
"ComparisonOperator": "LessThanThreshold",
"MetricName": "CPUUtilization"
}
}
},
"Outputs": {
"URL": {
"Value": {"Fn::Join": ["", ["http://", {"Fn::GetAtt": ["LoadBalancer", "DNSName"]}, "/quantsketch"]]},
"Description": "QuantSketch URL"
}
}
}
遗憾的是，我无法找到解决此堆栈架构的任何示例模板，尽管我认为这是一个常见问题。
如何连接这两个集群，以便 Web 服务器可以将工作卸载到工作服务器上？任何包含示例、教程或其他内容的建议都会大有帮助。
您通常可以通过在您的负载平衡和工作机器上的应用程序 运行 中调用 AWS APIs/SDKs 来发送和接收来自 SQS 的消息来实现这一点。
选项 A：直接 Send/Receive 通过 SQS
- Web 服务器准备您的消息并调用 SQS SendMessage 将其发送到队列。
- Worker 调用 SQS ReceiveMessage 从队列中接收一批最多 10 条消息，并处理它们。
- 如果您想减少对 SQS 的调用，工作人员调用 SQS DeleteMessage after processing, so that the message isn't retried. Could also call SQS DeleteMessageBatch。
选项 B：Pub/Sub 通过 SNS 发布者到 SQS 订阅者
另一种选择是先通过 SNS 发布您的消息，然后通过 SQS 订阅它。这为您提供了额外的好处，即能够订阅不同的应用程序（甚至人类，通过 SMS/email）到消息，如果您有这样的用例。
- Web 服务器准备您的消息并调用 SNS Publish 将您的消息发布给给定 SNS 主题的所有订阅者。
- SNS 向给定 SNS 主题的所有订阅者发布消息；您的队列就是这样的订阅者，因此消息最终出现在您的队列中。
- Worker 调用 SQS ReceiveMessage 从队列中接收一批最多 10 条消息，并处理它们。
- 如果您想减少对 SQS 的调用，工作人员调用 SQS DeleteMessage after processing, so that the message isn't retried. Could also call SQS DeleteMessageBatch。
讨论
您的 Web 服务器和工作堆栈都需要能够在您的 VPC 之外进行通信，以便访问 SNS and/or SQS 服务。您可以通过使用 NAT Gateways 允许您的实例与全球互联网通信来完成此操作。
您需要提前配置 SNS/SQS 资源。您可以通过 CloudFormation 模板或通过 SNS/SQS API 再次执行此操作。
您没有提到您的应用程序使用的是哪种语言；您将需要阅读您将使用的与我在此答案中链接到的 API 相匹配的特定 SDK 的文档。
要授予您的应用程序访问您的 AWS 账户中这些资源的权限，我建议创建一个 IAM instance profile，它至少有权读取和写入此 SQS 队列，如果需要，还可以选择该 SNS 主题你选择 pub/sub 路线。如果您没有明确向您的应用程序提供凭据，则凭据链将进入 IAM 实例配置文件并继承它。您可以创建这些 IAM 资源作为 CloudFormation 模板的一部分。
还有一些最后的哲学笔记：
不要把这个问题想成连接你的集群；在这两种情况下，他们只会知道 SNS/SQS。这里的目标是您的网络服务器不了解您的工作人员，反之亦然。
我认为您对这个问题的思考过于宽泛，这就是为什么您一直难以找到相关资源的原因。将您的查询分解为更小的工作单元，例如 "How can I send messages to SQS?" 或 "How can I receive messages from SQS?"，我认为您的运气会更好。
进一步阅读
- AWS Documentation - Sending Amazon SNS Messages to Amazon SQS Queues
- AWS Documentation - NAT Instances - 如果 NAT 网关对于您的解决方案来说过于昂贵，您可以阅读此处了解如何设置执行相同功能的 EC2 实例。
- AWS Documentation - Tutorial: Sending and Receiving Messages in Amazon SQS - 使用 Javascript SDK 进行发送和接收
- AWS Documentation - SQS Long Polling - 通过延长消息接收等待时间来减少来自 SQS SDK 的空响应的有用策略。