AWS Lambda NodeJS 启动 EMR Spark 问题

Question

我正在尝试在 Lambda (NodeJS-12) 中使用以下脚本启动 EMR 集群。

var AWS = require('aws-sdk');
var emr = new AWS.EMR({apiVersion: '2009-03-31', region: 'ap-northeast-1'});
exports.handler = (event, context, callback) => {
    var params = {
        "Name": "LaunchEMR",
        "Instances": {
            "KeepJobFlowAliveWhenNoSteps": true,
            "TerminationProtected": false,
            "Ec2SubnetId": "subnet-XXXXXXXX",
            "EmrManagedMasterSecurityGroup": "ElasticMapReduce-master",
            "EmrManagedSlaveSecurityGroup": "ElasticMapReduce-slave",
            "HadoopVersion": "2.8.5",
            "InstanceGroups": [{
                "Name": "Master",
                "InstanceRole": "MASTER",
                "InstanceCount": 1,
                "InstanceType": "m3.xlarge",
                "Market": "ON_DEMAND"
            }, {
                "Name": "Core",
                "InstanceRole": "CORE",
                "InstanceCount": 1,
                "InstanceType": "m3.xlarge",
                "Market": "ON_DEMAND"
            }]
        },
        "Applications": [{
            "Name": "Hadoop"
        }, {
            "Name": "Spark"
        }],
        "ServiceRole": "EMR_DefaultRole",
        "JobFlowRole": "EMR_EC2_DefaultRole",
        "ReleaseLabel": "emr-5.28.0"
    };
    emr.runJobFlow(params)
        .on('success', function(response){ console.log("success => " + response); console.log(response); })
        .on('error', function(response){ console.log("error => " + response); console.log(response); })
        .on('complete', function(response){ console.log("complete => "  + response); console.log(response); })
        .send( function(err, data){
            if (err) console.log(err, err.stack); // an error occurred
            else     console.log(data);           // successful response
            callback(null, {
                statusCode: 200,
                body: JSON.stringify(err),
            });
        });
};

我已经调整了 IAM 策略，我似乎能够将请求成功提交到 EMR，因为 Lambda 云监视日志中没有错误，我们可以在 EMR 事件中看到该请求。然而，我们将在请求事件之后立即获得集群终止事件。

错误日志

Amazon EMR Cluster j-31OE0F3OXGO4K (LaunchEMR) has terminated with errors at 2019-12-02 06:32 UTC with a reason of VALIDATION_ERROR.

SubnetID和Security-Group与我手动启动的集群相同。我在 "EMR_DefaultRole" 中添加了 "iam:PassRole"。我还尝试了其他一些实例类型。

有什么我遗漏的吗？

Answer 1

我可以通过取消这 3 行来成功运行它。

            "Ec2SubnetId": "subnet-XXXXXXXX",
            "EmrManagedMasterSecurityGroup": "ElasticMapReduce-master",
            "EmrManagedSlaveSecurityGroup": "ElasticMapReduce-slave",

但我仍然不知道它为什么有效。（以及为什么没有）

AWS Lambda NodeJS 启动 EMR Spark 问题

AWS Lambda NodeJS launch EMR Spark Problem

amazon-web-services

amazon-emr

aws-sdk

aws-lambda