Lambda function "Error: connect ETIMEDOUT" when receive too many requests for MySQL query

Question

我正在尝试对我的 AWS lambda 函数进行测试，该函数对 RDS MySQL (t2.medium) 进行查询。但是，如果我多次请求 API，虽然我可以使用正确的数据成功查询，但有时会得到 "Error: connect ETIMEDOUT"。

我的代码或设置有问题吗？

我阅读了一些设置参数的建议：

MySQL:

wait_timeout 1

max_connections 16000

interactive_timeout 6000

max_allowed_packet 1073741824

拉姆达巴：

Timeout 60 sec place the Lambda function in the same VPC as your RDS

Added VPC execution policy AWSLambdaVPCAccessExecutionRole

assign a security group to the lambda function

In the security attached to the RDS instance, added an inbound rule for mysql

Confirm that Lambda function has access to the same VPC RDS database

Lambda 错误日志

2019-03-28T18:51:47.353Z ab4fbbaf-1ea2-458b-a5b5-781cdfdd80df { Error: connect ETIMEDOUT

at Connection._handleConnectTimeout

(/var/task/node_modules/mysql/lib/Connection.js:411:13)

at Object.onceWrapper (events.js:313:30)

at emitNone (events.js:106:13)

at Socket.emit (events.js:208:7)

at Socket._onTimeout (net.js:420:8)

at ontimeout (timers.js:482:11)

at tryOnTimeout (timers.js:317:5)

at Timer.listOnTimeout (timers.js:277:5)

at Protocol._enqueue (/var/task/node_modules/mysql/lib/protocol/Protocol.js:144:48)

at Protocol.handshake (/var/task/node_modules/mysql/lib/protocol/Protocol.js:51:23)

at Connection.connect (/var/task/node_modules/mysql/lib/Connection.js:118:18)

at Connection._implyConnect (/var/task/node_modules/mysql/lib/Connection.js:453:10)

at Connection.query (/var/task/node_modules/mysql/lib/Connection.js:198:8)

at Promise (/var/task/db.js:62:9)

at new Promise ()

at Object.retrieve (/var/task/db.js:55:10)

at exports.getAlerts (/var/task/index.js:59:24)

errorno: 'ETIMEDOUT',

code: 'ETIMEDOUT',

syscall: 'connect',

fatal: true }

RDS 错误日志

2019-03-28T18:18:49.378320Z 9500 [Note] Aborted connection 9500 to db: 'db' user: 'user' host: '123.123.123.123' (Got timeout reading communication packets)

2019-03-28T18:18:49.392514Z 9498 [Note] Aborted connection 9498 to db: 'db' user: 'user' host: '123.123.123.123' (Got timeout reading communication packets)

2019-03-28T18:18:49.470617Z 9499 [Note] Aborted connection 9499 to db: 'db' user: 'user' host: '123.123.123.123' (Got timeout reading communication packets)

2019-03-28T18:18:49.636775Z 9501 [Note] Aborted connection 9501 to db: 'db' user: 'user' host: '123.123.123.123' (Got timeout reading communication packets)

2019-03-28T18:18:49.694669Z 9502 [Note] Aborted connection 9502 to db: 'db' user: 'user' host: '123.123.123.123' (Got timeout reading communication packets)

2019-03-28T18:18:49.803457Z 9503 [Note] Aborted connection 9503 to db: 'db' user: 'user' host: '123.123.123.123' (Got timeout reading communication packets)

2019-03-28T18:18:49.824250Z 9504 [Note] Aborted connection 9504 to db: 'db' user: 'user' host: '123.123.123.123' (Got timeout reading communication packets)

我的 db.js lambda 查询

const mysql = require('mysql')
let retrieve = (sql, objectArr, entityName) => {
      return new Promise((resolve, reject) => {
        let con = mysql.createConnection(dbParams)
        con.query(sql, objectArr, (err2, results) => {
          con.end()
          if (err2) {
            console.log(err2)
            return reject(new apiError.DatabaseError('An error occurred in retrieve'))
          }
          console.log('Data retrieve successfully')
          return resolve(results)
        })
      })
    }

我的test.js脚本

const request = require('request')
let errorCount = 0
let success = 0

for (let i = 0; i < 4000; i++) {
  console.log('Send')
  request.get('https://myapi/users', {
    headers: {
      'client_id': 'app',
      'Content-Type': 'application/json'
    }
  }, (error, response, body) => {
    if (error) {
      console.log('some error')
      errorCount++
    } else {
      let jsonBody = JSON.parse(body)
      if (jsonBody.code === 0) {
        success++
      } else {
        errorCount++
      }
    }

    console.log('Success: ', success)
    console.log('Error: ', errorCount)
  })
}

编辑：我还尝试更改测试脚本中的 i < 1，然后它总是给我“错误：连接 ETIMEDOUT。但是如果 i < 900，它有时会成功运行

index.js

const db = require('./db.js')
exports.getUsers = async (event, context) => {
  context.callbackWaitsForEmptyEventLoop = false
  try {
    let sql = 'SELECT * FROM User'
    let abc = await db.retrieve(sql, [], 'user')

    let response = {
      statusCode: 200,
      abc: abc,
      code: 0
    }
    return response
  } catch (err) {
    console.log(err)
    errorHandler(err)
  }
}

db.js 池

const mysql = require('mysql')
const constants = require('./constants.js')

let dbParams = {
  host: constants.SQL_CONNECTION_HOST,
  user: constants.SQL_CONNECTION_USER,
  password: constants.SQL_CONNECTION_PASSWORD,
  database: constants.SQL_CONNECTION_DATABASE,
  multipleStatements: true,
  maxConnections: 4
}

const pool = mysql.createPool(dbParams)
let retrieve = (sql, objectArr, entityName) => {
  return new Promise((resolve, reject) => {
    pool.getConnection((err1, con) => {
      if (err1) {
        console.log(err1)
        return reject(new apiError.DatabaseError('An error occurred in retrieve pool'))
      }
      console.log('Pool connect successfully')
      con.query(sql, objectArr, (err2, results) => {
        con.end()
        if (err2) {
          console.log(err2)
          return reject(new apiError.DatabaseError('An error occurred in retrieve'))
        }
        console.log('Data retrieve successfully')
        return resolve(results)
      })
    })
  })
}

使用池后的错误日志

2019-03-28T23:35:24.144Z 91b0fc78-e4d1-4fd9-bdf7-923715b165c0 { Error: Handshake inactivity timeout

at Handshake. (/var/task/node_modules/mysql/lib/protocol/Protocol.js:163:17)

at emitNone (events.js:106:13)

at Handshake.emit (events.js:208:7)

at Handshake._onTimeout (/var/task/node_modules/mysql/lib/protocol/sequences/Sequence.js:124:8)

at Timer._onTimeout (/var/task/node_modules/mysql/lib/protocol/Timer.js:32:23)

at ontimeout (timers.js:482:11)

at tryOnTimeout (timers.js:317:5)

at Timer.listOnTimeout (timers.js:277:5)

at Protocol._enqueue (/var/task/node_modules/mysql/lib/protocol/Protocol.js:144:48)

at Protocol.handshake (/var/task/node_modules/mysql/lib/protocol/Protocol.js:51:23)

at PoolConnection.connect (/var/task/node_modules/mysql/lib/Connection.js:118:18)

at Pool.getConnection (/var/task/node_modules/mysql/lib/Pool.js:48:16)

at Promise (/var/task/db.js:72:10)

at new Promise ()

at Object.retrieve (/var/task/db.js:67:10)

at exports.getAlerts (/var/task/index.js:59:24)

code: 'PROTOCOL_SEQUENCE_TIMEOUT',

fatal: true,

timeout: 10000 }

现在用 Pool 设置并用请求循环测试它：

i < 100 result => 成功：866 和错误：134 次请求。

i < 10 result => 成功：8 次错误：2 次请求。

握手不活动超时错误

db.js 外 con.createConnection

const mysql = require('mysql')
// initialize dbParams
let con = mysql.createConnection(dbParams)

let retrieve = (sql, objectArr, entityName) => {
  return new Promise((resolve, reject) => {
    con.query(sql, objectArr, (err2, results) => {
      //con.end() commet out connection end here 
      if (err2) {
        console.log(err2)
        return reject(new apiError.DatabaseError('An error occurred in retrieve'))
      }
      console.log('Data retrieve successfully')
      return resolve(results)
    })
  })
}

Answer 1

问题中没有解释 retrieve 函数是如何在你的 lambda 函数中被调用的。

尽管如此，它似乎会在您的测试中的每次迭代中执行，这会创建大量连接并可能解释您看到的行为。

我建议在 lambda 初始化时（又名 'coldstart'）创建连接，方法是将代码行 let con = mysql.createConnection(dbParams) 移到任何函数之外（这样只会发生一个连接）。

另一个不错的选择是使用 connection pooling。

Lambda function "Error: connect ETIMEDOUT" when receive too many requests for MySQL query

Lambda function "Error: connect ETIMEDOUT" when receive too many requests for MySQL query

mysql

load-testing

node.js

aws-lambda

serverless