Heroku 使用 Puppeteer 时内存使用量突然飙升

Question

如果昨天有这个提交，进程将直接进入 Heroku 的内存限制，给我一个 R15 错误 - 它在我的测试中和在 heroku 上都非常有效，直到它得到一个随机数量的检查项目，在这个时候它抛出错误。有趣的是，在错误发生后，我得到了另一个 R15 的，它说我只使用了 22.2% 的可用内存。

这是给我错误的代码：

let lenPlug = await client.db("PluginBoutique").collection("Kaye").countDocuments({});
let lenGuild = await client.db("Guilds").collection("Kaye").countDocuments({});
let lenSpit = await client.db("Spitfire").collection("Kaye").countDocuments({});
console.log("Plugins in database: " + lenPlug + " on Plugin Boutique and " + lenSpit + " on Spitfire" + "\nGuilds in database: " + lenGuild + '\n');

async function sendNote(Embed, lenGuild) {
  for (let j = 0; j < lenGuild; j++) {
    let guildId = await client.db("Guilds").collection("Kaye").distinct("id", {
      _id: j
    });
    let channelId = await client.db("Guilds").collection("Kaye").distinct("channel", {
      _id: j
    });
    try {
      await bot.guilds.cache.get(guildId[0]).channels.cache.get(channelId[0]).send(Embed);
    } catch {
      console.log("Could not send message to " + channelId + " ");
    }
  }
  return;
}

function percentage(price, discount) {
  let price_int = parseInt(price.substring(1), 10);
  let discount_int = parseInt(discount.substring(1), 10);
  return 100 - parseInt(discount_int * 100 / price_int, 10);
}

function percentageColor(percentage) {
  if (percentage < 10)
    return '#36B3E2';
  if (percentage < 25)
    return '#36E26B';
  if (percentage < 50)
    return '#FAB633';
  return '#E23E36';
}

const browser = await puppeteer.launch(args);

const page = await browser.newPage();
await page.setRequestInterception(true);
await page.on('request', (req) => config(req));
await page.setDefaultNavigationTimeout(0);

let existsDiscountSpit = false;
for (let i = 25; i < lenSpit; i++) {
  let url = (await client.db("Spitfire").collection("Kaye").distinct("url", {
    _id: i
  })).toString();
  let hasDiscount = await client.db("Spitfire").collection("Kaye").distinct("hasDiscount", {
    _id: i
  });
  await page.goto(url, {
    waitUntil: 'networkidle2',
  }).then(async() => {
    try {
      const el4 = await page.waitForSelector('body > div.page > section > div.catalogue-view-product > div.grid.sub-section.main-info > div.grid__item.two-thirds.palm--one-whole.copy > div.copy-items-wrapper > div.headline > div > div.grid__item.one-third.lap--one-whole.palm--one-whole > p > span.view-product-current-price.view-product-discounted-price', {
        timeout: 5000
      });
      const txt4 = await el4.getProperty('textContent');
      const price = await txt4.jsonValue();
      const el5 = await page.waitForSelector('body > div.page > section > div.catalogue-view-product > div.grid.sub-section.main-info > div.grid__item.two-thirds.palm--one-whole.copy > div.copy-items-wrapper > div.headline > div > div.grid__item.one-third.lap--one-whole.palm--one-whole > p > span.view-product-was-price', {
        timeout: 5000
      });
      const txt5 = await el5.getProperty('textContent');
      const oldprice = await txt5.jsonValue();
      const el1 = await page.waitForSelector('body > div.page > section > div.catalogue-view-product > div.grid.sub-section.main-info > div.grid__item.two-thirds.palm--one-whole.copy > div.copy-items-wrapper > div.headline > div > div.grid__item.two-thirds.lap--one-whole.palm--one-whole > div > h1', {
        timeout: 5000
      });
      const txt1 = await el1.getProperty('textContent');
      const name = await txt1.jsonValue();
      const image = await page.$$eval('.image:nth-child(1) > picture img[src]', image => image.map(image => image.getAttribute('src')));
      if (hasDiscount == 0) {
        existsDiscountSpit = true;
        await client.db("Spitfire").collection("Kaye").replaceOne({
          _id: i
        }, {
          url: url,
          hasDiscount: 1
        });
        const percentage = percentage(oldprice, price)
        const color = percentageColor(percentage)
        const Embed = new discord.MessageEmbed()
          .setColor(color)
          .setTitle("<:kaye:828581356392808458> " + name)
          .setURL(url)
          .addField('\u200B', name + " was **" + oldprice + "** and now is **" + price + "** with a **" + percentage + "%** discount on **[Spitfire](https://www.spitfireaudio.com/)**! " + messages[Math.floor(Math.random() * messages.length)])
          .setImage(image[0])
          .setFooter('kaye-app')
        sendNote(Embed, lenGuild);
        console.log("Discount found for url " + url + "!");
      } else {
        console.log("Already notified about plugin url " + url + "!")
      }
    } catch {
      console.log("No discount found for url " + url + "!");
      if (hasDiscount == 1)
        await client.db("Spitfire").collection("Kaye").replaceOne({
          _id: i
        }, {
          url: url,
          hasDiscount: 0
        });
    }
  });
}
if (!existsDiscountSpit) {
  const Embed = new discord.MessageEmbed()
    .setColor('#d6d3ce')
    .setTitle("<:kaye:828581356392808458> Didn't find discounts on Spitfire today!")
    .setURL("https://www.spitfireaudio.com/")
    .setFooter('kaye-app')
  sendNote(Embed, lenGuild);
}

这是 Heroku 日志的片段：

2021-04-19T07:54:58.701763+00:00 app[worker.1]: No discount found for url https://www.spitfireaudio.com/shop/a-z/spitfire-studio-brass-professional/!
2021-04-19T07:55:03.337663+00:00 app[worker.1]: The script uses approximately 21.2 MB
2021-04-19T07:55:03.337669+00:00 app[worker.1]:
2021-04-19T07:55:07.262079+00:00 app[worker.1]: No discount found for url https://www.spitfireaudio.com/shop/a-z/spitfire-studio-orchestra/!
2021-04-19T07:55:08.337457+00:00 app[worker.1]: The script uses approximately 21.35 MB
2021-04-19T07:55:08.337464+00:00 app[worker.1]:
2021-04-19T07:55:13.339770+00:00 app[worker.1]: The script uses approximately 21.4 MB
2021-04-19T07:55:13.339772+00:00 app[worker.1]:
2021-04-19T07:55:14.889726+00:00 app[worker.1]: No discount found for url https://www.spitfireaudio.com/shop/a-z/spitfire-studio-orchestra-professional/!
2021-04-19T07:55:16.146914+00:00 heroku[worker.1]: Process running mem=512M(100.0%)
2021-04-19T07:55:16.193131+00:00 heroku[worker.1]: Error R15 (Memory quota vastly exceeded)
2021-04-19T07:55:16.197917+00:00 heroku[worker.1]: Stopping process with SIGKILL
2021-04-19T07:55:16.212670+00:00 heroku[worker.1]: Process running mem=115M(22.2%)
2021-04-19T07:55:16.266903+00:00 heroku[worker.1]: Error R15 (Memory quota vastly exceeded)
2021-04-19T07:55:16.297708+00:00 heroku[worker.1]: Stopping process with SIGKILL
2021-04-19T07:55:16.308694+00:00 heroku[worker.1]: Process exited with status 137

注意：我还有一个记录内存使用情况的函数

谢谢！

Answer 1

显然，拒绝 javascript 请求是解决此问题的方法 - 尽管它并不完美并且某些站点可能需要 javascript 到运行，但这个是例外。只需更新阻止图像和 css 的配置函数（我忘记包括在内）以阻止 js。

这是该函数现在的样子

function config(req) {
        if (req.resourceType() === 'stylesheet' || req.resourceType() === 'font' || req.url().endsWith('.js')) {
            req.abort();
        }
        else {
            req.continue();
        }
    }

Heroku 使用 Puppeteer 时内存使用量突然飙升

Heroku sudden spike in memory usage with Puppeteer

javascript

heroku

mongodb

node.js

puppeteer