在 puppeteer 站点内获取数据

Question

我在从外部服务获取数据时遇到奇怪的问题。我有简单的脚本来打印来自 API:

的一些数据

fetch("http://sip.ztm.kielce.pl/Home/CNR_GetVehicles?r=0W&d=&nb=", {"credentials":"include","headers":{"accept":"application/xml, text/xml, */*; q=0.01","accept-language":"pl;q=1.3592","x-requested-with":"XMLHttpRequest"},"referrer":"http://sip.ztm.kielce.pl/","referrerPolicy":"no-referrer-when-downgrade","body":null,"method":"GET","mode":"cors"})
.then(response => {
        return response.text();
    }).then(function(data) {
    console.log(data); // this will be a string
});

此脚本有效，但仅当我处于 http://sip.ztm.kielce.pl 时有效。所以我计划使用 puppeteer 从这个网站上下文评估这个脚本。

所以我写了Node.JS代码：

const puppeteer = require('puppeteer');

(async () => {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    page.on('console', (log) => console[log._type](log._text));

    await page.goto('http://sip.ztm.kielce.pl');
    await page.waitFor(5000);

    await page.evaluate(() => {
        console.log('test');
        fetch("http://sip.ztm.kielce.pl/Home/CNR_GetVehicles?r=0W&d=&nb=", {"credentials":"include","headers":{"accept":"application/xml, text/xml, */*; q=0.01","accept-language":"pl;q=1.3592","x-requested-with":"XMLHttpRequest"},"referrer":"http://sip.ztm.kielce.pl/","referrerPolicy":"no-referrer-when-downgrade","body":null,"method":"GET","mode":"cors"})
        .then(response => {
                return response.text();
            }).then(function(data) {
            console.log(data); // this will be a string
        });
    });
    await browser.close();
})();

但我得到的回应不同于 "normal" 方式。你知道它有什么问题吗？

由于 API 安全性，此代码可能无效，在第一个版本中也是如此。我正在 "working" 从 Chrome 开发人员工具中的网络选项卡获取代码。

Answer 1

像这样使用相对 url：

/Home/CNR_GetVehicles?r=0W&d=&nb=

而不是完整路径

const puppeteer = require('puppeteer');

(async () => {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    page.on('console', (log) => console[log._type](log._text));

    await page.goto('http://sip.ztm.kielce.pl', {
        waitUntil: "networkidle2",
    })

    await page.evaluate(() => {
        console.log('test');
        fetch("/Home/CNR_GetVehicles?r=0W&d=&nb=", {"credentials":"include","headers":{"accept":"application/xml, text/xml, */*; q=0.01","accept-language":"pl;q=1.3592","x-requested-with":"XMLHttpRequest"},"referrer":"http://sip.ztm.kielce.pl/","referrerPolicy":"no-referrer-when-downgrade","body":null,"method":"GET","mode":"cors"})
        .then(response => {
                return response.text();
            }).then(function(data) {
            console.log(data); // this will be a string
        });
    });
    await browser.close();
})();

或者您可以直接转到那个 url

const puppeteer = require('puppeteer');

(async () => {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    page.on('console', (log) => console[log._type](log._text));

    await page.goto('http://sip.ztm.kielce.pl/Home/CNR_GetVehicles?r=0W&d=&nb=', {
        waitUntil: "networkidle2",
    });

    await page.evaluate(() => {
        const data = document.body.innerText
        console.log(data) // it will be string

    });
    await browser.close();
})();

在 puppeteer 站点内获取数据

Fetching data inside puppeteer site

web-crawler

node.js

fetch-api

puppeteer