无法使用 Selenium 和 scrapy 抓取下一页 URL

Question

使用 Selenium 单击“下一步”按钮后，我很难 parse/scrape 每一页。我可以转到第二页，但是之后就失败了。不确定如何解决这个问题，有什么建议吗？这是代码：

class PropertyFoxSpider(scrapy.Spider):
    name = 'property_fox'
    start_urls = [
        'https://propertyfox.co.za/listing-search?currentpage=1&term_id=62515&keywords=Western+Cape&orderby=createddate:desc&status%5B%5D=Active'
    ]


    def __init__(self):
        #path to driver
        self.driver = webdriver.Chrome('path')


    def parse(self,response):
        self.driver.get(response.url)
        while True: 
            try: 
                elem = WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable((By.ID, "pagerNext")))
                elem.click()
                url = self.driver.current_url
                yield scrapy.Request(url=url, callback=self.parse_page, dont_filter=False)
            except TimeoutException:
                break



    def parse_page(self, response):
        #self.driver.get(response.url)
        for prop in response.css('div.property-item'):
            link = prop.css('a::attr(href)').get()
            banner = prop.css('div.property-figure-icon div::text').get()
            sold_tag = None
            if banner:
                banner = banner.strip()
                sold_tag = 'sold' if 'sold' in banner.lower() else None

            yield scrapy.Request(
                link,
                callback=self.parse_property,
                meta={'item': {
                    'agency': self.name,
                    'url': link,
                    'offering': 'buy',
                    'banners': banner,
                    'sold_tag':  sold_tag,
                }},
            )
    def parse_property(self, response):
        item = response.meta.get('item')
     ...

Answer 1

你可以等到 URL 改变后再抓取它

from selenium.webdriver.support.ui import WebDriverWait

url = self.driver.current_url
elem = WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable((By.ID, "pagerNext")))
elem.click()
WebDriverWait(self.driver, 10).until(lambda driver: self.driver.current_url != url)
url = self.driver.current_url

无法使用 Selenium 和 scrapy 抓取下一页 URL

Unable to scrape the Next page URLs using Selenium and scrapy

python

selenium

scrapy

web-scraping