I do not anything how to scrape ajax pages there is no pagination on website the website will be load by clicking the load more button these is the page link https://aaos22.mapyourshow.com/8_0/explore/exhibitor-gallery.cfm?featured=false import scrapy from scrapy.http import Request from selenium import webdriver from scrapy_selenium import SeleniumRequest import pandas as pd class TestSpider(scrapy.Spider): name = 'test' def start_requests(self): yield SeleniumRequest( url="https://aaos22.mapyourshow.com/8_0/explore/exhibitor-gallery.cfm?featured=false", wait_time=3, screenshot=True, callback=self.parse, dont_filter=True ) def parse(self, response): books = response.xpath("//h3[@class='card-Title\nbreak-word\nf3\nmb1\nmt0']//a//@href").extract() for book in books: url = response.urljoin(book) ...