[Python] Scrapy Playwright scraping all table data after clicking "Show More" button

I have build a scraper with python scrapy to get table data from this website:

https://datacvr.virk.dk/enhed/virksomhed/28271026?fritekst=28271026&sideIndex=0&size=10

As you can see, this website has a table with employee data under "Antal Ansatte". I managed to scrape some of the data, but not all. You have to click on "Vis alle" (show more) to see all the data. In the script below I attempted to do just that by adding PageMethod('click', "button.show-more") to the playwright_page_methods. When I run the script, it does identify the button (locator resolved to 2 elements. Proceeding with the first one: <button type="button" class="show-more" data-v-509209b4="" id="antal-ansatte-pr-maaned-vis-mere-knap">Vis alle</button>) says "element is not visible". It tries several times, but element remains not visible.

Any help would be greatly appreciated, I think (and hope) we are almost there, but I just can't get the last bit to work.

import scrapy
from scrapy_playwright.page import PageMethod
from pathlib import Path
from urllib.parse import urlencode


class denmarkCVRSpider(scrapy.Spider):
# scrapy crawl denmarkCVR -O output.json
name = "denmarkCVR"

HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:98.0) Gecko/20100101 Firefox/98.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Cache-Control": "max-age=0",
}

def start_requests(self):
# https://datacvr.virk.dk/enhed/virksomhed/28271026?fritekst=28271026&sideIndex=0&size=10
CVR = '28271026'
urls = [f"https://datacvr.virk.dk/enhed/virksomhed/{CVR}?fritekst={CVR}&sideIndex=0&size=10"]
for url in urls:
yield scrapy.Request(url=url,
callback=self.parse,
headers=self.HEADERS,
meta={ 'playwright': True,
'playwright_include_page': True,
'playwright_page_methods': [
PageMethod("wait_for_load_state", "networkidle"),
PageMethod('click', "button.show-more")],
'errback': self.errback },
cb_kwargs=dict(cvr=CVR))

async def parse(self, response, cvr):
"""
extract div with table info. Then go through all tr (table row) elements
for each tr, get all variable-name / value pairs
"""
trs = response.css("div.antalAnsatte table tbody tr")
data = []
for tr in trs:
trContent = tr.css("td")
tdData = {}
for td in trContent:
variable = td.attrib["data-title"]
value = td.css("span::text").get()
tdData[variable] = value
data.append(tdData)

yield { 'CVR': cvr,
'data': data }

async def errback(self, failure):
page = failure.request.meta["playwright_page"]
await page.close()

Continue reading...