Make a screenshot ?

[caroheymes]

Hi Elias,
I appreciate very much the integration with playright ! Really great :

Is there any possibility to make screenshots ?

Thanks a million for your feedback,

Cheers,

Caroline

[caroheymes]

Hi @eliasdabbas ,

I was wondering if there was way to add a capture screenshot module in advertools. Something like

%%writefile screenshot_spider3.py 
#enregistre le fichier en local

import json
import subprocess
import scrapy
from scrapy import Spider, Request
from scrapy_playwright.page import PageMethod
import pandas as pd

df = None

class ScreenshotsSpider(scrapy.Spider):
    name = "screenshots"
    def __init__(self, csv_path=None, *args, **kwargs):
        super(ScreenshotsSpider, self).__init__(*args, **kwargs)
        
        if csv_path:
            df = pd.read_csv(csv_path)
        self.df = df

    def start_requests(self):
        
        for index, row in self.df.iterrows():
            url = row['url']
            image = row['nom']

            yield scrapy.Request(
                url,
                meta=dict(
                    image_name=image,
                    playwright=True,
                    playwright_include_page=True,
                    # playwright_page=page,
                    playwright_page_methods=[
                        PageMethod("wait_for_selector", "body"),
                        PageMethod("evaluate", "window.scrollBy(0, document.body.scrollHeight)"),
                    ],
                ),
                callback=self.parse,  #test
                errback=self.errback,
            )

    async def parse(self, response):
        page = response.meta["playwright_page"]
        screenshot_path = './test/' + response.meta['image_name'] + '.png'
        await page.screenshot(path=screenshot_path, full_page=True)
        await page.close()

    async def errback(self, failure):
        page = failure.request.meta["playwright_page"]
        await page.close()
        

import json
import subprocess
import scrapy
from scrapy import Spider, Request
from scrapy_playwright.page import PageMethod
# Meta settings pour Playwright
meta = {'playwright': True}


custom_settings = {
    "ROBOTSTXT_OBEY": False,
    "USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", 
    "DOWNLOAD_HANDLERS": {
        "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
        "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
    },
    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",

}

settings_list = []
if custom_settings is not None:
    for key, val in custom_settings.items():
        if isinstance(val, (dict, list, set, tuple)):
            setting = "=".join([key, json.dumps(val)])
        else:
            setting = "=".join([key, str(val)])
        settings_list.extend(["-s", setting])


def crawl_screenshots(df): 
  filepath ='./urls.csv'
  df.to_csv(filepath, index=False)

  command = [
        "scrapy",
        "runspider",
        '/content/screenshot_spider3.py', 
        "-a",
        "csv_path=" + filepath,
        "-a",
        "meta="+json.dumps(meta),
    ] + settings_list

  subprocess.run(command)```

[eliasdabbas]

Thanks for putting this code together.

The example shared with in the official docs shows how it can be integrated:

https://github.com/scrapy-plugins/scrapy-playwright?tab=readme-ov-file#pagemethod-class

The parse method needs to somehow be updated, but in the current setup it might be a bit complicated to implement. So, I'm thinking a separate function might be enough to do so.