From 863b0255ead90d47dd014aaa0ea6373f71a7a14c Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 1 Sep 2026 10:51:55 +0200 Subject: [PATCH 1/2] fix: Add navigation_timeout to _PlaywrightCrawlerAdditionalOptions --- src/crawlee/crawlers/_playwright/_playwright_crawler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index 2e2e403411..7d4c4c7be2 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -630,6 +630,9 @@ class _PlaywrightCrawlerAdditionalOptions(TypedDict): headless: NotRequired[bool] """Whether to run the browser in headless mode. This option should not be used if `browser_pool` is provided.""" + navigation_timeout: NotRequired[timedelta] + """Timeout for navigation (the process between opening a Playwright page and calling the request handler).""" + class PlaywrightCrawlerOptions( _PlaywrightCrawlerAdditionalOptions, From c7df284d5252edc33bbd26718e5ff8a02340abdf Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 1 Sep 2026 10:51:56 +0200 Subject: [PATCH 2/2] test: Deflake adaptive crawler timeout test against slow CI browser navigation --- .../_adaptive_playwright/test_adaptive_playwright_crawler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 4b27784e34..d8c0f36ba0 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -601,6 +601,11 @@ async def test_adaptive_playwright_crawler_timeout_in_sub_crawler(test_urls: lis max_request_retries=0, rendering_type_predictor=static_only_predictor_no_detection, request_handler_timeout=request_handler_timeout, + # The fallback browser request navigates under its own `navigation_timeout` budget, separate from + # the request handler timeout relaxed in the handler below. Give it the same generous ceiling: + # with `max_request_retries=0`, a single navigation slower than the default budget on a loaded CI + # runner would fail the only attempt before the handler ever runs. + playwright_crawler_specific_kwargs={'navigation_timeout': timedelta(seconds=120)}, ) mocked_static_handler = Mock(name='static_handler') mocked_browser_handler = Mock(name='browser_handler')