diff --git a/crawl4ai/processors/pdf/__init__.py b/crawl4ai/processors/pdf/__init__.py index 7722fe7fd..b41677c6b 100644 --- a/crawl4ai/processors/pdf/__init__.py +++ b/crawl4ai/processors/pdf/__init__.py @@ -171,11 +171,17 @@ def _get_pdf_path(self, url: str) -> str: # Redirects are followed manually so url_validator (when set) can vet every hop BEFORE it is fetched from urllib.parse import urljoin current_url = url + # One Session for the whole chain: a bare requests.get() per hop + # starts with an empty cookie jar, so a host that sets a cookie + # and then redirects (common for gated/CDN-signed PDFs) would + # get its own cookie back. allow_redirects=True used to carry + # them for us; following hops by hand means we carry them here. + session = requests.Session() for _ in range(MAX_PDF_DOWNLOAD_REDIRECTS): if self.url_validator: self.url_validator(current_url) - response = requests.get(current_url, stream=True, timeout=(20, 60 * 10), - allow_redirects=False) + response = session.get(current_url, stream=True, timeout=(20, 60 * 10), + allow_redirects=False) if response.is_redirect: location = response.headers.get("location") response.close() # hop response holds its connection open (stream=True) diff --git a/tests/test_docker_pdf_crawler_pairing.py b/tests/test_docker_pdf_crawler_pairing.py index 345c6c420..c6c5f1f1a 100644 --- a/tests/test_docker_pdf_crawler_pairing.py +++ b/tests/test_docker_pdf_crawler_pairing.py @@ -154,6 +154,21 @@ def do_GET(self): self.send_response(302) self.send_header("Location", "/loop") self.end_headers() + elif self.path == "/gated": + # Sets a cookie, then redirects to a target that demands it. + self.send_response(302) + self.send_header("Set-Cookie", "sess=abc123; Path=/") + self.send_header("Location", "/gated-doc.pdf") + self.end_headers() + elif self.path == "/gated-doc.pdf": + if "sess=abc123" not in self.headers.get("Cookie", ""): + self.send_response(403) + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Type", "application/pdf") + self.end_headers() + self.wfile.write(pdf_bytes) else: self.send_response(200) self.send_header("Content-Type", "application/pdf") @@ -199,6 +214,23 @@ def test_download_redirects_still_followed_without_validator(redirect_server): Path(path).unlink(missing_ok=True) +def test_download_carries_cookies_across_redirect_hops(redirect_server): + """Cookies set by a redirecting host must reach the next hop. + + Following redirects by hand (needed so url_validator can vet each hop) + means a bare requests.get() per hop starts with an empty cookie jar, so + a host that sets a cookie and then redirects never gets it back. That is + the normal shape for gated or CDN-signed PDFs, and allow_redirects=True + used to handle it implicitly. + """ + strategy = PDFContentScrapingStrategy() + path = strategy._get_pdf_path(f"{redirect_server}/gated") + try: + assert Path(path).read_bytes().startswith(b"%PDF") + finally: + Path(path).unlink(missing_ok=True) + + def test_download_redirect_loop_aborts(redirect_server): """An endless redirect chain must abort after the cap, not hang.""" strategy = PDFContentScrapingStrategy()