Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion scripts/postprocess_generated_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
RESOURCE_INPUT_TYPEDDICTS: frozenset[str] = frozenset(
{
'Request', # RequestQueueClient.update_request
'RequestDraft', # RequestQueueClient.add_request, batch_add_requests
'RequestWithoutId', # RequestQueueClient.add_request, batch_add_requests
'RequestDraftDelete', # RequestQueueClient.batch_delete_requests
'TaskInput', # Actor/Task start/call/update default input
'WebhookCreate', # Actor/Task start/call webhook list element
Expand Down
47 changes: 24 additions & 23 deletions src/apify_client/_resource_clients/request_queue.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
RequestQueueResponse,
RequestRegistration,
RequestResponse,
RequestWithoutId,
UnlockRequestsResponse,
UnlockRequestsResult,
)
Expand All @@ -50,10 +51,10 @@
from apify_client._typeddicts import (
RequestCamelDict,
RequestDict,
RequestDraftCamelDict,
RequestDraftDeleteCamelDict,
RequestDraftDeleteDict,
RequestDraftDict,
RequestWithoutIdCamelDict,
RequestWithoutIdDict,
)
from apify_client.types import Timeout

Expand All @@ -68,7 +69,7 @@


def _serialize_requests(
requests: list[RequestDraft] | list[RequestDraftDict] | list[RequestDraftCamelDict],
requests: list[RequestWithoutId] | list[RequestWithoutIdDict] | list[RequestWithoutIdCamelDict],
) -> list[bytes]:
"""Validate requests and serialize each one into the JSON bytes it will occupy in the batch request body.

Expand All @@ -80,8 +81,8 @@ def _serialize_requests(
"""
return [
json.dumps(
(request if isinstance(request, RequestDraft) else RequestDraft.model_validate(request)).model_dump(
by_alias=True, exclude_none=True
(request if isinstance(request, RequestWithoutId) else RequestWithoutId.model_validate(request)).model_dump(
mode='json', by_alias=True, exclude_none=True, fallback=str
),
ensure_ascii=False,
allow_nan=False,
Expand Down Expand Up @@ -228,7 +229,7 @@ def list_and_lock_head(

def add_request(
self,
request: RequestDraftDict | RequestDraftCamelDict | RequestDraft,
request: RequestWithoutIdDict | RequestWithoutIdCamelDict | RequestWithoutId,
*,
forefront: bool | None = None,
timeout: Timeout = 'short',
Expand All @@ -238,22 +239,22 @@ def add_request(
https://docs.apify.com/api/v2#/reference/request-queues/request-collection/add-request

Args:
request: The request to add to the queue.
request: The request to add to the queue. Must carry a `unique_key` and a `url`.
forefront: Whether to add the request to the head or the end of the queue.
timeout: Timeout for the API HTTP request.

Returns:
The added request.
"""
if not isinstance(request, RequestDraft):
request = RequestDraft.model_validate(request)
if not isinstance(request, RequestWithoutId):
request = RequestWithoutId.model_validate(request)

request_params = self._build_params(forefront=forefront, clientKey=self.client_key)

response = self._http_client.call(
url=self._build_url('requests'),
method='POST',
json=request.model_dump(by_alias=True, exclude_none=True),
json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str),
params=request_params,
timeout=timeout,
)
Expand Down Expand Up @@ -321,7 +322,7 @@ def update_request(
response = self._http_client.call(
url=self._build_url(f'requests/{to_path_segment(request.id)}'),
method='PUT',
json=request.model_dump(by_alias=True, exclude_none=True),
json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str),
params=request_params,
timeout=timeout,
)
Expand Down Expand Up @@ -410,7 +411,7 @@ def delete_request_lock(

def batch_add_requests(
self,
requests: list[RequestDraft] | list[RequestDraftDict] | list[RequestDraftCamelDict],
requests: list[RequestWithoutId] | list[RequestWithoutIdDict] | list[RequestWithoutIdCamelDict],
*,
forefront: bool = False,
max_parallel: int = 1,
Expand All @@ -423,7 +424,7 @@ def batch_add_requests(
https://docs.apify.com/api/v2#/reference/request-queues/batch-request-operations/add-requests

Args:
requests: List of requests to be added to the queue.
requests: List of requests to be added to the queue. Each must carry a `unique_key` and a `url`.
forefront: Whether to add requests to the front of the queue.
max_parallel: Specifies the maximum number of parallel tasks for API calls. This is only applicable
to the async client. For the sync client, this value must be set to 1, as parallel execution
Expand Down Expand Up @@ -510,7 +511,7 @@ def batch_delete_requests(
else RequestDraftDelete.model_validate(
request,
)
).model_dump(by_alias=True, exclude_none=True)
).root.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str)
for request in requests
]

Expand Down Expand Up @@ -761,7 +762,7 @@ async def list_and_lock_head(

async def add_request(
self,
request: RequestDraftDict | RequestDraftCamelDict | RequestDraft,
request: RequestWithoutIdDict | RequestWithoutIdCamelDict | RequestWithoutId,
*,
forefront: bool | None = None,
timeout: Timeout = 'short',
Expand All @@ -771,22 +772,22 @@ async def add_request(
https://docs.apify.com/api/v2#/reference/request-queues/request-collection/add-request

Args:
request: The request to add to the queue.
request: The request to add to the queue. Must carry a `unique_key` and a `url`.
forefront: Whether to add the request to the head or the end of the queue.
timeout: Timeout for the API HTTP request.

Returns:
The added request.
"""
if not isinstance(request, RequestDraft):
request = RequestDraft.model_validate(request)
if not isinstance(request, RequestWithoutId):
request = RequestWithoutId.model_validate(request)

request_params = self._build_params(forefront=forefront, clientKey=self.client_key)

response = await self._http_client.call(
url=self._build_url('requests'),
method='POST',
json=request.model_dump(by_alias=True, exclude_none=True),
json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str),
params=request_params,
timeout=timeout,
)
Expand Down Expand Up @@ -852,7 +853,7 @@ async def update_request(
response = await self._http_client.call(
url=self._build_url(f'requests/{to_path_segment(request.id)}'),
method='PUT',
json=request.model_dump(by_alias=True, exclude_none=True),
json=request.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str),
params=request_params,
timeout=timeout,
)
Expand Down Expand Up @@ -989,7 +990,7 @@ async def _batch_add_requests_worker(

async def batch_add_requests(
self,
requests: list[RequestDraft] | list[RequestDraftDict] | list[RequestDraftCamelDict],
requests: list[RequestWithoutId] | list[RequestWithoutIdDict] | list[RequestWithoutIdCamelDict],
*,
forefront: bool = False,
max_parallel: int = 5,
Expand All @@ -1002,7 +1003,7 @@ async def batch_add_requests(
https://docs.apify.com/api/v2#/reference/request-queues/batch-request-operations/add-requests

Args:
requests: List of requests to be added to the queue.
requests: List of requests to be added to the queue. Each must carry a `unique_key` and a `url`.
forefront: Whether to add requests to the front of the queue.
max_parallel: Specifies the maximum number of parallel tasks for API calls. This is only applicable
to the async client. For the sync client, this value must be set to 1, as parallel execution
Expand Down Expand Up @@ -1094,7 +1095,7 @@ async def batch_delete_requests(
else RequestDraftDelete.model_validate(
request,
)
).model_dump(by_alias=True, exclude_none=True)
).root.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str)
for request in requests
]

Expand Down
48 changes: 10 additions & 38 deletions src/apify_client/_typeddicts.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,44 +111,6 @@ class RequestCamelDict(RequestBaseCamelDict):
"""


@docs_group('Typed dicts')
class RequestDraftDict(TypedDict):
"""A request that failed to be processed during a request queue operation and can be retried."""

id: NotRequired[str]
"""
A unique identifier assigned to the request.
"""
unique_key: str
"""
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
"""
url: str
"""
The URL of the request.
"""
method: NotRequired[Literal['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']]


@docs_group('Typed dicts')
class RequestDraftCamelDict(TypedDict):
"""A request that failed to be processed during a request queue operation and can be retried."""

id: NotRequired[str]
"""
A unique identifier assigned to the request.
"""
uniqueKey: str
"""
A unique key used for request de-duplication. Requests with the same unique key are considered identical.
"""
url: str
"""
The URL of the request.
"""
method: NotRequired[Literal['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']]


@docs_group('Typed dicts')
class RequestDraftDeleteByIdDict(TypedDict):
"""A request that should be deleted, identified by its ID."""
Expand Down Expand Up @@ -221,6 +183,16 @@ class RequestDraftDeleteByUniqueKeyCamelDict(TypedDict):
RequestUserDataCamelDict: TypeAlias = dict[str, Any]


@docs_group('Typed dicts')
class RequestWithoutIdDict(RequestBaseDict):
"""A request stored in the request queue, including its metadata and processing state, without the assigned ID."""


@docs_group('Typed dicts')
class RequestWithoutIdCamelDict(RequestBaseCamelDict):
"""A request stored in the request queue, including its metadata and processing state, without the assigned ID."""


TaskInputDict: TypeAlias = dict[str, Any]

TaskInputCamelDict: TypeAlias = dict[str, Any]
Expand Down
Loading
Loading