From 6ec633406e1f68e6db1f5d63e3b422d7ee24da28 Mon Sep 17 00:00:00 2001 From: fern-api <115122769+fern-api[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 09:47:47 +0000 Subject: [PATCH 1/2] [fern-generated] Update SDK Generated by Fern CLI Version: unknown Generators: - fernapi/fern-python-sdk: 5.17.0 --- .fern/metadata.json | 6 +- README.md | 3 +- poetry.lock | 18 +- pyproject.toml | 2 +- reference.md | 139 ++++++- src/newscatcher_catchall/__init__.py | 3 + src/newscatcher_catchall/client.py | 28 +- .../core/client_wrapper.py | 22 +- src/newscatcher_catchall/core/http_client.py | 28 +- .../core/http_sse/_api.py | 365 ++++++++++++++++-- .../core/http_sse/_decoders.py | 13 + .../core/request_options.py | 7 +- src/newscatcher_catchall/datasets/client.py | 16 +- .../datasets/raw_client.py | 16 +- src/newscatcher_catchall/entities/client.py | 40 +- .../entities/raw_client.py | 20 +- src/newscatcher_catchall/jobs/client.py | 8 + src/newscatcher_catchall/jobs/raw_client.py | 8 + src/newscatcher_catchall/types/__init__.py | 3 + .../types/company_attributes.py | 5 - .../types/create_entity_request.py | 6 +- .../types/dataset_response.py | 18 + .../types/formatter_config_dto.py | 35 ++ .../types/initialize_response_dto.py | 5 +- .../types/webhook_response_dto.py | 5 +- src/newscatcher_catchall/webhooks/client.py | 149 ++++++- .../webhooks/raw_client.py | 227 ++++++++++- tests/utils/test_http_client.py | 69 +++- 28 files changed, 1109 insertions(+), 155 deletions(-) create mode 100644 src/newscatcher_catchall/types/formatter_config_dto.py diff --git a/.fern/metadata.json b/.fern/metadata.json index 0ff0268..e3a258d 100644 --- a/.fern/metadata.json +++ b/.fern/metadata.json @@ -1,7 +1,7 @@ { - "cliVersion": "5.50.6", + "cliVersion": "5.66.0", "generatorName": "fernapi/fern-python-sdk", - "generatorVersion": "5.14.21", + "generatorVersion": "5.17.0", "generatorConfig": { "package_name": "newscatcher_catchall", "client": { @@ -9,7 +9,7 @@ "filename": "client.py" } }, - "originGitCommit": "f9a7ba07dda24e9d9936bb543686bc542355479e", + "originGitCommit": "1e66c85a387d0d844162c965656c6a0d36334c11", "originGitCommitIsDirty": true, "invokedBy": "ci", "requestedVersion": "AUTO", diff --git a/README.md b/README.md index e2f2fc4..e486cdc 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ client = CatchAllApi(..., timeout=20.0) # Override timeout for a specific method client.jobs.create_job(..., request_options={ - "timeout_in_seconds": 1 + "timeout": 1 }) ``` @@ -219,3 +219,4 @@ On the other hand, contributions to the README are always very welcome! + diff --git a/poetry.lock b/poetry.lock index bc07681..a7caaa7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,15 +2,15 @@ [[package]] name = "aiohappyeyeballs" -version = "2.6.2" +version = "2.7.1" description = "Happy Eyeballs for asyncio" optional = true python-versions = ">=3.10" groups = ["main"] markers = "extra == \"aiohttp\"" files = [ - {file = "aiohappyeyeballs-2.6.2-py3-none-any.whl", hash = "sha256:4708045e2d7a6c6bdf8aafa8ed39649eaf926a4543b54560659129e3365953c4"}, - {file = "aiohappyeyeballs-2.6.2.tar.gz", hash = "sha256:e202810ee718bd01fc6ef49e8ea53d023d5cb6b581076d7925aa499fa55dbe64"}, + {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, + {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, ] [[package]] @@ -188,14 +188,14 @@ files = [ [[package]] name = "anyio" -version = "4.14.0" +version = "4.14.1" description = "High-level concurrency and networking framework on top of asyncio or Trio" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "anyio-4.14.0-py3-none-any.whl", hash = "sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9"}, - {file = "anyio-4.14.0.tar.gz", hash = "sha256:b47c1f9ccf73e67021df785332508f99379c68fa7d0684e8e3492cb1d4b23f89"}, + {file = "anyio-4.14.1-py3-none-any.whl", hash = "sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72"}, + {file = "anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e"}, ] [package.dependencies] @@ -1297,14 +1297,14 @@ files = [ [[package]] name = "typing-extensions" -version = "4.15.0" +version = "4.16.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, - {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, + {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, + {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 958beb2..c6579e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ dynamic = ["version"] [tool.poetry] name = "newscatcher-catchall-sdk" -version = "3.1.0" +version = "0.0.0.dev0" description = "" readme = "README.md" authors = [] diff --git a/reference.md b/reference.md index cd104dc..550ac17 100644 --- a/reference.md +++ b/reference.md @@ -385,6 +385,10 @@ client.jobs.create_job( Custom validators for filtering web page clusters. If not provided, validators are generated automatically based on the query. +The system may also inject validators during the `analyzing` stage — for +example, converting a relative time qualifier into an explicit event-date +gate. The returned `validators[]` in the job status shows the complete +applied set, including any system-added ones. @@ -1996,7 +2000,7 @@ Authentication forwarded with each delivery. Supported types:
-**formatter_config:** `typing.Optional[typing.Dict[str, typing.Any]]` — Custom payload transformation configuration. Required only when `type` is `custom`. +**formatter_config:** `typing.Optional[FormatterConfigDto]` — Custom payload formatter. Required when `type` is `custom`.
@@ -2291,7 +2295,7 @@ client.webhooks.update_webhook(
-**formatter_config:** `typing.Optional[typing.Dict[str, typing.Any]]` — Updated formatter configuration. +**formatter_config:** `typing.Optional[FormatterConfigDto]` — Updated custom payload formatter. Set only when `type` is `custom`.
@@ -2796,6 +2800,115 @@ client.webhooks.list_webhooks_for_resource( + + + + +
client.webhooks.trigger_webhook(...) +
+
+ +#### 📝 Description + +
+
+ +
+
+ +Manually dispatches a webhook delivery for a resource on demand, without +waiting for the next job or monitor cycle. + +Use this to re-deliver results after a failed delivery, replay a specific +job's results, or validate a webhook against live data. The webhook must +already be assigned to the resource. +
+
+
+
+ +#### 🔌 Usage + +
+
+ +
+
+ +```python +from newscatcher_catchall import CatchAllApi +from newscatcher_catchall.environment import CatchAllApiEnvironment + +client = CatchAllApi( + api_key="", + environment=CatchAllApiEnvironment.DEFAULT, +) + +client.webhooks.trigger_webhook( + resource_type="job", + resource_id="3fec5b07-8786-46d7-9486-d43ff67eccd4", + webhook_id="a1b2c3d4-e5f6-7890-abcd-ef1234567890", + job_id="3fec5b07-8786-46d7-9486-d43ff67eccd4", +) + +``` +
+
+
+
+ +#### ⚙️ Parameters + +
+
+ +
+
+ +**resource_type:** `MappableResourceType` + +
+
+ +
+
+ +**resource_id:** `str` — Unique resource identifier. + +
+
+ +
+
+ +**webhook_id:** `str` — Identifier of the webhook to trigger. Must be assigned to the resource. + +
+
+ +
+
+ +**job_id:** `typing.Optional[str]` + +Specific job run to deliver. Optional for `job` resources; for +`monitor` and `monitor_group` resources, selects a past run to replay. +When omitted, the latest available results are delivered. + +
+
+ +
+
+ +**request_options:** `typing.Optional[RequestOptions]` — Request-specific configuration. + +
+
+
+
+ +
@@ -3036,10 +3149,10 @@ client.entities.list_entities( Creates a new company entity and begins background enrichment. +Each entity requires a `name` plus at least one of: a `description` or a `domain`. Providing both is recommended — `domain` is the highest-signal identifier because it is unambiguous; a well-written `description` is the best alternative when no domain is available. + The entity status starts as `pending` and transitions to `ready` once -enrichment completes. Provide as much identifying information as -possible — `domain` is the highest-signal field because it is -unambiguous. +enrichment completes. @@ -3065,7 +3178,7 @@ client = CatchAllApi( client.entities.create_entity( name="NewsCatcher", entity_type="company", - description="AI-powered news data provider", + description="NewsCatcher is a data-as-a-service company providing news intelligence APIs including the CatchAll Web Search API (2B+ web pages indexed) and News API (140,000+ sources, 100+ countries).", additional_attributes=AdditionalAttributes( company_attributes=CompanyAttributes( domain="newscatcherapi.com", @@ -3074,7 +3187,7 @@ client.entities.create_entity( "Maksym Sugonyaka" ], alternative_names=[ - "NC", + "NewsCatcher CatchAll", "NewsCatcher API" ], ), @@ -3129,6 +3242,8 @@ client.entities.create_entity( Creates multiple entities in a single request. Each entity is processed independently — a failure in one does not affect others. +Each entity requires a `name` plus at least one of: a `description` or a `domain`. See [Create entity](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entity) for the full field reference. + Returns an array of `{id, status}` objects in the same order as the input array. @@ -3412,7 +3527,7 @@ client.entities.update_entity( additional_attributes=AdditionalAttributes( company_attributes=CompanyAttributes( alternative_names=[ - "NC", + "NewsCatcher CatchAll", "NewsCatcher API", "NCA" ], @@ -3629,7 +3744,9 @@ client.datasets.list_datasets(
-Creates a new dataset from a list of existing entity IDs. +Creates a new dataset from a list of existing entity IDs. The optional `description` field here describes the dataset itself — it is separate from the entity-level `description` used for matching. + +Entities must be created before adding them to a dataset. Each entity requires a `name` plus at least one of: a `description` or a `domain`. Use [Create entity](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entity) or [Create entities batch](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entities-batch) to create entities first. If any of the provided entity IDs do not exist or do not belong to your organization, the request fails with `400`. All entity IDs must @@ -3730,12 +3847,12 @@ client.datasets.create_dataset(
-Creates a new dataset by uploading a CSV file. Each row in the CSV becomes an entity. The `name` and `domain`columns are required; all other columns are optional. +Creates a new dataset by uploading a CSV file. Each row in the CSV becomes an entity. Each row requires a `name` plus at least one of: a `description` or a `domain`; all other columns are optional. Note: `description` in the CSV is the entity's matching description — it is separate from the dataset-level `description` field in the form data. **CSV format:** ```csv name,description,domain,alternative_names,key_persons -NewsCatcher,"AI-powered news data provider",newscatcherapi.com,"NC;NewsCatcher API","Artem Bugara;Maksym Sugonyaka" +NewsCatcher,"NewsCatcher is a data-as-a-service company providing news intelligence APIs including the CatchAll Web Search API (2B+ web pages indexed) and News API (140,000+ sources, 100+ countries).",newscatcherapi.com,"NewsCatcher CatchAll;NewsCatcher API","Artem Bugara;Maksym Sugonyaka" OpenAI,"Artificial intelligence research company",openai.com,"Open AI","Sam Altman" ``` diff --git a/src/newscatcher_catchall/__init__.py b/src/newscatcher_catchall/__init__.py index b217417..29bbcf3 100644 --- a/src/newscatcher_catchall/__init__.py +++ b/src/newscatcher_catchall/__init__.py @@ -58,6 +58,7 @@ EntityType, EntityValidationErrorBody, Error, + FormatterConfigDto, GetPlanLimitsResponseDto, GetWebhookResponseDto, HttpMethod, @@ -220,6 +221,7 @@ "EntityValidationErrorBody": ".types", "Error": ".types", "ForbiddenError": ".errors", + "FormatterConfigDto": ".types", "GetPlanLimitsResponseDto": ".types", "GetVersionResponse": ".meta", "GetWebhookResponseDto": ".types", @@ -401,6 +403,7 @@ def __dir__(): "EntityValidationErrorBody", "Error", "ForbiddenError", + "FormatterConfigDto", "GetPlanLimitsResponseDto", "GetVersionResponse", "GetWebhookResponseDto", diff --git a/src/newscatcher_catchall/client.py b/src/newscatcher_catchall/client.py index b7f052b..8721155 100644 --- a/src/newscatcher_catchall/client.py +++ b/src/newscatcher_catchall/client.py @@ -47,6 +47,12 @@ class CatchAllApi: max_retries : typing.Optional[int] The default maximum number of retries for failed requests. Defaults to 2. Per-request `max_retries` in `request_options` takes precedence over this value. + stream_reconnection_enabled : typing.Optional[bool] + Whether to automatically reconnect on stream disconnection for resumable streaming endpoints. Defaults to True. Per-request `stream_reconnection_enabled` in `request_options` takes precedence over this value. + + max_stream_reconnection_attempts : typing.Optional[int] + The maximum number of reconnection attempts for resumable streaming endpoints. Defaults to no limit. Per-request `max_stream_reconnection_attempts` in `request_options` takes precedence over this value. + follow_redirects : typing.Optional[bool] Whether the default httpx client follows redirects or not, this is irrelevant if a custom httpx client is passed in. @@ -74,13 +80,13 @@ def __init__( headers: typing.Optional[typing.Dict[str, str]] = None, timeout: typing.Optional[float] = None, max_retries: typing.Optional[int] = None, + stream_reconnection_enabled: typing.Optional[bool] = None, + max_stream_reconnection_attempts: typing.Optional[int] = None, follow_redirects: typing.Optional[bool] = True, httpx_client: typing.Optional[httpx.Client] = None, logging: typing.Optional[typing.Union[LogConfig, Logger]] = None, ): - _defaulted_timeout = ( - timeout if timeout is not None else 60 if httpx_client is None else httpx_client.timeout.read - ) + _defaulted_timeout = timeout if timeout is not None else 60 if httpx_client is None else None _defaulted_max_retries = max_retries if max_retries is not None else 2 self._client_wrapper = SyncClientWrapper( base_url=_get_base_url(base_url=base_url, environment=environment), @@ -93,6 +99,8 @@ def __init__( else httpx.Client(timeout=_defaulted_timeout), timeout=_defaulted_timeout, max_retries=_defaulted_max_retries, + stream_reconnection_enabled=stream_reconnection_enabled, + max_stream_reconnection_attempts=max_stream_reconnection_attempts, logging=logging, ) self._jobs: typing.Optional[JobsClient] = None @@ -206,6 +214,12 @@ class AsyncCatchAllApi: max_retries : typing.Optional[int] The default maximum number of retries for failed requests. Defaults to 2. Per-request `max_retries` in `request_options` takes precedence over this value. + stream_reconnection_enabled : typing.Optional[bool] + Whether to automatically reconnect on stream disconnection for resumable streaming endpoints. Defaults to True. Per-request `stream_reconnection_enabled` in `request_options` takes precedence over this value. + + max_stream_reconnection_attempts : typing.Optional[int] + The maximum number of reconnection attempts for resumable streaming endpoints. Defaults to no limit. Per-request `max_stream_reconnection_attempts` in `request_options` takes precedence over this value. + follow_redirects : typing.Optional[bool] Whether the default httpx client follows redirects or not, this is irrelevant if a custom httpx client is passed in. @@ -233,13 +247,13 @@ def __init__( headers: typing.Optional[typing.Dict[str, str]] = None, timeout: typing.Optional[float] = None, max_retries: typing.Optional[int] = None, + stream_reconnection_enabled: typing.Optional[bool] = None, + max_stream_reconnection_attempts: typing.Optional[int] = None, follow_redirects: typing.Optional[bool] = True, httpx_client: typing.Optional[httpx.AsyncClient] = None, logging: typing.Optional[typing.Union[LogConfig, Logger]] = None, ): - _defaulted_timeout = ( - timeout if timeout is not None else 60 if httpx_client is None else httpx_client.timeout.read - ) + _defaulted_timeout = timeout if timeout is not None else 60 if httpx_client is None else None _defaulted_max_retries = max_retries if max_retries is not None else 2 self._client_wrapper = AsyncClientWrapper( base_url=_get_base_url(base_url=base_url, environment=environment), @@ -250,6 +264,8 @@ def __init__( else _make_default_async_client(timeout=_defaulted_timeout, follow_redirects=follow_redirects), timeout=_defaulted_timeout, max_retries=_defaulted_max_retries, + stream_reconnection_enabled=stream_reconnection_enabled, + max_stream_reconnection_attempts=max_stream_reconnection_attempts, logging=logging, ) self._jobs: typing.Optional[AsyncJobsClient] = None diff --git a/src/newscatcher_catchall/core/client_wrapper.py b/src/newscatcher_catchall/core/client_wrapper.py index aeb35d6..8764f39 100644 --- a/src/newscatcher_catchall/core/client_wrapper.py +++ b/src/newscatcher_catchall/core/client_wrapper.py @@ -16,6 +16,8 @@ def __init__( base_url: str, timeout: typing.Optional[float] = None, max_retries: int = 2, + stream_reconnection_enabled: typing.Optional[bool] = None, + max_stream_reconnection_attempts: typing.Optional[int] = None, logging: typing.Optional[typing.Union[LogConfig, Logger]] = None, ): self.api_key = api_key @@ -23,18 +25,20 @@ def __init__( self._base_url = base_url self._timeout = timeout self._max_retries = max_retries + self._stream_reconnection_enabled = stream_reconnection_enabled + self._max_stream_reconnection_attempts = max_stream_reconnection_attempts self._logging = logging def get_headers(self) -> typing.Dict[str, str]: import platform headers: typing.Dict[str, str] = { - "User-Agent": "newscatcher-catchall-sdk/3.1.0", + "User-Agent": "newscatcher-catchall-sdk/0.0.0-fern-placeholder", "X-Fern-Language": "Python", "X-Fern-Runtime": f"python/{platform.python_version()}", "X-Fern-Platform": f"{platform.system().lower()}/{platform.release()}", "X-Fern-SDK-Name": "newscatcher-catchall-sdk", - "X-Fern-SDK-Version": "3.1.0", + "X-Fern-SDK-Version": "0.0.0.dev0", **(self.get_custom_headers() or {}), } headers["x-api-key"] = self.api_key @@ -52,6 +56,12 @@ def get_timeout(self) -> typing.Optional[float]: def get_max_retries(self) -> int: return self._max_retries + def get_stream_reconnection_enabled(self) -> bool: + return self._stream_reconnection_enabled if self._stream_reconnection_enabled is not None else True + + def get_max_stream_reconnection_attempts(self) -> typing.Optional[int]: + return self._max_stream_reconnection_attempts + class SyncClientWrapper(BaseClientWrapper): def __init__( @@ -62,6 +72,8 @@ def __init__( base_url: str, timeout: typing.Optional[float] = None, max_retries: int = 2, + stream_reconnection_enabled: typing.Optional[bool] = None, + max_stream_reconnection_attempts: typing.Optional[int] = None, logging: typing.Optional[typing.Union[LogConfig, Logger]] = None, httpx_client: httpx.Client, ): @@ -71,6 +83,8 @@ def __init__( base_url=base_url, timeout=timeout, max_retries=max_retries, + stream_reconnection_enabled=stream_reconnection_enabled, + max_stream_reconnection_attempts=max_stream_reconnection_attempts, logging=logging, ) self.httpx_client = HttpClient( @@ -92,6 +106,8 @@ def __init__( base_url: str, timeout: typing.Optional[float] = None, max_retries: int = 2, + stream_reconnection_enabled: typing.Optional[bool] = None, + max_stream_reconnection_attempts: typing.Optional[int] = None, logging: typing.Optional[typing.Union[LogConfig, Logger]] = None, async_token: typing.Optional[typing.Callable[[], typing.Awaitable[str]]] = None, httpx_client: httpx.AsyncClient, @@ -102,6 +118,8 @@ def __init__( base_url=base_url, timeout=timeout, max_retries=max_retries, + stream_reconnection_enabled=stream_reconnection_enabled, + max_stream_reconnection_attempts=max_stream_reconnection_attempts, logging=logging, ) self._async_token = async_token diff --git a/src/newscatcher_catchall/core/http_client.py b/src/newscatcher_catchall/core/http_client.py index f686c57..124dce2 100644 --- a/src/newscatcher_catchall/core/http_client.py +++ b/src/newscatcher_catchall/core/http_client.py @@ -312,11 +312,14 @@ def request( force_multipart: typing.Optional[bool] = None, ) -> httpx.Response: base_url = self.get_base_url(base_url) - timeout = ( - request_options.get("timeout_in_seconds") + _timeout = ( + request_options.get("timeout") + if request_options is not None and request_options.get("timeout") is not None + else request_options.get("timeout_in_seconds") if request_options is not None and request_options.get("timeout_in_seconds") is not None else self.base_timeout() ) + timeout = _timeout if _timeout is not None else httpx.USE_CLIENT_DEFAULT json_body, data_body = get_request_body(json=json, data=data, request_options=request_options, omit=omit) @@ -472,11 +475,14 @@ def stream( force_multipart: typing.Optional[bool] = None, ) -> typing.Iterator[httpx.Response]: base_url = self.get_base_url(base_url) - timeout = ( - request_options.get("timeout_in_seconds") + _timeout = ( + request_options.get("timeout") + if request_options is not None and request_options.get("timeout") is not None + else request_options.get("timeout_in_seconds") if request_options is not None and request_options.get("timeout_in_seconds") is not None else self.base_timeout() ) + timeout = _timeout if _timeout is not None else httpx.USE_CLIENT_DEFAULT request_files: typing.Optional[RequestFiles] = ( convert_file_dict_to_httpx_tuples(remove_omit_from_dict(remove_none_from_dict(files), omit)) @@ -601,11 +607,14 @@ async def request( force_multipart: typing.Optional[bool] = None, ) -> httpx.Response: base_url = self.get_base_url(base_url) - timeout = ( - request_options.get("timeout_in_seconds") + _timeout = ( + request_options.get("timeout") + if request_options is not None and request_options.get("timeout") is not None + else request_options.get("timeout_in_seconds") if request_options is not None and request_options.get("timeout_in_seconds") is not None else self.base_timeout() ) + timeout = _timeout if _timeout is not None else httpx.USE_CLIENT_DEFAULT request_files: typing.Optional[RequestFiles] = ( convert_file_dict_to_httpx_tuples(remove_omit_from_dict(remove_none_from_dict(files), omit)) @@ -764,11 +773,14 @@ async def stream( force_multipart: typing.Optional[bool] = None, ) -> typing.AsyncIterator[httpx.Response]: base_url = self.get_base_url(base_url) - timeout = ( - request_options.get("timeout_in_seconds") + _timeout = ( + request_options.get("timeout") + if request_options is not None and request_options.get("timeout") is not None + else request_options.get("timeout_in_seconds") if request_options is not None and request_options.get("timeout_in_seconds") is not None else self.base_timeout() ) + timeout = _timeout if _timeout is not None else httpx.USE_CLIENT_DEFAULT request_files: typing.Optional[RequestFiles] = ( convert_file_dict_to_httpx_tuples(remove_omit_from_dict(remove_none_from_dict(files), omit)) diff --git a/src/newscatcher_catchall/core/http_sse/_api.py b/src/newscatcher_catchall/core/http_sse/_api.py index fd13730..9ca5602 100644 --- a/src/newscatcher_catchall/core/http_sse/_api.py +++ b/src/newscatcher_catchall/core/http_sse/_api.py @@ -2,9 +2,20 @@ import codecs import re +import time from contextlib import asynccontextmanager, contextmanager -from typing import Any, AsyncGenerator, AsyncIterator, Iterator - +from typing import ( + Any, + AsyncContextManager, + AsyncGenerator, + AsyncIterator, + Callable, + ContextManager, + Iterator, + Optional, +) + +import anyio import httpx from ._decoders import SSEDecoder from ._exceptions import SSEError @@ -12,21 +23,60 @@ MAX_LINE_SIZE: int = 1_048_576 # 1 MiB +# Reconnection defaults, mirroring the TypeScript SDK's Stream implementation. +DEFAULT_MAX_RECONNECTION_ATTEMPTS: int = 5 +DEFAULT_RECONNECT_DELAY_MS: int = 1_000 +MAX_RECONNECT_DELAY_MS: int = 30_000 + +# A reconnect callback re-issues the original request (with a ``Last-Event-ID`` +# header set to the supplied event id) and returns a *context manager* yielding +# a fresh streaming ``httpx.Response``. Sync clients supply a sync context +# manager; async clients supply an async one. class EventSource: - def __init__(self, response: httpx.Response) -> None: + def __init__( + self, + response: httpx.Response, + *, + resumable: bool = False, + stream_reconnection_enabled: bool = True, + max_stream_reconnection_attempts: Optional[int] = None, + stream_terminator: Optional[str] = None, + reconnect: Optional[Callable[[str], Any]] = None, + ) -> None: self._response = response + self._resumable = resumable + self._stream_reconnection_enabled = stream_reconnection_enabled + self._max_stream_reconnection_attempts = max_stream_reconnection_attempts + self._stream_terminator = stream_terminator + self._reconnect = reconnect + + @staticmethod + def _is_event_stream(response: httpx.Response) -> bool: + content_type = response.headers.get("content-type", "").partition(";")[0] + return "text/event-stream" in content_type def _check_content_type(self) -> None: - content_type = self._response.headers.get("content-type", "").partition(";")[0] - if "text/event-stream" not in content_type: + if not self._is_event_stream(self._response): + content_type = self._response.headers.get("content-type", "").partition(";")[0] raise SSEError( f"Expected response header Content-Type to contain 'text/event-stream', got {content_type!r}" ) - def _get_charset(self) -> str: + def _is_reconnect_response_usable(self, response: httpx.Response) -> bool: + """Whether a reconnected response can be resumed as an SSE stream. + + ``httpx.stream`` does not raise on non-success status, so a resume that + returns an error page (e.g. ``200 text/html`` or a ``500`` body) would + otherwise be parsed as SSE and yield garbage/zero events. Such a + response is treated as a failed attempt (back off and retry) instead. + """ + return response.status_code < 400 and self._is_event_stream(response) + + def _get_charset(self, response: Optional[httpx.Response] = None) -> str: """Extract charset from Content-Type header, fallback to UTF-8.""" - content_type = self._response.headers.get("content-type", "") + resolved = response if response is not None else self._response + content_type = resolved.headers.get("content-type", "") # Parse charset parameter using regex charset_match = re.search(r"charset=([^;\s]+)", content_type, re.IGNORECASE) @@ -60,14 +110,83 @@ def _normalize_sse_line_endings(buf: str) -> str: return buf[:-1].replace("\r", "\n") + "\r" return buf.replace("\r", "\n") - def iter_sse(self) -> Iterator[ServerSentEvent]: - self._check_content_type() - decoder = SSEDecoder() - charset = self._get_charset() - text_decoder = codecs.getincrementaldecoder(charset)(errors="replace") - + def _new_text_decoder(self, response: Optional[httpx.Response] = None) -> "codecs.IncrementalDecoder": + return codecs.getincrementaldecoder(self._get_charset(response))(errors="replace") + + def _reconnect_applicable(self) -> bool: + """Whether reconnection is configured for this stream at all. + + This is the terminator-gating half of the reconnect decision, kept + separate from :meth:`_should_reconnect` (which additionally requires a + last *dispatched* id and an unexhausted attempt budget). The split lets + a mid-stream transport error terminate consistently: + - a stream that can never reconnect (non-resumable, no terminator, + disabled, or no callback) must re-raise the error to the caller, so a + truncated stream is not mistaken for a clean completion; + - a resumable stream that has merely run out of attempts (or has no id + to resume from) ends cleanly — the same way an exhausted empty/error + -body resume already does, matching the TypeScript ``return``. + """ + return ( + self._resumable + and self._stream_terminator is not None + and self._stream_reconnection_enabled + and self._reconnect is not None + ) + + def _should_reconnect(self, last_dispatched_id: Optional[str], reconnect_attempts: int) -> bool: + """Decide whether a prematurely-ended stream should be reconnected. + + Mirrors the TypeScript ``shouldReconnect`` gating: + - only resumable SSE endpoints with a configured terminator, reconnect + enabled, and a reconnect callback are eligible (see + :meth:`_reconnect_applicable`); + - a last *dispatched* event id must exist to resume from; + - the consecutive-failed-attempt cap must not be exceeded. + """ + if not self._reconnect_applicable(): + return False + if not last_dispatched_id: + return False + max_attempts = ( + self._max_stream_reconnection_attempts + if self._max_stream_reconnection_attempts is not None + else DEFAULT_MAX_RECONNECTION_ATTEMPTS + ) + if reconnect_attempts >= max_attempts: + return False + return True + + def _reconnect_delay_seconds(self, last_retry: Optional[int]) -> float: + """Backoff before a reconnect. + + Uses the server's most recent ``retry:`` directive (milliseconds) when + present, otherwise a default of ``DEFAULT_RECONNECT_DELAY_MS``, clamped + to ``MAX_RECONNECT_DELAY_MS``. + """ + base_ms = last_retry if (last_retry is not None and last_retry > 0) else DEFAULT_RECONNECT_DELAY_MS + return min(base_ms, MAX_RECONNECT_DELAY_MS) / 1000.0 + + def _sleep_before_reconnect(self, last_retry: Optional[int]) -> None: + # ``time.sleep`` blocks the calling thread but remains interruptible by + # signals (e.g. ``KeyboardInterrupt``), which propagate out and abort + # the reconnect without issuing another request. + time.sleep(self._reconnect_delay_seconds(last_retry)) + + async def _asleep_before_reconnect(self, last_retry: Optional[int]) -> None: + # ``anyio.sleep`` is cancellation-aware: if the consumer cancels the task + # or closes the async generator mid-delay, this raises (and no further + # request is issued) instead of blocking for the whole interval. + await anyio.sleep(self._reconnect_delay_seconds(last_retry)) + + def _decode_response( + self, + response: httpx.Response, + decoder: SSEDecoder, + text_decoder: "codecs.IncrementalDecoder", + ) -> Iterator[ServerSentEvent]: buf = "" - for chunk in self._response.iter_bytes(): + for chunk in response.iter_bytes(): buf += text_decoder.decode(chunk) buf = self._normalize_sse_line_endings(buf) @@ -82,34 +201,16 @@ def iter_sse(self) -> Iterator[ServerSentEvent]: f"SSE line exceeded maximum size of {MAX_LINE_SIZE} characters without encountering a newline" ) - # Flush any remaining bytes from the incremental decoder - buf += text_decoder.decode(b"", final=True) - buf = buf.replace("\r\n", "\n").replace("\r", "\n") - - if len(buf) > MAX_LINE_SIZE: - raise SSEError( - f"SSE line exceeded maximum size of {MAX_LINE_SIZE} characters without encountering a newline" - ) - - while "\n" in buf: - line, buf = buf.split("\n", 1) - sse = decoder.decode(line) - if sse is not None: - yield sse - - if buf.strip(): - sse = decoder.decode(buf) - if sse is not None: - yield sse - - async def aiter_sse(self) -> AsyncGenerator[ServerSentEvent, None]: - self._check_content_type() - decoder = SSEDecoder() - charset = self._get_charset() - text_decoder = codecs.getincrementaldecoder(charset)(errors="replace") + yield from self._flush_decoder(buf, decoder, text_decoder) + async def _adecode_response( + self, + response: httpx.Response, + decoder: SSEDecoder, + text_decoder: "codecs.IncrementalDecoder", + ) -> AsyncGenerator[ServerSentEvent, None]: buf = "" - async for chunk in self._response.aiter_bytes(): + async for chunk in response.aiter_bytes(): buf += text_decoder.decode(chunk) buf = self._normalize_sse_line_endings(buf) @@ -124,6 +225,15 @@ async def aiter_sse(self) -> AsyncGenerator[ServerSentEvent, None]: f"SSE line exceeded maximum size of {MAX_LINE_SIZE} characters without encountering a newline" ) + for sse in self._flush_decoder(buf, decoder, text_decoder): + yield sse + + def _flush_decoder( + self, + buf: str, + decoder: SSEDecoder, + text_decoder: "codecs.IncrementalDecoder", + ) -> Iterator[ServerSentEvent]: # Flush any remaining bytes from the incremental decoder buf += text_decoder.decode(b"", final=True) buf = buf.replace("\r\n", "\n").replace("\r", "\n") @@ -144,6 +254,181 @@ async def aiter_sse(self) -> AsyncGenerator[ServerSentEvent, None]: if sse is not None: yield sse + def iter_sse(self) -> Iterator[ServerSentEvent]: + self._check_content_type() + decoder = SSEDecoder() + text_decoder = self._new_text_decoder() + + last_dispatched_id: Optional[str] = None + last_retry: Optional[int] = None + # Consecutive failed reconnection attempts. Reset to 0 whenever an event + # is successfully dispatched (reset-on-progress) — matching browser + # `EventSource` semantics: a server that emits >=1 event then drops on + # every connection can reconnect indefinitely. + reconnect_attempts = 0 + + # ``None`` means there is no live stream to read this iteration (e.g. a + # failed reconnect); the loop then re-evaluates the reconnect decision + # without re-reading an exhausted response. + response: Optional[httpx.Response] = self._response + # Context manager for a response we opened ourselves and must close. + # The initial response is owned by the caller, so it starts as None. + owned_cm: Optional[ContextManager[httpx.Response]] = None + try: + while True: + if response is not None: + events = self._decode_response(response, decoder, text_decoder) + while True: + try: + sse = next(events) + except StopIteration: + break + except SSEError: + # A protocol violation (e.g. an oversized line) is a + # genuine error, not a dropped connection; propagate it. + # Listed first because ``SSEError`` subclasses + # ``httpx.TransportError``. + raise + except httpx.TransportError: + # A transport error mid-stream (e.g. the server dropped + # the connection: ``ReadError``/``RemoteProtocolError``) + # is a premature end. Only swallow it when reconnection + # is configured for this stream; otherwise re-raise so a + # non-resumable stream still surfaces the error to the + # caller instead of looking like a clean completion. + # When reconnection is applicable but the attempt budget + # is exhausted, we ``break`` and end cleanly below — the + # same way an exhausted empty/error-body resume does, so + # give-up is consistent regardless of failure shape. + # ``next`` is used rather than ``for`` so this cannot + # swallow a ``GeneratorExit`` raised at a ``yield``. + if not self._reconnect_applicable(): + raise + break + yield sse + if sse.id: + last_dispatched_id = sse.id + if sse.retry is not None: + last_retry = sse.retry + reconnect_attempts = 0 + + if not self._should_reconnect(last_dispatched_id, reconnect_attempts): + return + reconnect_attempts += 1 + + self._sleep_before_reconnect(last_retry) + + # Close the previously-opened reconnect response before opening + # a new one so we never hold more than one extra connection. + if owned_cm is not None: + owned_cm.__exit__(None, None, None) + owned_cm = None + + assert self._reconnect is not None # guaranteed by _should_reconnect + try: + cm: ContextManager[httpx.Response] = self._reconnect(last_dispatched_id or "") + new_response = cm.__enter__() + except Exception: + # A failed reconnect consumes an attempt; back off and retry. + response = None + continue + owned_cm = cm + if new_response is None or not self._is_reconnect_response_usable(new_response): + # Null/empty body or a non-SSE/error response (e.g. 204/304, + # a 500, or an HTML error page): treat as a failed attempt. + response = None + continue + + response = new_response + # Drop any partial event left over from the dropped stream, but + # keep the last event id (per the SSE spec) and start a fresh + # incremental text decoder for the new connection. + decoder.reset_in_progress_event() + text_decoder = self._new_text_decoder(new_response) + finally: + if owned_cm is not None: + owned_cm.__exit__(None, None, None) + + async def aiter_sse(self) -> AsyncGenerator[ServerSentEvent, None]: + self._check_content_type() + decoder = SSEDecoder() + text_decoder = self._new_text_decoder() + + last_dispatched_id: Optional[str] = None + last_retry: Optional[int] = None + reconnect_attempts = 0 + + response: Optional[httpx.Response] = self._response + owned_cm: Optional[AsyncContextManager[httpx.Response]] = None + try: + while True: + if response is not None: + events = self._adecode_response(response, decoder, text_decoder) + while True: + try: + sse = await events.__anext__() + except StopAsyncIteration: + break + except SSEError: + # A protocol violation (e.g. an oversized line) is a + # genuine error, not a dropped connection; propagate it. + # Listed first because ``SSEError`` subclasses + # ``httpx.TransportError``. + raise + except httpx.TransportError: + # A transport error mid-stream (e.g. the server dropped + # the connection: ``ReadError``/``RemoteProtocolError``) + # is a premature end. Only swallow it when reconnection + # is configured for this stream; otherwise re-raise so a + # non-resumable stream still surfaces the error to the + # caller instead of looking like a clean completion. + # When reconnection is applicable but the attempt budget + # is exhausted, we ``break`` and end cleanly below — the + # same way an exhausted empty/error-body resume does, so + # give-up is consistent regardless of failure shape. + if not self._reconnect_applicable(): + raise + break + yield sse + if sse.id: + last_dispatched_id = sse.id + if sse.retry is not None: + last_retry = sse.retry + reconnect_attempts = 0 + + if not self._should_reconnect(last_dispatched_id, reconnect_attempts): + return + reconnect_attempts += 1 + + await self._asleep_before_reconnect(last_retry) + + if owned_cm is not None: + await owned_cm.__aexit__(None, None, None) + owned_cm = None + + assert self._reconnect is not None # guaranteed by _should_reconnect + try: + cm: AsyncContextManager[httpx.Response] = self._reconnect(last_dispatched_id or "") + new_response = await cm.__aenter__() + except Exception: + response = None + continue + owned_cm = cm + if new_response is None or not self._is_reconnect_response_usable(new_response): + response = None + continue + + response = new_response + decoder.reset_in_progress_event() + text_decoder = self._new_text_decoder(new_response) + finally: + if owned_cm is not None: + # Shield the close so a cancellation delivered while reading a + # reconnected response still fully tears the connection down + # instead of leaking it until the client is closed. + with anyio.CancelScope(shield=True): + await owned_cm.__aexit__(None, None, None) + @contextmanager def connect_sse(client: httpx.Client, method: str, url: str, **kwargs: Any) -> Iterator[EventSource]: diff --git a/src/newscatcher_catchall/core/http_sse/_decoders.py b/src/newscatcher_catchall/core/http_sse/_decoders.py index 339b089..1f6b35e 100644 --- a/src/newscatcher_catchall/core/http_sse/_decoders.py +++ b/src/newscatcher_catchall/core/http_sse/_decoders.py @@ -12,6 +12,19 @@ def __init__(self) -> None: self._last_event_id = "" self._retry: Optional[int] = None + def reset_in_progress_event(self) -> None: + """Discard any partially-parsed (undispatched) event. + + Used when a stream ends mid-event before reconnecting: the buffered + ``event``/``data``/``retry`` fields of the never-dispatched event must + be dropped so they do not corrupt the first event of the reconnected + stream. Per the SSE spec the last event id is *not* reset here — it + persists across connections. + """ + self._event = "" + self._data = [] + self._retry = None + def decode(self, line: str) -> Optional[ServerSentEvent]: # See: https://html.spec.whatwg.org/multipage/server-sent-events.html#event-stream-interpretation # noqa: E501 diff --git a/src/newscatcher_catchall/core/request_options.py b/src/newscatcher_catchall/core/request_options.py index 1b38804..caa6f66 100644 --- a/src/newscatcher_catchall/core/request_options.py +++ b/src/newscatcher_catchall/core/request_options.py @@ -14,7 +14,9 @@ class RequestOptions(typing.TypedDict, total=False): This is used primarily as an optional final parameter for service functions. Attributes: - - timeout_in_seconds: int. The number of seconds to await an API call before timing out. + - timeout: float. The number of seconds to await an API call before timing out. + + - timeout_in_seconds: int. Deprecated alias for `timeout`; both are in seconds. Prefer `timeout`. - max_retries: int. The max number of retries to attempt if the API call fails. @@ -27,9 +29,12 @@ class RequestOptions(typing.TypedDict, total=False): - chunk_size: int. The size, in bytes, to process each chunk of data being streamed back within the response. This equates to leveraging `chunk_size` within `requests` or `httpx`, and is only leveraged for file downloads. """ + timeout: NotRequired[float] timeout_in_seconds: NotRequired[int] max_retries: NotRequired[int] additional_headers: NotRequired[typing.Dict[str, typing.Any]] additional_query_parameters: NotRequired[typing.Dict[str, typing.Any]] additional_body_parameters: NotRequired[typing.Dict[str, typing.Any]] chunk_size: NotRequired[int] + stream_reconnection_enabled: NotRequired[bool] + max_stream_reconnection_attempts: NotRequired[int] diff --git a/src/newscatcher_catchall/datasets/client.py b/src/newscatcher_catchall/datasets/client.py index a3858b3..43a0ca6 100644 --- a/src/newscatcher_catchall/datasets/client.py +++ b/src/newscatcher_catchall/datasets/client.py @@ -121,7 +121,9 @@ def create_dataset( request_options: typing.Optional[RequestOptions] = None, ) -> DatasetResponse: """ - Creates a new dataset from a list of existing entity IDs. + Creates a new dataset from a list of existing entity IDs. The optional `description` field here describes the dataset itself — it is separate from the entity-level `description` used for matching. + + Entities must be created before adding them to a dataset. Each entity requires a `name` plus at least one of: a `description` or a `domain`. Use [Create entity](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entity) or [Create entities batch](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entities-batch) to create entities first. If any of the provided entity IDs do not exist or do not belong to your organization, the request fails with `400`. All entity IDs must @@ -179,12 +181,12 @@ def create_dataset_from_csv( request_options: typing.Optional[RequestOptions] = None, ) -> CreateDatasetCsvResponse: """ - Creates a new dataset by uploading a CSV file. Each row in the CSV becomes an entity. The `name` and `domain`columns are required; all other columns are optional. + Creates a new dataset by uploading a CSV file. Each row in the CSV becomes an entity. Each row requires a `name` plus at least one of: a `description` or a `domain`; all other columns are optional. Note: `description` in the CSV is the entity's matching description — it is separate from the dataset-level `description` field in the form data. **CSV format:** ```csv name,description,domain,alternative_names,key_persons - NewsCatcher,"AI-powered news data provider",newscatcherapi.com,"NC;NewsCatcher API","Artem Bugara;Maksym Sugonyaka" + NewsCatcher,"NewsCatcher is a data-as-a-service company providing news intelligence APIs including the CatchAll Web Search API (2B+ web pages indexed) and News API (140,000+ sources, 100+ countries).",newscatcherapi.com,"NewsCatcher CatchAll;NewsCatcher API","Artem Bugara;Maksym Sugonyaka" OpenAI,"Artificial intelligence research company",openai.com,"Open AI","Sam Altman" ``` @@ -679,7 +681,9 @@ async def create_dataset( request_options: typing.Optional[RequestOptions] = None, ) -> DatasetResponse: """ - Creates a new dataset from a list of existing entity IDs. + Creates a new dataset from a list of existing entity IDs. The optional `description` field here describes the dataset itself — it is separate from the entity-level `description` used for matching. + + Entities must be created before adding them to a dataset. Each entity requires a `name` plus at least one of: a `description` or a `domain`. Use [Create entity](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entity) or [Create entities batch](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entities-batch) to create entities first. If any of the provided entity IDs do not exist or do not belong to your organization, the request fails with `400`. All entity IDs must @@ -745,12 +749,12 @@ async def create_dataset_from_csv( request_options: typing.Optional[RequestOptions] = None, ) -> CreateDatasetCsvResponse: """ - Creates a new dataset by uploading a CSV file. Each row in the CSV becomes an entity. The `name` and `domain`columns are required; all other columns are optional. + Creates a new dataset by uploading a CSV file. Each row in the CSV becomes an entity. Each row requires a `name` plus at least one of: a `description` or a `domain`; all other columns are optional. Note: `description` in the CSV is the entity's matching description — it is separate from the dataset-level `description` field in the form data. **CSV format:** ```csv name,description,domain,alternative_names,key_persons - NewsCatcher,"AI-powered news data provider",newscatcherapi.com,"NC;NewsCatcher API","Artem Bugara;Maksym Sugonyaka" + NewsCatcher,"NewsCatcher is a data-as-a-service company providing news intelligence APIs including the CatchAll Web Search API (2B+ web pages indexed) and News API (140,000+ sources, 100+ countries).",newscatcherapi.com,"NewsCatcher CatchAll;NewsCatcher API","Artem Bugara;Maksym Sugonyaka" OpenAI,"Artificial intelligence research company",openai.com,"Open AI","Sam Altman" ``` diff --git a/src/newscatcher_catchall/datasets/raw_client.py b/src/newscatcher_catchall/datasets/raw_client.py index a69ef87..27ed31d 100644 --- a/src/newscatcher_catchall/datasets/raw_client.py +++ b/src/newscatcher_catchall/datasets/raw_client.py @@ -142,7 +142,9 @@ def create_dataset( request_options: typing.Optional[RequestOptions] = None, ) -> HttpResponse[DatasetResponse]: """ - Creates a new dataset from a list of existing entity IDs. + Creates a new dataset from a list of existing entity IDs. The optional `description` field here describes the dataset itself — it is separate from the entity-level `description` used for matching. + + Entities must be created before adding them to a dataset. Each entity requires a `name` plus at least one of: a `description` or a `domain`. Use [Create entity](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entity) or [Create entities batch](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entities-batch) to create entities first. If any of the provided entity IDs do not exist or do not belong to your organization, the request fails with `400`. All entity IDs must @@ -245,12 +247,12 @@ def create_dataset_from_csv( request_options: typing.Optional[RequestOptions] = None, ) -> HttpResponse[CreateDatasetCsvResponse]: """ - Creates a new dataset by uploading a CSV file. Each row in the CSV becomes an entity. The `name` and `domain`columns are required; all other columns are optional. + Creates a new dataset by uploading a CSV file. Each row in the CSV becomes an entity. Each row requires a `name` plus at least one of: a `description` or a `domain`; all other columns are optional. Note: `description` in the CSV is the entity's matching description — it is separate from the dataset-level `description` field in the form data. **CSV format:** ```csv name,description,domain,alternative_names,key_persons - NewsCatcher,"AI-powered news data provider",newscatcherapi.com,"NC;NewsCatcher API","Artem Bugara;Maksym Sugonyaka" + NewsCatcher,"NewsCatcher is a data-as-a-service company providing news intelligence APIs including the CatchAll Web Search API (2B+ web pages indexed) and News API (140,000+ sources, 100+ countries).",newscatcherapi.com,"NewsCatcher CatchAll;NewsCatcher API","Artem Bugara;Maksym Sugonyaka" OpenAI,"Artificial intelligence research company",openai.com,"Open AI","Sam Altman" ``` @@ -1105,7 +1107,9 @@ async def create_dataset( request_options: typing.Optional[RequestOptions] = None, ) -> AsyncHttpResponse[DatasetResponse]: """ - Creates a new dataset from a list of existing entity IDs. + Creates a new dataset from a list of existing entity IDs. The optional `description` field here describes the dataset itself — it is separate from the entity-level `description` used for matching. + + Entities must be created before adding them to a dataset. Each entity requires a `name` plus at least one of: a `description` or a `domain`. Use [Create entity](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entity) or [Create entities batch](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entities-batch) to create entities first. If any of the provided entity IDs do not exist or do not belong to your organization, the request fails with `400`. All entity IDs must @@ -1208,12 +1212,12 @@ async def create_dataset_from_csv( request_options: typing.Optional[RequestOptions] = None, ) -> AsyncHttpResponse[CreateDatasetCsvResponse]: """ - Creates a new dataset by uploading a CSV file. Each row in the CSV becomes an entity. The `name` and `domain`columns are required; all other columns are optional. + Creates a new dataset by uploading a CSV file. Each row in the CSV becomes an entity. Each row requires a `name` plus at least one of: a `description` or a `domain`; all other columns are optional. Note: `description` in the CSV is the entity's matching description — it is separate from the dataset-level `description` field in the form data. **CSV format:** ```csv name,description,domain,alternative_names,key_persons - NewsCatcher,"AI-powered news data provider",newscatcherapi.com,"NC;NewsCatcher API","Artem Bugara;Maksym Sugonyaka" + NewsCatcher,"NewsCatcher is a data-as-a-service company providing news intelligence APIs including the CatchAll Web Search API (2B+ web pages indexed) and News API (140,000+ sources, 100+ countries).",newscatcherapi.com,"NewsCatcher CatchAll;NewsCatcher API","Artem Bugara;Maksym Sugonyaka" OpenAI,"Artificial intelligence research company",openai.com,"Open AI","Sam Altman" ``` diff --git a/src/newscatcher_catchall/entities/client.py b/src/newscatcher_catchall/entities/client.py index 7d1f12d..dedf3d2 100644 --- a/src/newscatcher_catchall/entities/client.py +++ b/src/newscatcher_catchall/entities/client.py @@ -113,10 +113,10 @@ def create_entity( """ Creates a new company entity and begins background enrichment. + Each entity requires a `name` plus at least one of: a `description` or a `domain`. Providing both is recommended — `domain` is the highest-signal identifier because it is unambiguous; a well-written `description` is the best alternative when no domain is available. + The entity status starts as `pending` and transitions to `ready` once - enrichment completes. Provide as much identifying information as - possible — `domain` is the highest-signal field because it is - unambiguous. + enrichment completes. Parameters ---------- @@ -126,7 +126,7 @@ def create_entity( entity_type : typing.Optional[EntityType] description : typing.Optional[str] - Free-text description of the entity used for disambiguation when similar names exist. + Free-text description of the entity used for disambiguation when similar names exist. See [Writing effective descriptions](https://www.newscatcherapi.com/docs/web-search-api/concepts/company-search#writing-effective-descriptions) for guidance on improving matching quality. external_entity_id : typing.Optional[str] Optional external identifier for this entity. Free-form string, not enforced as unique. Use it to store your own CRM, data warehouse, or internal database ID so you can join CatchAll results back to your systems. @@ -155,12 +155,12 @@ def create_entity( client.entities.create_entity( name="NewsCatcher", entity_type="company", - description="AI-powered news data provider", + description="NewsCatcher is a data-as-a-service company providing news intelligence APIs including the CatchAll Web Search API (2B+ web pages indexed) and News API (140,000+ sources, 100+ countries).", additional_attributes=AdditionalAttributes( company_attributes=CompanyAttributes( domain="newscatcherapi.com", key_persons=["Artem Bugara", "Maksym Sugonyaka"], - alternative_names=["NC", "NewsCatcher API"], + alternative_names=["NewsCatcher CatchAll", "NewsCatcher API"], ), ), ) @@ -181,6 +181,8 @@ def create_entities_batch( """ Creates multiple entities in a single request. Each entity is processed independently — a failure in one does not affect others. + Each entity requires a `name` plus at least one of: a `description` or a `domain`. See [Create entity](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entity) for the full field reference. + Returns an array of `{id, status}` objects in the same order as the input array. Parameters @@ -355,7 +357,11 @@ def update_entity( description="Updated description", additional_attributes=AdditionalAttributes( company_attributes=CompanyAttributes( - alternative_names=["NC", "NewsCatcher API", "NCA"], + alternative_names=[ + "NewsCatcher CatchAll", + "NewsCatcher API", + "NCA", + ], ), ), ) @@ -472,10 +478,10 @@ async def create_entity( """ Creates a new company entity and begins background enrichment. + Each entity requires a `name` plus at least one of: a `description` or a `domain`. Providing both is recommended — `domain` is the highest-signal identifier because it is unambiguous; a well-written `description` is the best alternative when no domain is available. + The entity status starts as `pending` and transitions to `ready` once - enrichment completes. Provide as much identifying information as - possible — `domain` is the highest-signal field because it is - unambiguous. + enrichment completes. Parameters ---------- @@ -485,7 +491,7 @@ async def create_entity( entity_type : typing.Optional[EntityType] description : typing.Optional[str] - Free-text description of the entity used for disambiguation when similar names exist. + Free-text description of the entity used for disambiguation when similar names exist. See [Writing effective descriptions](https://www.newscatcherapi.com/docs/web-search-api/concepts/company-search#writing-effective-descriptions) for guidance on improving matching quality. external_entity_id : typing.Optional[str] Optional external identifier for this entity. Free-form string, not enforced as unique. Use it to store your own CRM, data warehouse, or internal database ID so you can join CatchAll results back to your systems. @@ -519,12 +525,12 @@ async def main() -> None: await client.entities.create_entity( name="NewsCatcher", entity_type="company", - description="AI-powered news data provider", + description="NewsCatcher is a data-as-a-service company providing news intelligence APIs including the CatchAll Web Search API (2B+ web pages indexed) and News API (140,000+ sources, 100+ countries).", additional_attributes=AdditionalAttributes( company_attributes=CompanyAttributes( domain="newscatcherapi.com", key_persons=["Artem Bugara", "Maksym Sugonyaka"], - alternative_names=["NC", "NewsCatcher API"], + alternative_names=["NewsCatcher CatchAll", "NewsCatcher API"], ), ), ) @@ -548,6 +554,8 @@ async def create_entities_batch( """ Creates multiple entities in a single request. Each entity is processed independently — a failure in one does not affect others. + Each entity requires a `name` plus at least one of: a `description` or a `domain`. See [Create entity](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entity) for the full field reference. + Returns an array of `{id, status}` objects in the same order as the input array. Parameters @@ -753,7 +761,11 @@ async def main() -> None: description="Updated description", additional_attributes=AdditionalAttributes( company_attributes=CompanyAttributes( - alternative_names=["NC", "NewsCatcher API", "NCA"], + alternative_names=[ + "NewsCatcher CatchAll", + "NewsCatcher API", + "NCA", + ], ), ), ) diff --git a/src/newscatcher_catchall/entities/raw_client.py b/src/newscatcher_catchall/entities/raw_client.py index 2607f89..2188aac 100644 --- a/src/newscatcher_catchall/entities/raw_client.py +++ b/src/newscatcher_catchall/entities/raw_client.py @@ -134,10 +134,10 @@ def create_entity( """ Creates a new company entity and begins background enrichment. + Each entity requires a `name` plus at least one of: a `description` or a `domain`. Providing both is recommended — `domain` is the highest-signal identifier because it is unambiguous; a well-written `description` is the best alternative when no domain is available. + The entity status starts as `pending` and transitions to `ready` once - enrichment completes. Provide as much identifying information as - possible — `domain` is the highest-signal field because it is - unambiguous. + enrichment completes. Parameters ---------- @@ -147,7 +147,7 @@ def create_entity( entity_type : typing.Optional[EntityType] description : typing.Optional[str] - Free-text description of the entity used for disambiguation when similar names exist. + Free-text description of the entity used for disambiguation when similar names exist. See [Writing effective descriptions](https://www.newscatcherapi.com/docs/web-search-api/concepts/company-search#writing-effective-descriptions) for guidance on improving matching quality. external_entity_id : typing.Optional[str] Optional external identifier for this entity. Free-form string, not enforced as unique. Use it to store your own CRM, data warehouse, or internal database ID so you can join CatchAll results back to your systems. @@ -224,6 +224,8 @@ def create_entities_batch( """ Creates multiple entities in a single request. Each entity is processed independently — a failure in one does not affect others. + Each entity requires a `name` plus at least one of: a `description` or a `domain`. See [Create entity](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entity) for the full field reference. + Returns an array of `{id, status}` objects in the same order as the input array. Parameters @@ -628,10 +630,10 @@ async def create_entity( """ Creates a new company entity and begins background enrichment. + Each entity requires a `name` plus at least one of: a `description` or a `domain`. Providing both is recommended — `domain` is the highest-signal identifier because it is unambiguous; a well-written `description` is the best alternative when no domain is available. + The entity status starts as `pending` and transitions to `ready` once - enrichment completes. Provide as much identifying information as - possible — `domain` is the highest-signal field because it is - unambiguous. + enrichment completes. Parameters ---------- @@ -641,7 +643,7 @@ async def create_entity( entity_type : typing.Optional[EntityType] description : typing.Optional[str] - Free-text description of the entity used for disambiguation when similar names exist. + Free-text description of the entity used for disambiguation when similar names exist. See [Writing effective descriptions](https://www.newscatcherapi.com/docs/web-search-api/concepts/company-search#writing-effective-descriptions) for guidance on improving matching quality. external_entity_id : typing.Optional[str] Optional external identifier for this entity. Free-form string, not enforced as unique. Use it to store your own CRM, data warehouse, or internal database ID so you can join CatchAll results back to your systems. @@ -718,6 +720,8 @@ async def create_entities_batch( """ Creates multiple entities in a single request. Each entity is processed independently — a failure in one does not affect others. + Each entity requires a `name` plus at least one of: a `description` or a `domain`. See [Create entity](https://www.newscatcherapi.com/docs/web-search-api/api-reference/entities/create-entity) for the full field reference. + Returns an array of `{id, status}` objects in the same order as the input array. Parameters diff --git a/src/newscatcher_catchall/jobs/client.py b/src/newscatcher_catchall/jobs/client.py index e277aa9..ae4033b 100644 --- a/src/newscatcher_catchall/jobs/client.py +++ b/src/newscatcher_catchall/jobs/client.py @@ -228,6 +228,10 @@ def create_job( Custom validators for filtering web page clusters. If not provided, validators are generated automatically based on the query. + The system may also inject validators during the `analyzing` stage — for + example, converting a relative time qualifier into an explicit event-date + gate. The returned `validators[]` in the job status shows the complete + applied set, including any system-added ones. enrichments : typing.Optional[typing.Sequence[EnrichmentSchema]] Custom enrichment fields for data extraction. @@ -728,6 +732,10 @@ async def create_job( Custom validators for filtering web page clusters. If not provided, validators are generated automatically based on the query. + The system may also inject validators during the `analyzing` stage — for + example, converting a relative time qualifier into an explicit event-date + gate. The returned `validators[]` in the job status shows the complete + applied set, including any system-added ones. enrichments : typing.Optional[typing.Sequence[EnrichmentSchema]] Custom enrichment fields for data extraction. diff --git a/src/newscatcher_catchall/jobs/raw_client.py b/src/newscatcher_catchall/jobs/raw_client.py index 86a8d58..388d1ba 100644 --- a/src/newscatcher_catchall/jobs/raw_client.py +++ b/src/newscatcher_catchall/jobs/raw_client.py @@ -326,6 +326,10 @@ def create_job( Custom validators for filtering web page clusters. If not provided, validators are generated automatically based on the query. + The system may also inject validators during the `analyzing` stage — for + example, converting a relative time qualifier into an explicit event-date + gate. The returned `validators[]` in the job status shows the complete + applied set, including any system-added ones. enrichments : typing.Optional[typing.Sequence[EnrichmentSchema]] Custom enrichment fields for data extraction. @@ -1101,6 +1105,10 @@ async def create_job( Custom validators for filtering web page clusters. If not provided, validators are generated automatically based on the query. + The system may also inject validators during the `analyzing` stage — for + example, converting a relative time qualifier into an explicit event-date + gate. The returned `validators[]` in the job status shows the complete + applied set, including any system-added ones. enrichments : typing.Optional[typing.Sequence[EnrichmentSchema]] Custom enrichment fields for data extraction. diff --git a/src/newscatcher_catchall/types/__init__.py b/src/newscatcher_catchall/types/__init__.py index 76e0f9f..683e13e 100644 --- a/src/newscatcher_catchall/types/__init__.py +++ b/src/newscatcher_catchall/types/__init__.py @@ -59,6 +59,7 @@ from .entity_type import EntityType from .entity_validation_error_body import EntityValidationErrorBody from .error import Error + from .formatter_config_dto import FormatterConfigDto from .get_plan_limits_response_dto import GetPlanLimitsResponseDto from .get_webhook_response_dto import GetWebhookResponseDto from .http_method import HttpMethod @@ -182,6 +183,7 @@ "EntityType": ".entity_type", "EntityValidationErrorBody": ".entity_validation_error_body", "Error": ".error", + "FormatterConfigDto": ".formatter_config_dto", "GetPlanLimitsResponseDto": ".get_plan_limits_response_dto", "GetWebhookResponseDto": ".get_webhook_response_dto", "HttpMethod": ".http_method", @@ -329,6 +331,7 @@ def __dir__(): "EntityType", "EntityValidationErrorBody", "Error", + "FormatterConfigDto", "GetPlanLimitsResponseDto", "GetWebhookResponseDto", "HttpMethod", diff --git a/src/newscatcher_catchall/types/company_attributes.py b/src/newscatcher_catchall/types/company_attributes.py index 611aa00..591ab43 100644 --- a/src/newscatcher_catchall/types/company_attributes.py +++ b/src/newscatcher_catchall/types/company_attributes.py @@ -18,11 +18,6 @@ class CompanyAttributes(UniversalBaseModel): The most reliable identifier — strongly recommended when available. """ - description: typing.Optional[str] = pydantic.Field(default=None) - """ - Detailed description of the company used for matching. - """ - key_persons: typing.Optional[typing.List[str]] = pydantic.Field(default=None) """ Names of key people associated with the company (founders, executives, etc.). Improves matching for articles that mention people rather than the company name. diff --git a/src/newscatcher_catchall/types/create_entity_request.py b/src/newscatcher_catchall/types/create_entity_request.py index 3539791..b5f92fd 100644 --- a/src/newscatcher_catchall/types/create_entity_request.py +++ b/src/newscatcher_catchall/types/create_entity_request.py @@ -10,7 +10,9 @@ class CreateEntityRequest(UniversalBaseModel): """ - Request body for creating a single entity. Only `name` is required. + Request body for creating a single entity. The `name` field is required. + You must also provide at least one of: a top-level `description` or a `domain` + inside `additional_attributes.company_attributes`. The more fields you provide, the better the matching quality. """ @@ -22,7 +24,7 @@ class CreateEntityRequest(UniversalBaseModel): entity_type: typing.Optional[EntityType] = None description: typing.Optional[str] = pydantic.Field(default=None) """ - Free-text description of the entity used for disambiguation when similar names exist. + Free-text description of the entity used for disambiguation when similar names exist. See [Writing effective descriptions](https://www.newscatcherapi.com/docs/web-search-api/concepts/company-search#writing-effective-descriptions) for guidance on improving matching quality. """ external_entity_id: typing.Optional[str] = pydantic.Field(default=None) diff --git a/src/newscatcher_catchall/types/dataset_response.py b/src/newscatcher_catchall/types/dataset_response.py index 7dce5c6..5dd5e46 100644 --- a/src/newscatcher_catchall/types/dataset_response.py +++ b/src/newscatcher_catchall/types/dataset_response.py @@ -34,6 +34,24 @@ class DatasetResponse(UniversalBaseModel): Total number of entities in this dataset. """ + entity_status_breakdown: typing.Optional[typing.Dict[str, int]] = pydantic.Field(default=None) + """ + Count of entities grouped by processing status. Keys are status values + (`ready`, `pending`, `enriching`, `failed`); values are entity counts. + """ + + health_score: typing.Optional[float] = pydantic.Field(default=None) + """ + Overall health score of the dataset, from 0 to 100. Reflects how many + entities have sufficient identifying information for reliable matching. + """ + + health_breakdown: typing.Optional[typing.Dict[str, float]] = pydantic.Field(default=None) + """ + Health scores broken down by entity type. Keys are entity types + (e.g. `company`); values are scores from 0 to 100. + """ + latest_status: typing.Optional[DatasetStatus] = None created_by_user_id: typing.Optional[str] = pydantic.Field(default=None) """ diff --git a/src/newscatcher_catchall/types/formatter_config_dto.py b/src/newscatcher_catchall/types/formatter_config_dto.py new file mode 100644 index 0000000..b6014eb --- /dev/null +++ b/src/newscatcher_catchall/types/formatter_config_dto.py @@ -0,0 +1,35 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel + + +class FormatterConfigDto(UniversalBaseModel): + """ + Custom payload formatter for `custom` webhooks. A Liquid template is + rendered at dispatch time to build a fully tailored request body — trim or + rename fields, apply conditional logic, or emit non-JSON formats. + """ + + template: str = pydantic.Field() + """ + Liquid template string rendered at dispatch time. Maximum 10 KB. + """ + + content_type: typing.Optional[str] = pydantic.Field(default=None) + """ + Content-Type of the rendered output. Supported values: `application/json`, + `application/ld+json`, `text/html`, `text/plain`, `text/xml`, + `application/xml`, `text/csv`. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/src/newscatcher_catchall/types/initialize_response_dto.py b/src/newscatcher_catchall/types/initialize_response_dto.py index caac2ea..06b19a4 100644 --- a/src/newscatcher_catchall/types/initialize_response_dto.py +++ b/src/newscatcher_catchall/types/initialize_response_dto.py @@ -23,7 +23,10 @@ class InitializeResponseDto(UniversalBaseModel): validators: typing.List[ValidatorSchema] = pydantic.Field() """ - Suggested validators for filtering relevant web pages. + Suggested validators for filtering relevant web pages. When the job is + submitted, the system may inject additional validators during the `analyzing` + stage — check the returned `validators[]` in the job status to see the + complete applied set. """ enrichments: typing.List[EnrichmentSchema] = pydantic.Field() diff --git a/src/newscatcher_catchall/types/webhook_response_dto.py b/src/newscatcher_catchall/types/webhook_response_dto.py index f71e80f..6ba183f 100644 --- a/src/newscatcher_catchall/types/webhook_response_dto.py +++ b/src/newscatcher_catchall/types/webhook_response_dto.py @@ -6,6 +6,7 @@ import pydantic from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel from .delivery_mode import DeliveryMode +from .formatter_config_dto import FormatterConfigDto from .http_method import HttpMethod from .webhook_type import WebhookType @@ -39,9 +40,9 @@ class WebhookResponseDto(UniversalBaseModel): Query parameters appended to the webhook URL. """ - formatter_config: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field(default=None) + formatter_config: typing.Optional[FormatterConfigDto] = pydantic.Field(default=None) """ - Custom payload transformation configuration. Used only when `type` is `custom`. + Custom payload formatter. Set only when `type` is `custom`. """ is_active: bool = pydantic.Field() diff --git a/src/newscatcher_catchall/webhooks/client.py b/src/newscatcher_catchall/webhooks/client.py index ec01532..2fc4a7d 100644 --- a/src/newscatcher_catchall/webhooks/client.py +++ b/src/newscatcher_catchall/webhooks/client.py @@ -8,6 +8,7 @@ from ..types.create_webhook_response_dto import CreateWebhookResponseDto from ..types.delivery_history_response_dto import DeliveryHistoryResponseDto from ..types.delivery_mode import DeliveryMode +from ..types.formatter_config_dto import FormatterConfigDto from ..types.get_webhook_response_dto import GetWebhookResponseDto from ..types.http_method import HttpMethod from ..types.list_webhook_resources_response_dto import ListWebhookResourcesResponseDto @@ -94,7 +95,7 @@ def create_webhook( headers: typing.Optional[typing.Dict[str, str]] = OMIT, params: typing.Optional[typing.Dict[str, str]] = OMIT, auth: typing.Optional[CreateWebhookRequestDtoAuth] = OMIT, - formatter_config: typing.Optional[typing.Dict[str, typing.Any]] = OMIT, + formatter_config: typing.Optional[FormatterConfigDto] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> CreateWebhookResponseDto: """ @@ -134,8 +135,8 @@ def create_webhook( - `api_key`: Adds a custom header with the specified name and value. - `basic`: Adds an `Authorization: Basic ` header. - formatter_config : typing.Optional[typing.Dict[str, typing.Any]] - Custom payload transformation configuration. Required only when `type` is `custom`. + formatter_config : typing.Optional[FormatterConfigDto] + Custom payload formatter. Required when `type` is `custom`. request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -250,7 +251,7 @@ def update_webhook( headers: typing.Optional[typing.Dict[str, str]] = OMIT, params: typing.Optional[typing.Dict[str, str]] = OMIT, auth: typing.Optional[UpdateWebhookRequestDtoAuth] = OMIT, - formatter_config: typing.Optional[typing.Dict[str, typing.Any]] = OMIT, + formatter_config: typing.Optional[FormatterConfigDto] = OMIT, is_active: typing.Optional[bool] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> UpdateWebhookResponseDto: @@ -283,8 +284,8 @@ def update_webhook( auth : typing.Optional[UpdateWebhookRequestDtoAuth] Updated authentication configuration. Replaces existing auth entirely. - formatter_config : typing.Optional[typing.Dict[str, typing.Any]] - Updated formatter configuration. + formatter_config : typing.Optional[FormatterConfigDto] + Updated custom payload formatter. Set only when `type` is `custom`. is_active : typing.Optional[bool] Set to `false` to disable delivery without deleting the webhook. @@ -578,6 +579,64 @@ def list_webhooks_for_resource( ) return _response.data + def trigger_webhook( + self, + resource_type: MappableResourceType, + resource_id: str, + *, + webhook_id: str, + job_id: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> None: + """ + Manually dispatches a webhook delivery for a resource on demand, without + waiting for the next job or monitor cycle. + + Use this to re-deliver results after a failed delivery, replay a specific + job's results, or validate a webhook against live data. The webhook must + already be assigned to the resource. + + Parameters + ---------- + resource_type : MappableResourceType + + resource_id : str + Unique resource identifier. + + webhook_id : str + Identifier of the webhook to trigger. Must be assigned to the resource. + + job_id : typing.Optional[str] + Specific job run to deliver. Optional for `job` resources; for + `monitor` and `monitor_group` resources, selects a past run to replay. + When omitted, the latest available results are delivered. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + None + + Examples + -------- + from newscatcher_catchall import CatchAllApi + + client = CatchAllApi( + api_key="YOUR_API_KEY", + ) + client.webhooks.trigger_webhook( + resource_type="job", + resource_id="3fec5b07-8786-46d7-9486-d43ff67eccd4", + webhook_id="a1b2c3d4-e5f6-7890-abcd-ef1234567890", + job_id="3fec5b07-8786-46d7-9486-d43ff67eccd4", + ) + """ + _response = self._raw_client.trigger_webhook( + resource_type, resource_id, webhook_id=webhook_id, job_id=job_id, request_options=request_options + ) + return _response.data + def get_webhook_delivery_history( self, *, @@ -714,7 +773,7 @@ async def create_webhook( headers: typing.Optional[typing.Dict[str, str]] = OMIT, params: typing.Optional[typing.Dict[str, str]] = OMIT, auth: typing.Optional[CreateWebhookRequestDtoAuth] = OMIT, - formatter_config: typing.Optional[typing.Dict[str, typing.Any]] = OMIT, + formatter_config: typing.Optional[FormatterConfigDto] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> CreateWebhookResponseDto: """ @@ -754,8 +813,8 @@ async def create_webhook( - `api_key`: Adds a custom header with the specified name and value. - `basic`: Adds an `Authorization: Basic ` header. - formatter_config : typing.Optional[typing.Dict[str, typing.Any]] - Custom payload transformation configuration. Required only when `type` is `custom`. + formatter_config : typing.Optional[FormatterConfigDto] + Custom payload formatter. Required when `type` is `custom`. request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -894,7 +953,7 @@ async def update_webhook( headers: typing.Optional[typing.Dict[str, str]] = OMIT, params: typing.Optional[typing.Dict[str, str]] = OMIT, auth: typing.Optional[UpdateWebhookRequestDtoAuth] = OMIT, - formatter_config: typing.Optional[typing.Dict[str, typing.Any]] = OMIT, + formatter_config: typing.Optional[FormatterConfigDto] = OMIT, is_active: typing.Optional[bool] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> UpdateWebhookResponseDto: @@ -927,8 +986,8 @@ async def update_webhook( auth : typing.Optional[UpdateWebhookRequestDtoAuth] Updated authentication configuration. Replaces existing auth entirely. - formatter_config : typing.Optional[typing.Dict[str, typing.Any]] - Updated formatter configuration. + formatter_config : typing.Optional[FormatterConfigDto] + Updated custom payload formatter. Set only when `type` is `custom`. is_active : typing.Optional[bool] Set to `false` to disable delivery without deleting the webhook. @@ -1270,6 +1329,72 @@ async def main() -> None: ) return _response.data + async def trigger_webhook( + self, + resource_type: MappableResourceType, + resource_id: str, + *, + webhook_id: str, + job_id: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> None: + """ + Manually dispatches a webhook delivery for a resource on demand, without + waiting for the next job or monitor cycle. + + Use this to re-deliver results after a failed delivery, replay a specific + job's results, or validate a webhook against live data. The webhook must + already be assigned to the resource. + + Parameters + ---------- + resource_type : MappableResourceType + + resource_id : str + Unique resource identifier. + + webhook_id : str + Identifier of the webhook to trigger. Must be assigned to the resource. + + job_id : typing.Optional[str] + Specific job run to deliver. Optional for `job` resources; for + `monitor` and `monitor_group` resources, selects a past run to replay. + When omitted, the latest available results are delivered. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + None + + Examples + -------- + import asyncio + + from newscatcher_catchall import AsyncCatchAllApi + + client = AsyncCatchAllApi( + api_key="YOUR_API_KEY", + ) + + + async def main() -> None: + await client.webhooks.trigger_webhook( + resource_type="job", + resource_id="3fec5b07-8786-46d7-9486-d43ff67eccd4", + webhook_id="a1b2c3d4-e5f6-7890-abcd-ef1234567890", + job_id="3fec5b07-8786-46d7-9486-d43ff67eccd4", + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.trigger_webhook( + resource_type, resource_id, webhook_id=webhook_id, job_id=job_id, request_options=request_options + ) + return _response.data + async def get_webhook_delivery_history( self, *, diff --git a/src/newscatcher_catchall/webhooks/raw_client.py b/src/newscatcher_catchall/webhooks/raw_client.py index 8feaaa6..0b813ff 100644 --- a/src/newscatcher_catchall/webhooks/raw_client.py +++ b/src/newscatcher_catchall/webhooks/raw_client.py @@ -20,6 +20,7 @@ from ..types.delivery_history_response_dto import DeliveryHistoryResponseDto from ..types.delivery_mode import DeliveryMode from ..types.error import Error +from ..types.formatter_config_dto import FormatterConfigDto from ..types.get_webhook_response_dto import GetWebhookResponseDto from ..types.http_method import HttpMethod from ..types.list_webhook_resources_response_dto import ListWebhookResourcesResponseDto @@ -133,7 +134,7 @@ def create_webhook( headers: typing.Optional[typing.Dict[str, str]] = OMIT, params: typing.Optional[typing.Dict[str, str]] = OMIT, auth: typing.Optional[CreateWebhookRequestDtoAuth] = OMIT, - formatter_config: typing.Optional[typing.Dict[str, typing.Any]] = OMIT, + formatter_config: typing.Optional[FormatterConfigDto] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> HttpResponse[CreateWebhookResponseDto]: """ @@ -173,8 +174,8 @@ def create_webhook( - `api_key`: Adds a custom header with the specified name and value. - `basic`: Adds an `Authorization: Basic ` header. - formatter_config : typing.Optional[typing.Dict[str, typing.Any]] - Custom payload transformation configuration. Required only when `type` is `custom`. + formatter_config : typing.Optional[FormatterConfigDto] + Custom payload formatter. Required when `type` is `custom`. request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -198,7 +199,9 @@ def create_webhook( "auth": convert_and_respect_annotation_metadata( object_=auth, annotation=CreateWebhookRequestDtoAuth, direction="write" ), - "formatter_config": formatter_config, + "formatter_config": convert_and_respect_annotation_metadata( + object_=formatter_config, annotation=typing.Optional[FormatterConfigDto], direction="write" + ), }, headers={ "content-type": "application/json", @@ -394,7 +397,7 @@ def update_webhook( headers: typing.Optional[typing.Dict[str, str]] = OMIT, params: typing.Optional[typing.Dict[str, str]] = OMIT, auth: typing.Optional[UpdateWebhookRequestDtoAuth] = OMIT, - formatter_config: typing.Optional[typing.Dict[str, typing.Any]] = OMIT, + formatter_config: typing.Optional[FormatterConfigDto] = OMIT, is_active: typing.Optional[bool] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> HttpResponse[UpdateWebhookResponseDto]: @@ -427,8 +430,8 @@ def update_webhook( auth : typing.Optional[UpdateWebhookRequestDtoAuth] Updated authentication configuration. Replaces existing auth entirely. - formatter_config : typing.Optional[typing.Dict[str, typing.Any]] - Updated formatter configuration. + formatter_config : typing.Optional[FormatterConfigDto] + Updated custom payload formatter. Set only when `type` is `custom`. is_active : typing.Optional[bool] Set to `false` to disable delivery without deleting the webhook. @@ -455,7 +458,9 @@ def update_webhook( "auth": convert_and_respect_annotation_metadata( object_=auth, annotation=UpdateWebhookRequestDtoAuth, direction="write" ), - "formatter_config": formatter_config, + "formatter_config": convert_and_respect_annotation_metadata( + object_=formatter_config, annotation=typing.Optional[FormatterConfigDto], direction="write" + ), "is_active": is_active, }, headers={ @@ -982,6 +987,99 @@ def list_webhooks_for_resource( ) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + def trigger_webhook( + self, + resource_type: MappableResourceType, + resource_id: str, + *, + webhook_id: str, + job_id: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[None]: + """ + Manually dispatches a webhook delivery for a resource on demand, without + waiting for the next job or monitor cycle. + + Use this to re-deliver results after a failed delivery, replay a specific + job's results, or validate a webhook against live data. The webhook must + already be assigned to the resource. + + Parameters + ---------- + resource_type : MappableResourceType + + resource_id : str + Unique resource identifier. + + webhook_id : str + Identifier of the webhook to trigger. Must be assigned to the resource. + + job_id : typing.Optional[str] + Specific job run to deliver. Optional for `job` resources; for + `monitor` and `monitor_group` resources, selects a past run to replay. + When omitted, the latest available results are delivered. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[None] + """ + _response = self._client_wrapper.httpx_client.request( + f"catchAll/webhook/trigger/{encode_path_param(resource_type)}/{encode_path_param(resource_id)}", + method="POST", + params={ + "webhook_id": webhook_id, + "job_id": job_id, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return HttpResponse(response=_response, data=None) + if _response.status_code == 403: + raise ForbiddenError( + headers=dict(_response.headers), + body=typing.cast( + Error, + parse_obj_as( + type_=Error, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + Error, + parse_obj_as( + type_=Error, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 422: + raise UnprocessableEntityError( + headers=dict(_response.headers), + body=typing.cast( + ValidationErrorResponse, + parse_obj_as( + type_=ValidationErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + except ValidationError as e: + raise ParsingError( + status_code=_response.status_code, headers=dict(_response.headers), body=_response.json(), cause=e + ) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + def get_webhook_delivery_history( self, *, @@ -1178,7 +1276,7 @@ async def create_webhook( headers: typing.Optional[typing.Dict[str, str]] = OMIT, params: typing.Optional[typing.Dict[str, str]] = OMIT, auth: typing.Optional[CreateWebhookRequestDtoAuth] = OMIT, - formatter_config: typing.Optional[typing.Dict[str, typing.Any]] = OMIT, + formatter_config: typing.Optional[FormatterConfigDto] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> AsyncHttpResponse[CreateWebhookResponseDto]: """ @@ -1218,8 +1316,8 @@ async def create_webhook( - `api_key`: Adds a custom header with the specified name and value. - `basic`: Adds an `Authorization: Basic ` header. - formatter_config : typing.Optional[typing.Dict[str, typing.Any]] - Custom payload transformation configuration. Required only when `type` is `custom`. + formatter_config : typing.Optional[FormatterConfigDto] + Custom payload formatter. Required when `type` is `custom`. request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -1243,7 +1341,9 @@ async def create_webhook( "auth": convert_and_respect_annotation_metadata( object_=auth, annotation=CreateWebhookRequestDtoAuth, direction="write" ), - "formatter_config": formatter_config, + "formatter_config": convert_and_respect_annotation_metadata( + object_=formatter_config, annotation=typing.Optional[FormatterConfigDto], direction="write" + ), }, headers={ "content-type": "application/json", @@ -1439,7 +1539,7 @@ async def update_webhook( headers: typing.Optional[typing.Dict[str, str]] = OMIT, params: typing.Optional[typing.Dict[str, str]] = OMIT, auth: typing.Optional[UpdateWebhookRequestDtoAuth] = OMIT, - formatter_config: typing.Optional[typing.Dict[str, typing.Any]] = OMIT, + formatter_config: typing.Optional[FormatterConfigDto] = OMIT, is_active: typing.Optional[bool] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> AsyncHttpResponse[UpdateWebhookResponseDto]: @@ -1472,8 +1572,8 @@ async def update_webhook( auth : typing.Optional[UpdateWebhookRequestDtoAuth] Updated authentication configuration. Replaces existing auth entirely. - formatter_config : typing.Optional[typing.Dict[str, typing.Any]] - Updated formatter configuration. + formatter_config : typing.Optional[FormatterConfigDto] + Updated custom payload formatter. Set only when `type` is `custom`. is_active : typing.Optional[bool] Set to `false` to disable delivery without deleting the webhook. @@ -1500,7 +1600,9 @@ async def update_webhook( "auth": convert_and_respect_annotation_metadata( object_=auth, annotation=UpdateWebhookRequestDtoAuth, direction="write" ), - "formatter_config": formatter_config, + "formatter_config": convert_and_respect_annotation_metadata( + object_=formatter_config, annotation=typing.Optional[FormatterConfigDto], direction="write" + ), "is_active": is_active, }, headers={ @@ -2027,6 +2129,99 @@ async def list_webhooks_for_resource( ) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + async def trigger_webhook( + self, + resource_type: MappableResourceType, + resource_id: str, + *, + webhook_id: str, + job_id: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[None]: + """ + Manually dispatches a webhook delivery for a resource on demand, without + waiting for the next job or monitor cycle. + + Use this to re-deliver results after a failed delivery, replay a specific + job's results, or validate a webhook against live data. The webhook must + already be assigned to the resource. + + Parameters + ---------- + resource_type : MappableResourceType + + resource_id : str + Unique resource identifier. + + webhook_id : str + Identifier of the webhook to trigger. Must be assigned to the resource. + + job_id : typing.Optional[str] + Specific job run to deliver. Optional for `job` resources; for + `monitor` and `monitor_group` resources, selects a past run to replay. + When omitted, the latest available results are delivered. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[None] + """ + _response = await self._client_wrapper.httpx_client.request( + f"catchAll/webhook/trigger/{encode_path_param(resource_type)}/{encode_path_param(resource_id)}", + method="POST", + params={ + "webhook_id": webhook_id, + "job_id": job_id, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return AsyncHttpResponse(response=_response, data=None) + if _response.status_code == 403: + raise ForbiddenError( + headers=dict(_response.headers), + body=typing.cast( + Error, + parse_obj_as( + type_=Error, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + Error, + parse_obj_as( + type_=Error, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 422: + raise UnprocessableEntityError( + headers=dict(_response.headers), + body=typing.cast( + ValidationErrorResponse, + parse_obj_as( + type_=ValidationErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + except ValidationError as e: + raise ParsingError( + status_code=_response.status_code, headers=dict(_response.headers), body=_response.json(), cause=e + ) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + async def get_webhook_delivery_history( self, *, diff --git a/tests/utils/test_http_client.py b/tests/utils/test_http_client.py index 0c538e2..7634cc0 100644 --- a/tests/utils/test_http_client.py +++ b/tests/utils/test_http_client.py @@ -1,6 +1,6 @@ # This file was auto-generated by Fern from our API Definition. -from typing import Any, Dict +from typing import Any, Dict, Tuple, cast from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -692,3 +692,70 @@ def test_should_not_retry_non_retryable_status_codes(status_code: int) -> None: def test_should_retry_599_upper_boundary() -> None: """Legacy mode retries on >= 500, which includes 599.""" assert _should_retry(_make_response(599)) is True + + +# --------------------------------------------------------------------------- +# RequestOptions timeout resolution tests (timeout / deprecated timeout_in_seconds) +# --------------------------------------------------------------------------- + + +def _sync_client_with_base_timeout(base_timeout: Any) -> Tuple[HttpClient, _DummySyncClient]: + dummy_client = _DummySyncClient() + http_client = HttpClient( + httpx_client=dummy_client, # type: ignore[arg-type] + base_timeout=lambda: base_timeout, + base_headers=lambda: {}, + base_url=lambda: "https://example.com", + ) + return http_client, dummy_client + + +def test_sync_request_options_timeout_used() -> None: + """The new `timeout` request option is passed through to httpx (in seconds).""" + http_client, dummy_client = _sync_client_with_base_timeout(60) + http_client.request(path="/test", method="GET", request_options={"timeout": 30}) + assert dummy_client.last_request_kwargs["timeout"] == 30 + + +def test_sync_request_options_timeout_in_seconds_still_works() -> None: + """The deprecated `timeout_in_seconds` request option remains backwards compatible.""" + http_client, dummy_client = _sync_client_with_base_timeout(60) + http_client.request(path="/test", method="GET", request_options={"timeout_in_seconds": 45}) + assert dummy_client.last_request_kwargs["timeout"] == 45 + + +def test_sync_request_options_timeout_takes_precedence() -> None: + """When both are set, `timeout` wins over `timeout_in_seconds` (same seconds unit).""" + http_client, dummy_client = _sync_client_with_base_timeout(60) + http_client.request(path="/test", method="GET", request_options={"timeout": 30, "timeout_in_seconds": 45}) + assert dummy_client.last_request_kwargs["timeout"] == 30 + + +def test_sync_request_options_timeout_falls_back_to_base() -> None: + """When neither key is set, the client-level base timeout is used.""" + http_client, dummy_client = _sync_client_with_base_timeout(60) + http_client.request(path="/test", method="GET", request_options=None) + assert dummy_client.last_request_kwargs["timeout"] == 60 + + +def test_sync_request_options_timeout_none_falls_back_to_deprecated() -> None: + """An explicit `timeout=None` (dynamic caller) falls back to the deprecated `timeout_in_seconds`.""" + http_client, dummy_client = _sync_client_with_base_timeout(60) + request_options = cast(RequestOptions, {"timeout": None, "timeout_in_seconds": 45}) + http_client.request(path="/test", method="GET", request_options=request_options) + assert dummy_client.last_request_kwargs["timeout"] == 45 + + +@pytest.mark.asyncio +async def test_async_request_options_timeout_takes_precedence() -> None: + """Async: `timeout` wins over the deprecated `timeout_in_seconds`.""" + dummy_client = _DummyAsyncClient() + http_client = AsyncHttpClient( + httpx_client=dummy_client, # type: ignore[arg-type] + base_timeout=lambda: 60, + base_headers=lambda: {}, + base_url=lambda: "https://example.com", + async_base_headers=None, + ) + await http_client.request(path="/test", method="GET", request_options={"timeout": 30, "timeout_in_seconds": 45}) + assert dummy_client.last_request_kwargs["timeout"] == 30 From ff89f8abb1b1cf09d087490addc432942d2e89f7 Mon Sep 17 00:00:00 2001 From: "fern-api[bot]" <115122769+fern-api[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 09:48:30 +0000 Subject: [PATCH 2/2] [fern-replay] Applied customizations Patches with unresolved conflicts (1): - patch-7c34ccca: chore(pyproject): Manually increase the package version Run `fern-replay resolve` to apply these customizations. --- .fern/replay.lock | 290 +++++++++++++++++- pyproject.toml | 2 +- .../core/client_wrapper.py | 2 +- 3 files changed, 290 insertions(+), 4 deletions(-) diff --git a/.fern/replay.lock b/.fern/replay.lock index f04ebfb..677000c 100644 --- a/.fern/replay.lock +++ b/.fern/replay.lock @@ -12,5 +12,291 @@ generations: cli_version: unknown generator_versions: fernapi/fern-python-sdk: 5.14.21 -current_generation: e6179f9ae1194808b56ce70363413efa8fb470f4 -patches: [] + - commit_sha: 6ec633406e1f68e6db1f5d63e3b422d7ee24da28 + tree_hash: aa67e54c05e34792ab3896cf901d9cd97a7d111e + timestamp: 2026-07-08T09:47:47.377Z + cli_version: unknown + generator_versions: + fernapi/fern-python-sdk: 5.17.0 +current_generation: 6ec633406e1f68e6db1f5d63e3b422d7ee24da28 +patches: + - id: patch-7c34ccca + content_hash: sha256:8985648436261cded14f5dab01d668399cfbcda46ccd199918ffad9e37882f02 + original_commit: 7c34ccca9dc9cf1a117dcf162f8108e9210e4de8 + original_message: "chore(pyproject): Manually increase the package version" + original_author: kizy25 + base_generation: 129023272cad7c50e64ea9081d2a8b4a76a9f9e7 + files: + - pyproject.toml + - src/newscatcher_catchall/core/client_wrapper.py + patch_content: |+ + From 7c34ccca9dc9cf1a117dcf162f8108e9210e4de8 Mon Sep 17 00:00:00 2001 + From: kizy25 + Date: Thu, 25 Jun 2026 10:48:17 +0200 + Subject: [PATCH] chore(pyproject): Manually increase the package version + + --- + pyproject.toml | 2 +- + src/newscatcher_catchall/core/client_wrapper.py | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + + diff --git a/pyproject.toml b/pyproject.toml + index c6579e4..958beb2 100644 + --- a/pyproject.toml + +++ b/pyproject.toml + @@ -4,7 +4,7 @@ dynamic = ["version"] + + [tool.poetry] + name = "newscatcher-catchall-sdk" + -version = "0.0.0.dev0" + +version = "3.1.0" + description = "" + readme = "README.md" + authors = [] + diff --git a/src/newscatcher_catchall/core/client_wrapper.py b/src/newscatcher_catchall/core/client_wrapper.py + index 4841652..aeb35d6 100644 + --- a/src/newscatcher_catchall/core/client_wrapper.py + +++ b/src/newscatcher_catchall/core/client_wrapper.py + @@ -29,12 +29,12 @@ class BaseClientWrapper: + import platform + + headers: typing.Dict[str, str] = { + - "User-Agent": "newscatcher-catchall-sdk/0.0.0.dev0", + + "User-Agent": "newscatcher-catchall-sdk/3.1.0", + "X-Fern-Language": "Python", + "X-Fern-Runtime": f"python/{platform.python_version()}", + "X-Fern-Platform": f"{platform.system().lower()}/{platform.release()}", + "X-Fern-SDK-Name": "newscatcher-catchall-sdk", + - "X-Fern-SDK-Version": "0.0.0.dev0", + + "X-Fern-SDK-Version": "3.1.0", + **(self.get_custom_headers() or {}), + } + headers["x-api-key"] = self.api_key + -- + 2.52.0 + + theirs_snapshot: + pyproject.toml: | + [project] + name = "newscatcher-catchall-sdk" + dynamic = ["version"] + + [tool.poetry] + name = "newscatcher-catchall-sdk" + version = "3.1.0" + description = "" + readme = "README.md" + authors = [] + keywords = [] + license = "MIT" + classifiers = [ + "Intended Audience :: Developers", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: MacOS", + "Operating System :: POSIX :: Linux", + "Operating System :: Microsoft :: Windows", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", + "License :: OSI Approved :: MIT License" + ] + packages = [ + { include = "newscatcher_catchall", from = "src"} + ] + + [tool.poetry.urls] + Repository = 'https://github.com/Newscatcher/newscatcher-catchall-python' + + [tool.poetry.dependencies] + python = "^3.10" + aiohttp = { version = ">=3.14.1,<4", optional = true, python = ">=3.10"} + httpx = ">=0.21.2" + httpx-aiohttp = { version = "0.1.8", optional = true, python = ">=3.10"} + pydantic = ">= 1.9.2" + pydantic-core = ">=2.18.2,<3.0.0" + typing_extensions = ">= 4.0.0" + + [tool.poetry.group.dev.dependencies] + mypy = "==1.13.0" + pytest = "^9.0.3" + pytest-asyncio = "^1.0.0" + pytest-xdist = "^3.6.1" + python-dateutil = "^2.9.0" + types-python-dateutil = "^2.9.0.20240316" + urllib3 = ">=2.6.3,<3.0.0" + ruff = "==0.11.5" + + [tool.pytest.ini_options] + testpaths = [ "tests" ] + asyncio_mode = "auto" + norecursedirs = [ "src" ] + markers = [ + "aiohttp: tests that require httpx_aiohttp to be installed", + ] + + [tool.mypy] + plugins = ["pydantic.mypy"] + + [tool.ruff] + line-length = 120 + + [tool.ruff.lint] + select = [ + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort + ] + ignore = [ + "E402", # Module level import not at top of file + "E501", # Line too long + "E711", # Comparison to `None` should be `cond is not None` + "E712", # Avoid equality comparisons to `True`; use `if ...:` checks + "E721", # Use `is` and `is not` for type comparisons, or `isinstance()` for insinstance checks + "E722", # Do not use bare `except` + "E731", # Do not assign a `lambda` expression, use a `def` + "F821", # Undefined name + "F841" # Local variable ... is assigned to but never used + ] + + [tool.ruff.lint.isort] + section-order = ["future", "standard-library", "third-party", "first-party"] + + [build-system] + requires = ["poetry-core"] + build-backend = "poetry.core.masonry.api" + + [tool.poetry.extras] + aiohttp=["aiohttp", "httpx-aiohttp"] + src/newscatcher_catchall/core/client_wrapper.py: | + # This file was auto-generated by Fern from our API Definition. + + import typing + + import httpx + from .http_client import AsyncHttpClient, HttpClient + from .logging import LogConfig, Logger + + + class BaseClientWrapper: + def __init__( + self, + *, + api_key: str, + headers: typing.Optional[typing.Dict[str, str]] = None, + base_url: str, + timeout: typing.Optional[float] = None, + max_retries: int = 2, + logging: typing.Optional[typing.Union[LogConfig, Logger]] = None, + ): + self.api_key = api_key + self._headers = headers + self._base_url = base_url + self._timeout = timeout + self._max_retries = max_retries + self._logging = logging + + def get_headers(self) -> typing.Dict[str, str]: + import platform + + headers: typing.Dict[str, str] = { + "User-Agent": "newscatcher-catchall-sdk/3.1.0", + "X-Fern-Language": "Python", + "X-Fern-Runtime": f"python/{platform.python_version()}", + "X-Fern-Platform": f"{platform.system().lower()}/{platform.release()}", + "X-Fern-SDK-Name": "newscatcher-catchall-sdk", + "X-Fern-SDK-Version": "3.1.0", + **(self.get_custom_headers() or {}), + } + headers["x-api-key"] = self.api_key + return headers + + def get_custom_headers(self) -> typing.Optional[typing.Dict[str, str]]: + return self._headers + + def get_base_url(self) -> str: + return self._base_url + + def get_timeout(self) -> typing.Optional[float]: + return self._timeout + + def get_max_retries(self) -> int: + return self._max_retries + + + class SyncClientWrapper(BaseClientWrapper): + def __init__( + self, + *, + api_key: str, + headers: typing.Optional[typing.Dict[str, str]] = None, + base_url: str, + timeout: typing.Optional[float] = None, + max_retries: int = 2, + logging: typing.Optional[typing.Union[LogConfig, Logger]] = None, + httpx_client: httpx.Client, + ): + super().__init__( + api_key=api_key, + headers=headers, + base_url=base_url, + timeout=timeout, + max_retries=max_retries, + logging=logging, + ) + self.httpx_client = HttpClient( + httpx_client=httpx_client, + base_headers=self.get_headers, + base_timeout=self.get_timeout, + base_url=self.get_base_url, + base_max_retries=self.get_max_retries(), + logging_config=self._logging, + ) + + + class AsyncClientWrapper(BaseClientWrapper): + def __init__( + self, + *, + api_key: str, + headers: typing.Optional[typing.Dict[str, str]] = None, + base_url: str, + timeout: typing.Optional[float] = None, + max_retries: int = 2, + logging: typing.Optional[typing.Union[LogConfig, Logger]] = None, + async_token: typing.Optional[typing.Callable[[], typing.Awaitable[str]]] = None, + httpx_client: httpx.AsyncClient, + ): + super().__init__( + api_key=api_key, + headers=headers, + base_url=base_url, + timeout=timeout, + max_retries=max_retries, + logging=logging, + ) + self._async_token = async_token + self.httpx_client = AsyncHttpClient( + httpx_client=httpx_client, + base_headers=self.get_headers, + base_timeout=self.get_timeout, + base_url=self.get_base_url, + base_max_retries=self.get_max_retries(), + async_base_headers=self.async_get_headers, + logging_config=self._logging, + ) + + async def async_get_headers(self) -> typing.Dict[str, str]: + headers = self.get_headers() + if self._async_token is not None: + token = await self._async_token() + headers["Authorization"] = f"Bearer {token}" + return headers + status: unresolved diff --git a/pyproject.toml b/pyproject.toml index c6579e4..958beb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ dynamic = ["version"] [tool.poetry] name = "newscatcher-catchall-sdk" -version = "0.0.0.dev0" +version = "3.1.0" description = "" readme = "README.md" authors = [] diff --git a/src/newscatcher_catchall/core/client_wrapper.py b/src/newscatcher_catchall/core/client_wrapper.py index 8764f39..4eb5f21 100644 --- a/src/newscatcher_catchall/core/client_wrapper.py +++ b/src/newscatcher_catchall/core/client_wrapper.py @@ -38,7 +38,7 @@ def get_headers(self) -> typing.Dict[str, str]: "X-Fern-Runtime": f"python/{platform.python_version()}", "X-Fern-Platform": f"{platform.system().lower()}/{platform.release()}", "X-Fern-SDK-Name": "newscatcher-catchall-sdk", - "X-Fern-SDK-Version": "0.0.0.dev0", + "X-Fern-SDK-Version": "3.1.0", **(self.get_custom_headers() or {}), } headers["x-api-key"] = self.api_key