Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .spec-sha
Original file line number Diff line number Diff line change
@@ -1 +1 @@
8c8c6f7f8bdffec317e65c2abf9e2af93692424f
7f5e61af37f1710647ae79fecc56f81d00c7cfd5
16 changes: 8 additions & 8 deletions guards.log
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
........................................................................ [ 17%]
..................................................................s..... [ 34%]
........................................................................ [ 51%]
.........F.............................................................. [ 68%]
...........................................................F............ [ 51%]
........................................................................ [ 68%]
........................................................................ [ 85%]
............................................................... [100%]
=================================== FAILURES ===================================
__________________ test_no_unexposed_sdk_params[voices.list] ___________________
_________________ test_no_unexposed_sdk_params[workflows.list] _________________

cmd = Cmd(group='voices', name='list', method='voices.list', help='List available voices.', subgroup=None, args=[], options=...ider', 'gender', 'category'], success_msg=None, destructive=False, redact=False, fallback_methods=(), gate='buildable')
cmd = Cmd(group='workflows', name='list', method='workflows.list', help='List workflows in the workspace.', subgroup=None, a...', 'last_run_status', 'updated_at'], success_msg=None, destructive=False, redact=False, fallback_methods=(), gate=None)

@pytest.mark.parametrize("cmd", TABLE, ids=lambda c: ".".join(c.path))
def test_no_unexposed_sdk_params(cmd: Cmd) -> None:
Expand All @@ -24,10 +24,10 @@ cmd = Cmd(group='voices', name='list', method='voices.list', help='List availabl
f"for it. Either add an Opt(...) to the row in _spec.py, or record the decision in "
f"_INTENTIONALLY_UNEXPOSED with the reason."
)
E AssertionError: voices.list: voices.list accepts ['provider_voice_id'] but the CLI has no flag for it. Either add an Opt(...) to the row in _spec.py, or record the decision in _INTENTIONALLY_UNEXPOSED with the reason.
E assert not {'provider_voice_id'}
E AssertionError: workflows.list: workflows.list accepts ['exclude_empty'] but the CLI has no flag for it. Either add an Opt(...) to the row in _spec.py, or record the decision in _INTENTIONALLY_UNEXPOSED with the reason.
E assert not {'exclude_empty'}

tests/build/test_sdk_contract.py:233: AssertionError
=========================== short test summary info ============================
FAILED tests/build/test_sdk_contract.py::test_no_unexposed_sdk_params[voices.list] - AssertionError: voices.list: voices.list accepts ['provider_voice_id'] but the CLI has no flag for it. Either add an Opt(...) to the row in _spec.py, or record the decision in _INTENTIONALLY_UNEXPOSED with the reason.
assert not {'provider_voice_id'}
FAILED tests/build/test_sdk_contract.py::test_no_unexposed_sdk_params[workflows.list] - AssertionError: workflows.list: workflows.list accepts ['exclude_empty'] but the CLI has no flag for it. Either add an Opt(...) to the row in _spec.py, or record the decision in _INTENTIONALLY_UNEXPOSED with the reason.
assert not {'exclude_empty'}
2 changes: 1 addition & 1 deletion src/onepin/.fern/metadata.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
}
]
},
"originGitCommit": "45e96724ab4ffc17fdc44d6f21e1f71206c318cc",
"originGitCommit": "5f8c05331687cedffed0c6ac9e97d81b9c338aa9",
"originGitCommitIsDirty": false,
"invokedBy": "ci",
"ciProvider": "github"
Expand Down
81 changes: 58 additions & 23 deletions src/onepin/core/http_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,33 @@ def _should_retry(response: httpx.Response) -> bool:
)


_AUTH_HEADERS: typing.FrozenSet[str] = frozenset()


def _redact_headers(headers: typing.Dict[str, str]) -> typing.Dict[str, str]:
return {k: ("[REDACTED]" if k.lower() in _SENSITIVE_HEADERS else v) for k, v in headers.items()}
return {
k: ("[REDACTED]" if k.lower() in _SENSITIVE_HEADERS or k.lower() in _AUTH_HEADERS else v)
for k, v in headers.items()
}


def _merge_headers(*header_dicts: typing.Optional[typing.Dict[str, typing.Any]]) -> typing.Dict[str, typing.Any]:
"""
Merge header dicts left to right. Header names are case-insensitive, so a later
`x-api-key` replaces an earlier `X-API-Key` instead of producing two headers.
"""
merged: typing.Dict[str, typing.Any] = {}
keys_by_lower: typing.Dict[str, str] = {}
for header_dict in header_dicts:
if not header_dict:
continue
for key, value in header_dict.items():
existing_key = keys_by_lower.get(key.lower())
if existing_key is not None:
del merged[existing_key]
keys_by_lower[key.lower()] = key
merged[key] = value
return merged


def _build_url(base_url: str, path: typing.Optional[str]) -> str:
Expand Down Expand Up @@ -425,11 +450,11 @@ def request(
_request_url = _build_url(base_url, path)
_request_headers = jsonable_encoder(
remove_none_from_dict(
{
**self.base_headers(),
**(headers if headers is not None else {}),
**(request_options.get("additional_headers", {}) or {} if request_options is not None else {}),
}
_merge_headers(
self.base_headers(),
headers,
request_options.get("additional_headers") if request_options is not None else None,
)
)
)
_request_headers = drop_content_type_without_body(
Expand Down Expand Up @@ -463,7 +488,12 @@ def request(
files=request_files,
timeout=timeout,
)
except (httpx.ConnectError, httpx.RemoteProtocolError):
except (httpx.ConnectError, httpx.RemoteProtocolError) as error:
# A protocol error may occur after the server applied the request, even while reading a 200 body.
if isinstance(error, httpx.RemoteProtocolError) and not (
request_options is not None and request_options.get("retry_remote_protocol_errors", False)
):
raise
if retries < max_retries:
time.sleep(_retry_timeout_from_retries(retries=retries))
return self.request(
Expand Down Expand Up @@ -594,11 +624,11 @@ def stream(
_request_url = _build_url(base_url, path)
_request_headers = jsonable_encoder(
remove_none_from_dict(
{
**self.base_headers(),
**(headers if headers is not None else {}),
**(request_options.get("additional_headers", {}) if request_options is not None else {}),
}
_merge_headers(
self.base_headers(),
headers,
request_options.get("additional_headers") if request_options is not None else None,
)
)
)
_request_headers = drop_content_type_without_body(
Expand Down Expand Up @@ -735,11 +765,11 @@ async def request(
_request_url = _build_url(base_url, path)
_request_headers = jsonable_encoder(
remove_none_from_dict(
{
**_headers,
**(headers if headers is not None else {}),
**(request_options.get("additional_headers", {}) or {} if request_options is not None else {}),
}
_merge_headers(
_headers,
headers,
request_options.get("additional_headers") if request_options is not None else None,
)
)
)
_request_headers = drop_content_type_without_body(
Expand Down Expand Up @@ -773,7 +803,12 @@ async def request(
files=request_files,
timeout=timeout,
)
except (httpx.ConnectError, httpx.RemoteProtocolError):
except (httpx.ConnectError, httpx.RemoteProtocolError) as error:
# A protocol error may occur after the server applied the request, even while reading a 200 body.
if isinstance(error, httpx.RemoteProtocolError) and not (
request_options is not None and request_options.get("retry_remote_protocol_errors", False)
):
raise
if retries < max_retries:
await asyncio.sleep(_retry_timeout_from_retries(retries=retries))
return await self.request(
Expand Down Expand Up @@ -907,11 +942,11 @@ async def stream(
_request_url = _build_url(base_url, path)
_request_headers = jsonable_encoder(
remove_none_from_dict(
{
**_headers,
**(headers if headers is not None else {}),
**(request_options.get("additional_headers", {}) if request_options is not None else {}),
}
_merge_headers(
_headers,
headers,
request_options.get("additional_headers") if request_options is not None else None,
)
)
)
_request_headers = drop_content_type_without_body(
Expand Down
85 changes: 84 additions & 1 deletion src/onepin/core/pagination.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,20 @@
from __future__ import annotations

from dataclasses import dataclass
from typing import AsyncIterator, Awaitable, Callable, Generic, Iterator, List, Optional, TypeVar
from typing import (
Any,
AsyncIterator,
Awaitable,
Callable,
Generic,
Iterator,
List,
Mapping,
Optional,
Sequence,
TypeVar,
get_args,
)

# Generic to represent the underlying type of the results within a page
T = TypeVar("T")
Expand Down Expand Up @@ -80,3 +93,73 @@ async def iter_pages(self) -> AsyncIterator[AsyncPager[T, R]]:

async def next_page(self) -> Optional[AsyncPager[T, R]]:
return await self.get_next() if self.get_next is not None else None


def get_nested_page_value(container: Any, path: Sequence[str], default: Any = None) -> Any:
"""
Read a nested paging value (e.g. `options.offset`) from a request parameter that may be
a dict, a model, None, or omitted (`...`). Returns `default` when any segment is missing.
"""
value = container
for key in path:
if value is None or value is ...:
return default
value = value.get(key) if isinstance(value, Mapping) else getattr(value, key, None)
return default if value is None or value is ... else value


def with_nested_page_value(container: Any, path: Sequence[str], value: Any, container_type: Any = None) -> Any:
"""
Return a copy of a request parameter with the nested paging value at `path` replaced,
creating omitted containers and keeping every other field. When `container_type` (or the
annotation of the parent model's field) is a model, created containers are built through that
model so field aliases (e.g. `page_offset` -> `pageOffset`) apply on the wire.
"""
if len(path) == 0:
return value
key, rest = path[0], path[1:]
if container is None or container is ...:
return _build_container(container_type, {key: with_nested_page_value(None, rest, value)})
if isinstance(container, Mapping):
return {**container, key: with_nested_page_value(container.get(key), rest, value)}
updated = with_nested_page_value(
getattr(container, key, None), rest, value, _get_field_annotation(type(container), key)
)
if hasattr(container, "model_copy"):
return container.model_copy(update={key: updated})
return container.copy(update={key: updated})


def _build_container(container_type: Any, data: Mapping[str, Any]) -> Any:
model = _find_model_class(container_type)
if model is None:
return data
try:
if hasattr(model, "model_validate"):
return model.model_validate(data)
return model.parse_obj(data)
except (TypeError, ValueError):
return data


def _find_model_class(annotation: Any) -> Any:
models = []
pending = [annotation]
while pending:
candidate = pending.pop()
if isinstance(candidate, type):
if hasattr(candidate, "model_validate") or hasattr(candidate, "parse_obj"):
models.append(candidate)
else:
pending.extend(get_args(candidate))
return models[0] if len(models) == 1 else None


def _get_field_annotation(model: Any, key: str) -> Any:
fields = getattr(model, "model_fields", None)
if isinstance(fields, Mapping):
return fields[key].annotation if key in fields else None
legacy_fields = getattr(model, "__fields__", None)
if isinstance(legacy_fields, Mapping) and key in legacy_fields:
return legacy_fields[key].outer_type_
return None
64 changes: 59 additions & 5 deletions src/onepin/core/pydantic_utilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
Callable,
ClassVar,
Dict,
FrozenSet,
List,
Mapping,
Optional,
Expand All @@ -37,8 +38,8 @@
IS_PYDANTIC_V2 = pydantic.VERSION.startswith("2.")

if IS_PYDANTIC_V2:
_datetime_adapter = pydantic.TypeAdapter(dt.datetime) # type: ignore[attr-defined]
_date_adapter = pydantic.TypeAdapter(dt.date) # type: ignore[attr-defined]
_datetime_adapter = pydantic.TypeAdapter(dt.datetime) # type: ignore[attr-defined,var-annotated]
_date_adapter = pydantic.TypeAdapter(dt.date) # type: ignore[attr-defined,var-annotated]

def parse_datetime(value: Any) -> dt.datetime: # type: ignore[misc]
if isinstance(value, dt.datetime):
Expand Down Expand Up @@ -129,7 +130,7 @@ def _decimal_encoder(dec_value: Any) -> Any:
from pydantic.typing import is_union as is_union # type: ignore[no-redef]

from .datetime_utils import serialize_datetime
from .serialization import convert_and_respect_annotation_metadata
from .serialization import convert_and_respect_annotation_metadata, get_field_to_alias_mapping
from typing_extensions import TypeAlias

T = TypeVar("T")
Expand Down Expand Up @@ -237,6 +238,59 @@ def _coerce_keys_to_aliases(model: type, data: Any) -> Any:
return rewritten


_declared_keys_cache: "weakref.WeakKeyDictionary[type, Tuple[FrozenSet[str], FrozenSet[str]]]" = (
weakref.WeakKeyDictionary()
)


def _get_declared_keys(model: type) -> Tuple[FrozenSet[str], FrozenSet[str]]:
"""
Keys that refer to a declared field (field names, Pydantic aliases and FieldMetadata aliases), together with
their casefolded forms. Computed once per model class.
"""
cached = _declared_keys_cache.get(model)
if cached is None:
fields: Mapping[str, Any] = (
getattr(model, "model_fields", {}) if IS_PYDANTIC_V2 else getattr(model, "__fields__", {})
)
keys: Set[str] = set(fields)
for field in fields.values():
alias = getattr(field, "alias", None)
if isinstance(alias, str):
keys.add(alias)
keys.update(get_field_to_alias_mapping(model).values())
cached = (frozenset(keys), frozenset(key.casefold() for key in keys))
_declared_keys_cache[model] = cached
return cached


def _allows_extra_fields(model: type) -> bool:
if IS_PYDANTIC_V2:
return getattr(model, "model_config", {}).get("extra") == "allow"
extra = getattr(getattr(model, "__config__", None), "extra", None)
return getattr(extra, "value", extra) == "allow"


def _drop_extras_colliding_with_fields(model: type, data: Any) -> Any:
"""
Drop unknown keys that match a declared field case-insensitively (e.g. `VALUE` next to `value`). Kept as
extras, they would be sent again after the declared field, and a case-insensitive decoder such as Go's
encoding/json would read them instead of it.
"""
if not isinstance(data, Mapping) or not _allows_extra_fields(model):
return data

declared_keys, declared_casefolded = _get_declared_keys(model)
colliding = {
key
for key in data
if isinstance(key, str) and key not in declared_keys and key.casefold() in declared_casefolded
}
if not colliding:
return data
return {key: value for key, value in data.items() if key not in colliding}


def parse_obj_as(type_: Type[T], object_: Any) -> T:
# convert_and_respect_annotation_metadata is required for TypedDict aliasing.
#
Expand Down Expand Up @@ -279,7 +333,7 @@ class UniversalBaseModel(pydantic.BaseModel):
@pydantic.model_validator(mode="before") # type: ignore[attr-defined]
@classmethod
def _coerce_field_names_to_aliases(cls, data: Any) -> Any:
return _coerce_keys_to_aliases(cls, data)
return _coerce_keys_to_aliases(cls, _drop_extras_colliding_with_fields(cls, data))

@pydantic.model_serializer(mode="plain", when_used="json") # type: ignore[attr-defined]
def serialize_model(self) -> Any: # type: ignore[name-defined]
Expand All @@ -295,7 +349,7 @@ class Config:

@pydantic.root_validator(pre=True)
def _coerce_field_names_to_aliases(cls, values: Any) -> Any:
return _coerce_keys_to_aliases(cls, values) # type: ignore[arg-type]
return _coerce_keys_to_aliases(cls, _drop_extras_colliding_with_fields(cls, values)) # type: ignore[arg-type]

@classmethod
def model_construct(cls: Type["Model"], _fields_set: Optional[Set[str]] = None, **values: Any) -> "Model":
Expand Down
3 changes: 3 additions & 0 deletions src/onepin/core/request_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ class RequestOptions(typing.TypedDict, total=False):

- max_retries: int. The max number of retries to attempt if the API call fails.

- retry_remote_protocol_errors: bool. Retry server disconnects and incomplete responses. Defaults to False, since the server may already have applied the request. Enable only when replaying the request is safe; bounded by max_retries.

- additional_headers: typing.Dict[str, typing.Any]. A dictionary containing additional parameters to spread into the request's header dict

- additional_query_parameters: typing.Dict[str, typing.Any]. A dictionary containing additional parameters to spread into the request's query parameters dict
Expand All @@ -32,6 +34,7 @@ class RequestOptions(typing.TypedDict, total=False):
timeout: NotRequired[float]
timeout_in_seconds: NotRequired[int]
max_retries: NotRequired[int]
retry_remote_protocol_errors: NotRequired[bool]
additional_headers: NotRequired[typing.Dict[str, typing.Any]]
additional_query_parameters: NotRequired[typing.Dict[str, typing.Any]]
additional_body_parameters: NotRequired[typing.Dict[str, typing.Any]]
Expand Down
Loading
Loading