diff --git a/python/README.md b/python/README.md index 97224185..47794706 100644 --- a/python/README.md +++ b/python/README.md @@ -54,6 +54,34 @@ To learn more about TypeChat, visit the [documentation](https://microsoft.github.io/TypeChat/docs/python/basic-usage/) which includes more information on TypeChat and how to get started. +## Documenting schema fields + +Trailing Python comments on directly declared `TypedDict` and dataclass fields become +documentation in the generated TypeScript schema: + +```python +from typing import Annotated +from typing_extensions import Doc, TypedDict + +class Point(TypedDict): + x: int # X-coordinate + y: Annotated[int, Doc("Y-coordinate")] # Explicit documentation takes precedence. +``` + +Explicit `Annotated` documentation, supplied through `Doc` or a string, takes precedence +over an inline comment, including when that explicit documentation is empty. +Comments are recognized using Python's tokenizer, so `#` inside a string literal is not +treated as documentation. For multiline field declarations, put the comment on the final line. +`# type: ignore` directives, including those with bracketed error codes, are not treated +as field documentation. + +Source inspection is read-only and supports nested class definitions with multiline strings. +If source is unavailable, such as for dynamically created types or deployments without source files, +conversion continues using the type annotations +and explicit documentation without inline comments. If available source cannot be parsed, +the problem is reported in `TypeScriptSchemaConversionResult.errors` rather than silently +discarding the comments. + ## Accessing failed output `Failure` retains its existing `message` and can also carry an unvalidated `value: object` and diff --git a/python/src/typechat/_internal/ts_conversion/python_type_to_ts_nodes.py b/python/src/typechat/_internal/ts_conversion/python_type_to_ts_nodes.py index 2ee12982..69a628a3 100644 --- a/python/src/typechat/_internal/ts_conversion/python_type_to_ts_nodes.py +++ b/python/src/typechat/_internal/ts_conversion/python_type_to_ts_nodes.py @@ -1,9 +1,13 @@ from __future__ import annotations +import ast import collections.abc from collections import OrderedDict import inspect +import io +import re import sys +import tokenize import typing import typing_extensions from dataclasses import MISSING, Field, dataclass @@ -163,6 +167,48 @@ class TypeScriptNodeTranslationResult: } +def _extract_inline_field_comments(py_type: type) -> dict[str, str]: + """Extract comments for directly declared fields when the class source is available.""" + try: + source_lines, _ = inspect.getsourcelines(py_type) + except (OSError, TypeError): + return {} + + source = "".join(source_lines) + try: + statements = ast.parse(source).body + except IndentationError: + # An outer suite accepts nested source without changing multiline string contents. + source = "if True:\n" + source + statements = ast.parse(source).body + if len(statements) == 1 and isinstance(statements[0], ast.If): + statements = statements[0].body + if len(statements) != 1 or not isinstance(statements[0], ast.ClassDef): + raise ValueError("Source does not contain a single class definition.") + + comments_by_line = { + token.start[0]: token.string[1:].strip() + for token in tokenize.generate_tokens(io.StringIO(source).readline) + if token.type == tokenize.COMMENT and not re.match(r"#\s*type:\s*ignore\b", token.string) + } + lines = source.split("\n") + comments: dict[str, str] = {} + for stmt in statements[0].body: + if ( + isinstance(stmt, ast.AnnAssign) + and isinstance(stmt.target, ast.Name) + and stmt.end_lineno is not None + and stmt.end_col_offset is not None + ): + comment = comments_by_line.get(stmt.end_lineno) + if comment: + # AST column offsets count UTF-8 bytes, not characters. + trailing = lines[stmt.end_lineno - 1].encode("utf-8")[stmt.end_col_offset:].decode("utf-8") + if trailing.lstrip(" \t;").startswith("#"): + comments[stmt.target.id] = comment + return comments + + def python_type_to_typescript_nodes(root_py_type: object) -> TypeScriptNodeTranslationResult: # TODO: handle conflicting names @@ -314,10 +360,10 @@ def convert_to_type_node(py_type: object) -> TypeNode: errors.append(f"'{py_type}' cannot be used as a type annotation.") return AnyTypeReferenceNode - def declare_property(name: str, py_annotation: type | TypeAliasType, is_typeddict_attribute: bool, optionality_default: bool): + def declare_property(name: str, py_annotation: type | TypeAliasType, is_typeddict_attribute: bool, optionality_default: bool, fallback_comment: str = ""): """ Declare a property for a given type. - If 'optionality_default' is + If 'optionality_default' is """ current_annotation: object = py_annotation origin: object @@ -354,7 +400,7 @@ def declare_property(name: str, py_annotation: type | TypeAliasType, is_typeddic optional = optionality_default type_annotation = convert_to_type_node(skip_annotations(current_annotation)) - return PropertyDeclarationNode(name, optional, comment or "", type_annotation) + return PropertyDeclarationNode(name, optional, comment if comment is not None else fallback_comment, type_annotation) def reserve_name(val: type | TypeAliasType): type_name = val.__name__ @@ -393,6 +439,12 @@ def declare_type(py_type: object): base_attributes.setdefault(prop, set()).add(type_hint) bases = [convert_to_type_node(base) for base in raw_but_filtered_bases] + try: + inline_comments = _extract_inline_field_comments(py_type) + except (SyntaxError, tokenize.TokenError, ValueError) as exc: + errors.append(f"Cannot extract inline field comments for {py_type.__name__}: {exc}") + inline_comments = {} + properties: list[PropertyDeclarationNode | IndexSignatureDeclarationNode] = [] if is_typeddict(py_type): for attr_name, type_hint in annotated_members.items(): @@ -400,7 +452,7 @@ def declare_type(py_type: object): continue assume_optional = cast(TypeOfTypedDict, py_type).__total__ is False - prop = declare_property(attr_name, type_hint, is_typeddict_attribute=True, optionality_default=assume_optional) + prop = declare_property(attr_name, type_hint, is_typeddict_attribute=True, optionality_default=assume_optional, fallback_comment=inline_comments.get(attr_name, "")) properties.append(prop) else: # When a dataclass is created with no explicit docstring, @dataclass will @@ -413,7 +465,7 @@ def declare_type(py_type: object): for attr_name, field in cast(Dataclassish, py_type).__dataclass_fields__.items(): type_hint = annotated_members[attr_name] optional = not(field.default is MISSING and field.default_factory is MISSING) - prop = declare_property(attr_name, type_hint, is_typeddict_attribute=False, optionality_default=optional) + prop = declare_property(attr_name, type_hint, is_typeddict_attribute=False, optionality_default=optional, fallback_comment=inline_comments.get(attr_name, "")) properties.append(prop) reserve_name(py_type) diff --git a/python/tests/__snapshots__/test_inline_comments/test_dataclass_inline_comments.schema.d.ts b/python/tests/__snapshots__/test_inline_comments/test_dataclass_inline_comments.schema.d.ts new file mode 100644 index 00000000..7187696b --- /dev/null +++ b/python/tests/__snapshots__/test_inline_comments/test_dataclass_inline_comments.schema.d.ts @@ -0,0 +1,9 @@ +// Entry point is: 'Box' + +interface Box { + // width in pixels + width: number; + // height in pixels + height: number; + label?: string; +} diff --git a/python/tests/__snapshots__/test_inline_comments/test_doc_takes_priority_over_inline.schema.d.ts b/python/tests/__snapshots__/test_inline_comments/test_doc_takes_priority_over_inline.schema.d.ts new file mode 100644 index 00000000..37a9ec4e --- /dev/null +++ b/python/tests/__snapshots__/test_inline_comments/test_doc_takes_priority_over_inline.schema.d.ts @@ -0,0 +1,9 @@ +// Entry point is: 'Mixed' + +interface Mixed { + // from Doc + labeled: string; + // just an inline comment + plain: string; + silent: number; +} diff --git a/python/tests/__snapshots__/test_inline_comments/test_typeddict_inline_comments.schema.d.ts b/python/tests/__snapshots__/test_inline_comments/test_typeddict_inline_comments.schema.d.ts new file mode 100644 index 00000000..7750b066 --- /dev/null +++ b/python/tests/__snapshots__/test_inline_comments/test_typeddict_inline_comments.schema.d.ts @@ -0,0 +1,8 @@ +// Entry point is: 'Point' + +interface Point { + // X-coordinate + x: number; + // Y-coordinate + y: number; +} diff --git a/python/tests/test_inline_comments.py b/python/tests/test_inline_comments.py new file mode 100644 index 00000000..0f1001c6 --- /dev/null +++ b/python/tests/test_inline_comments.py @@ -0,0 +1,364 @@ +from dataclasses import dataclass, make_dataclass +import inspect +from textwrap import dedent +from typing import Annotated, Literal + +import pytest +from typing_extensions import Any, TypedDict, Doc + +from typechat import python_type_to_typescript_schema +from .utilities import TypeScriptSchemaSnapshotExtension + + +class Point(TypedDict): + x: int # X-coordinate + y: int # Y-coordinate + + +class Mixed(TypedDict): + labeled: Annotated[str, Doc("from Doc")] # inline comment (Doc takes priority) + plain: str # just an inline comment + silent: float + + +@dataclass +class Box: + width: float # width in pixels + height: float # height in pixels + label: str = "" + + +class Theme(TypedDict): + background: Literal["#FFFFFF", "#000000"] + accent: Literal["#FF0000"] + + +class Swatch(TypedDict): + color: Literal["#00FF00"] # the accent color as a hex triplet + + +@dataclass +class GitConfig: + comment_marker: str = "value # not a comment" + + +@dataclass +class ApiEndpoint: + url: str = "https://api.example.com/v1/items#latest" + + +class ButtonSpec(TypedDict): + variant: Literal[ + "primary", + "hash-#-danger"] # pick one variant + + +class ExplicitDocumentation(TypedDict): + empty_doc: Annotated[int, Doc("")] # not the documentation + legacy: Annotated[int, "explicit documentation"] # not the documentation + empty_legacy: Annotated[int, ""] # not the documentation + + +@dataclass +class Outer: + value: int # outer field + plain: str + + @dataclass + class Inner: + value: int # inner field + plain: str # inner only + + +DynamicPoint = TypedDict("DynamicPoint", {"value": Annotated[int, Doc("explicit documentation")]}) +DynamicBox = make_dataclass("DynamicBox", [("value", Annotated[int, Doc("explicit documentation")])]) + + +def assert_schema(py_type: type, expected: str) -> None: + result = python_type_to_typescript_schema(py_type) + assert result.errors == [] + assert result.typescript_schema_str == dedent(expected).lstrip() + + +def test_typeddict_inline_comments(snapshot: Any): + assert python_type_to_typescript_schema(Point) == snapshot(extension_class=TypeScriptSchemaSnapshotExtension) + + +def test_doc_takes_priority_over_inline(snapshot: Any): + assert python_type_to_typescript_schema(Mixed) == snapshot(extension_class=TypeScriptSchemaSnapshotExtension) + + +def test_dataclass_inline_comments(snapshot: Any): + assert python_type_to_typescript_schema(Box) == snapshot(extension_class=TypeScriptSchemaSnapshotExtension) + + +@pytest.mark.parametrize(("py_type", "expected"), [ + (Theme, """ + interface Theme { + background: "#FFFFFF" | "#000000"; + accent: "#FF0000"; + } + """), + (Swatch, """ + interface Swatch { + // the accent color as a hex triplet + color: "#00FF00"; + } + """), + (GitConfig, """ + interface GitConfig { + comment_marker?: string; + } + """), + (ApiEndpoint, """ + interface ApiEndpoint { + url?: string; + } + """), + (ButtonSpec, """ + interface ButtonSpec { + // pick one variant + variant: "primary" | "hash-#-danger"; + } + """), +]) +def test_hashes_in_types_and_defaults_are_not_comments(py_type: type, expected: str): + assert_schema(py_type, expected) + + +def test_explicit_documentation_takes_precedence_even_when_empty(): + assert_schema(ExplicitDocumentation, """ + interface ExplicitDocumentation { + empty_doc: number; + // explicit documentation + legacy: number; + empty_legacy: number; + } + """) + + +def test_nested_class_comments_do_not_leak_into_the_outer_class(): + assert_schema(Outer, """ + interface Outer { + // outer field + value: number; + plain: string; + } + """) + + +def test_nested_classes_can_be_translated_independently(): + assert_schema(Outer.Inner, """ + interface Inner { + // inner field + value: number; + // inner only + plain: string; + } + """) + + +def test_nested_dataclass_multiline_default_without_comments(): + @dataclass + class NestedDefault: + value: str = """first line +column-zero line +""" + + assert_schema(NestedDefault, """ + interface NestedDefault { + value?: string; + } + """) + + +def test_nested_dataclass_multiline_default_preserves_comment_coordinates(): + @dataclass + class NestedDefault: + value: str = """first line +# part of the string, not documentation +""" # value documentation + count: int = 1 # count documentation + + assert_schema(NestedDefault, """ + interface NestedDefault { + // value documentation + value?: string; + // count documentation + count?: number; + } + """) + + +def test_nested_typeddict_multiline_literal_preserves_comment_coordinates(): + class NestedLiteral(TypedDict): + value: Literal["""first line +# part of the literal +last line"""] # literal documentation + + assert_schema(NestedLiteral, """ + interface NestedLiteral { + // literal documentation + value: "first line\\n# part of the literal\\nlast line"; + } + """) + + +def test_nested_class_multiline_docstring_preserves_documentation(): + @dataclass + class NestedDocstring: + """Class documentation. +Column-zero documentation. +""" + value: int # value documentation + + assert_schema(NestedDocstring, """ + // Class documentation. + // Column-zero documentation. + interface NestedDocstring { + // value documentation + value: number; + } + """) + + +@pytest.mark.parametrize("error_type", [OSError, TypeError]) +def test_unavailable_source_preserves_types_and_explicit_documentation( + monkeypatch: pytest.MonkeyPatch, + error_type: type[Exception], +): + def unavailable(_obj: object) -> tuple[list[str], int]: + raise error_type("source unavailable") + + monkeypatch.setattr(inspect, "getsourcelines", unavailable) + assert_schema(Mixed, """ + interface Mixed { + // from Doc + labeled: string; + plain: string; + silent: number; + } + """) + + +@pytest.mark.parametrize("py_type", [DynamicPoint, DynamicBox]) +def test_dynamically_created_types_remain_supported(py_type: type): + assert_schema(py_type, f""" + interface {py_type.__name__} {{ + // explicit documentation + value: number; + }} + """) + + +def test_unexpected_source_lookup_failures_are_not_silenced(monkeypatch: pytest.MonkeyPatch): + def broken(_obj: object) -> tuple[list[str], int]: + raise RuntimeError("broken source lookup") + + monkeypatch.setattr(inspect, "getsourcelines", broken) + with pytest.raises(RuntimeError, match="broken source lookup"): + python_type_to_typescript_schema(Point) + + +@pytest.mark.parametrize("source", [ + "class Point(TypedDict):\n x: Literal[\n", + "value = 1\n", + " class Point(TypedDict):\n x: int\nvalue = 1\n", +]) +def test_invalid_source_is_reported_as_a_conversion_error(monkeypatch: pytest.MonkeyPatch, source: str): + def get_source(_obj: object) -> tuple[list[str], int]: + return source.splitlines(keepends=True), 1 + + monkeypatch.setattr(inspect, "getsourcelines", get_source) + result = python_type_to_typescript_schema(Point) + assert len(result.errors) == 1 + assert result.errors[0].startswith("Cannot extract inline field comments for Point:") + + +@pytest.mark.parametrize(("source", "expected"), [ + (""" + class Point(TypedDict): + x: int; y: int # second field only + """, """ + interface Point { + x: number; + // second field only + y: number; + } + """), + (""" + class Point(TypedDict): + x: int; # first field + # not documentation for y + y: int + """, """ + interface Point { + // first field + x: number; + y: number; + } + """), + (""" + class Point(TypedDict): + x: Literal["\u00e9"] # first field + y: int + """, """ + interface Point { + // first field + x: number; + y: number; + } + """), + (""" + class Point(TypedDict): + x: Literal["line\u2028separator"] # first field + y: int # second field + """, """ + interface Point { + // first field + x: number; + // second field + y: number; + } + """), + (""" + class Point(TypedDict): + x: int # type: ignore + y: int #type:ignore[assignment, attr-defined] # explanation + """, """ + interface Point { + x: number; + y: number; + } + """), + (""" + class Point(TypedDict): + x: Literal["# type: ignore"] # first field + y: int # type: ignored is ordinary documentation + """, """ + interface Point { + // first field + x: number; + // type: ignored is ordinary documentation + y: number; + } + """), +]) +def test_only_trailing_field_comments_are_attached( + monkeypatch: pytest.MonkeyPatch, + source: str, + expected: str, +): + def get_source(_obj: object) -> tuple[list[str], int]: + return dedent(source).lstrip().splitlines(keepends=True), 1 + + monkeypatch.setattr(inspect, "getsourcelines", get_source) + assert_schema(Point, expected) + + +def test_inline_comments_do_not_hide_unsupported_schema_types(): + class Invalid(TypedDict): + value: complex # documentation does not make this type supported + + result = python_type_to_typescript_schema(Invalid) + assert any("complex" in error and "cannot be translated" in error for error in result.errors)