Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,34 @@ To learn more about TypeChat, visit the
[documentation](https://microsoft.github.io/TypeChat/docs/python/basic-usage/)
which includes more information on TypeChat and how to get started.

## Documenting schema fields

Trailing Python comments on directly declared `TypedDict` and dataclass fields become
documentation in the generated TypeScript schema:

```python
from typing import Annotated
from typing_extensions import Doc, TypedDict

class Point(TypedDict):
x: int # X-coordinate
y: Annotated[int, Doc("Y-coordinate")] # Explicit documentation takes precedence.
```

Explicit `Annotated` documentation, supplied through `Doc` or a string, takes precedence
over an inline comment, including when that explicit documentation is empty.
Comments are recognized using Python's tokenizer, so `#` inside a string literal is not
treated as documentation. For multiline field declarations, put the comment on the final line.
`# type: ignore` directives, including those with bracketed error codes, are not treated
as field documentation.

Source inspection is read-only and supports nested class definitions with multiline strings.
If source is unavailable, such as for dynamically created types or deployments without source files,
conversion continues using the type annotations
and explicit documentation without inline comments. If available source cannot be parsed,
the problem is reported in `TypeScriptSchemaConversionResult.errors` rather than silently
discarding the comments.

## Accessing failed output

`Failure` retains its existing `message` and can also carry an unvalidated `value: object` and
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
from __future__ import annotations

import ast
import collections.abc
from collections import OrderedDict
import inspect
import io
import re
import sys
import tokenize
import typing
import typing_extensions
from dataclasses import MISSING, Field, dataclass
Expand Down Expand Up @@ -163,6 +167,48 @@ class TypeScriptNodeTranslationResult:
}


def _extract_inline_field_comments(py_type: type) -> dict[str, str]:
"""Extract comments for directly declared fields when the class source is available."""
try:
source_lines, _ = inspect.getsourcelines(py_type)
except (OSError, TypeError):
return {}

source = "".join(source_lines)
try:
statements = ast.parse(source).body
except IndentationError:
# An outer suite accepts nested source without changing multiline string contents.
source = "if True:\n" + source
statements = ast.parse(source).body
if len(statements) == 1 and isinstance(statements[0], ast.If):
statements = statements[0].body
if len(statements) != 1 or not isinstance(statements[0], ast.ClassDef):
raise ValueError("Source does not contain a single class definition.")

comments_by_line = {
token.start[0]: token.string[1:].strip()
for token in tokenize.generate_tokens(io.StringIO(source).readline)
if token.type == tokenize.COMMENT and not re.match(r"#\s*type:\s*ignore\b", token.string)
}
lines = source.split("\n")
comments: dict[str, str] = {}
for stmt in statements[0].body:
if (
isinstance(stmt, ast.AnnAssign)
and isinstance(stmt.target, ast.Name)
and stmt.end_lineno is not None
and stmt.end_col_offset is not None
):
comment = comments_by_line.get(stmt.end_lineno)
if comment:
# AST column offsets count UTF-8 bytes, not characters.
trailing = lines[stmt.end_lineno - 1].encode("utf-8")[stmt.end_col_offset:].decode("utf-8")
if trailing.lstrip(" \t;").startswith("#"):
comments[stmt.target.id] = comment
return comments


def python_type_to_typescript_nodes(root_py_type: object) -> TypeScriptNodeTranslationResult:
# TODO: handle conflicting names

Expand Down Expand Up @@ -314,10 +360,10 @@ def convert_to_type_node(py_type: object) -> TypeNode:
errors.append(f"'{py_type}' cannot be used as a type annotation.")
return AnyTypeReferenceNode

def declare_property(name: str, py_annotation: type | TypeAliasType, is_typeddict_attribute: bool, optionality_default: bool):
def declare_property(name: str, py_annotation: type | TypeAliasType, is_typeddict_attribute: bool, optionality_default: bool, fallback_comment: str = ""):
"""
Declare a property for a given type.
If 'optionality_default' is
If 'optionality_default' is
"""
current_annotation: object = py_annotation
origin: object
Expand Down Expand Up @@ -354,7 +400,7 @@ def declare_property(name: str, py_annotation: type | TypeAliasType, is_typeddic
optional = optionality_default

type_annotation = convert_to_type_node(skip_annotations(current_annotation))
return PropertyDeclarationNode(name, optional, comment or "", type_annotation)
return PropertyDeclarationNode(name, optional, comment if comment is not None else fallback_comment, type_annotation)

def reserve_name(val: type | TypeAliasType):
type_name = val.__name__
Expand Down Expand Up @@ -393,14 +439,20 @@ def declare_type(py_type: object):
base_attributes.setdefault(prop, set()).add(type_hint)
bases = [convert_to_type_node(base) for base in raw_but_filtered_bases]

try:
inline_comments = _extract_inline_field_comments(py_type)
except (SyntaxError, tokenize.TokenError, ValueError) as exc:
errors.append(f"Cannot extract inline field comments for {py_type.__name__}: {exc}")
inline_comments = {}

properties: list[PropertyDeclarationNode | IndexSignatureDeclarationNode] = []
if is_typeddict(py_type):
for attr_name, type_hint in annotated_members.items():
if attribute_identical_in_all_bases(attr_name, type_hint, base_attributes):
continue

assume_optional = cast(TypeOfTypedDict, py_type).__total__ is False
prop = declare_property(attr_name, type_hint, is_typeddict_attribute=True, optionality_default=assume_optional)
prop = declare_property(attr_name, type_hint, is_typeddict_attribute=True, optionality_default=assume_optional, fallback_comment=inline_comments.get(attr_name, ""))
properties.append(prop)
else:
# When a dataclass is created with no explicit docstring, @dataclass will
Expand All @@ -413,7 +465,7 @@ def declare_type(py_type: object):
for attr_name, field in cast(Dataclassish, py_type).__dataclass_fields__.items():
type_hint = annotated_members[attr_name]
optional = not(field.default is MISSING and field.default_factory is MISSING)
prop = declare_property(attr_name, type_hint, is_typeddict_attribute=False, optionality_default=optional)
prop = declare_property(attr_name, type_hint, is_typeddict_attribute=False, optionality_default=optional, fallback_comment=inline_comments.get(attr_name, ""))
properties.append(prop)

reserve_name(py_type)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
// Entry point is: 'Box'

interface Box {
// width in pixels
width: number;
// height in pixels
height: number;
label?: string;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
// Entry point is: 'Mixed'

interface Mixed {
// from Doc
labeled: string;
// just an inline comment
plain: string;
silent: number;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
// Entry point is: 'Point'

interface Point {
// X-coordinate
x: number;
// Y-coordinate
y: number;
}
Loading
Loading