From bc415d83d2ab467a9a5c82721f731ee6ae36a136 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 12 Aug 2026 08:13:23 -0600 Subject: [PATCH 1/7] python: Check context Implements the context checks that are present in the other language bindings in python as well --- .../lang/templates/python/model.py.j2 | 18 ++++++++++++++++-- .../lang/templates/python/model.pyi.j2 | 2 +- tests/data/python/bad-object-type-inline.json | 2 +- .../data/python/bad-object-type-ref-after.json | 2 +- .../python/bad-object-type-ref-before.json | 2 +- tests/test_python.py | 14 ++++++++------ 6 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/shacl2code/lang/templates/python/model.py.j2 b/src/shacl2code/lang/templates/python/model.py.j2 index 17cae1a3..38d676e6 100644 --- a/src/shacl2code/lang/templates/python/model.py.j2 +++ b/src/shacl2code/lang/templates/python/model.py.j2 @@ -2219,9 +2219,15 @@ class JSONLDDeserializer(object): """Decode SHACL objects from a pre-parsed JSON-LD data structure into the given object set.""" h = JSONLDDecoder(data, True) + ctx = [] with h.read_property("@context") as context_prop: if context_prop: - decode_context(context_prop, objectset) + ctx = decode_context(context_prop, objectset) + + if sorted(ctx) != sorted(CONTEXT_URLS): + raise ValueError( + f"Missing @context in JSON-LD data, expected one of {CONTEXT_URLS}" + ) if _JSS_SIGNATURE: with h.read_property(_JSS_SIGNATURE) as signature_prop: @@ -2714,8 +2720,14 @@ def encode_context(encoder: Encoder, objectset: SHACLObjectSet) -> None: write_context(context_list_item, ctx) -def decode_context(decoder: Decoder, objectset: SHACLObjectSet) -> None: +def decode_context(decoder: Decoder, objectset: SHACLObjectSet) -> List[str]: + static_context = [] + def _decode_ctx(d: Decoder) -> None: + if s := d.read_string(): + static_context.append(s) + return + if not d.is_object(): return @@ -2731,6 +2743,8 @@ def decode_context(decoder: Decoder, objectset: SHACLObjectSet) -> None: else: _decode_ctx(decoder) + return static_context + try: from rdflib import BNode, Literal, URIRef diff --git a/src/shacl2code/lang/templates/python/model.pyi.j2 b/src/shacl2code/lang/templates/python/model.pyi.j2 index 365f41b5..4ac1c65b 100644 --- a/src/shacl2code/lang/templates/python/model.pyi.j2 +++ b/src/shacl2code/lang/templates/python/model.pyi.j2 @@ -392,7 +392,7 @@ class JSONLDInlineSerializer: def encode_context(encoder: Any, objectset: SHACLObjectSet) -> None: ... -def decode_context(decoder: Any, objectset: SHACLObjectSet) -> None: ... +def decode_context(decoder: Any, objectset: SHACLObjectSet) -> List[str]: ... # fmt: off diff --git a/tests/data/python/bad-object-type-inline.json b/tests/data/python/bad-object-type-inline.json index 73183422..59f97226 100644 --- a/tests/data/python/bad-object-type-inline.json +++ b/tests/data/python/bad-object-type-inline.json @@ -1,5 +1,5 @@ { - "@context": "https://spdx.github.io/spdx-3-model/context.json", + "@context": "@CONTEXT_URL@", "@graph": [ { "@id": "http://serialize.example.com/self", diff --git a/tests/data/python/bad-object-type-ref-after.json b/tests/data/python/bad-object-type-ref-after.json index 340a33c4..c8a6b35f 100644 --- a/tests/data/python/bad-object-type-ref-after.json +++ b/tests/data/python/bad-object-type-ref-after.json @@ -1,5 +1,5 @@ { - "@context": "https://spdx.github.io/spdx-3-model/context.json", + "@context": "@CONTEXT_URL@", "@graph": [ { "@id": "http://serialize.example.com/self", diff --git a/tests/data/python/bad-object-type-ref-before.json b/tests/data/python/bad-object-type-ref-before.json index d0bcce4f..2d420c7b 100644 --- a/tests/data/python/bad-object-type-ref-before.json +++ b/tests/data/python/bad-object-type-ref-before.json @@ -1,5 +1,5 @@ { - "@context": "https://spdx.github.io/spdx-3-model/context.json", + "@context": "@CONTEXT_URL@", "@graph": [ { "@id": "http://serialize.example.com/test", diff --git a/tests/test_python.py b/tests/test_python.py index b99f2e9d..fae51da3 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -578,15 +578,17 @@ def test_links(filename, name, expect_tag, model, tmp_path, test_context_url): ("bad-object-type-ref-after.json", TypeError), ], ) -def test_deserialize(model, filename, expect): +def test_deserialize(filename, expect, model, test_context_url): objset = model.SHACLObjectSet() deserializer = model.JSONLDDeserializer() with (DATA_DIR / "python" / filename).open("r") as f: - if issubclass(expect, Exception): - with pytest.raises(expect): - deserializer.read(f, objset) - else: - deserializer.read(f, objset) + d = json.loads(f.read().replace("@CONTEXT_URL@", test_context_url)) + + if issubclass(expect, Exception): + with pytest.raises(expect): + deserializer.deserialize_data(d, objset) + else: + deserializer.deserialize_data(d, objset) def test_node_kind_blank(model, test_context_url): From 38f1ad6611ccdd57957166a436348fac63b560a4 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 12 Aug 2026 08:13:42 -0600 Subject: [PATCH 2/7] python: Implement same class resolution as other bindings Implements the same class resolution logic as is present in the other language bindings. --- .../lang/templates/python/model.py.j2 | 82 +++++++++++++------ .../lang/templates/python/model.pyi.j2 | 10 +-- 2 files changed, 62 insertions(+), 30 deletions(-) diff --git a/src/shacl2code/lang/templates/python/model.py.j2 b/src/shacl2code/lang/templates/python/model.py.j2 index 38d676e6..e363679c 100644 --- a/src/shacl2code/lang/templates/python/model.py.j2 +++ b/src/shacl2code/lang/templates/python/model.py.j2 @@ -414,7 +414,7 @@ class ObjectProp(IRIProp[Union[str, "SHACLObject"]]): def decode(self, decoder: Decoder, state: DecodeState) -> Union[str, "SHACLObject"]: if decoder.is_object(): - return self.cls.decode(decoder, state) + return SHACLObject.decode(decoder, state, self.cls) iri = decoder.read_iri() if iri is None: @@ -1230,23 +1230,58 @@ class SHACLObject(metaclass=SHACLObjectMeta): ) as prop_s: p.prop.encode(prop_s, value, state) - @classmethod - def _make_object(cls: Type["SHACLObject"], typ: str) -> "SHACLObject": - if typ not in cls.CLASSES: - raise TypeError(f"Unknown type {typ}") - - return cls.CLASSES[typ]() - - @classmethod + @staticmethod def decode( - cls: Type[T_SHACLObject], decoder: Decoder, state: DecodeState + decoder: Decoder, + state: DecodeState, + target_type: Optional[Type["SHACLObject"]] = None, ) -> "SHACLObject": - typ, obj_d = decoder.read_object() - if typ is None: + if not decoder.is_object(): + raise TypeError("Object expected") + + typ_iri, obj_d = decoder.read_object() + if typ_iri is None: raise TypeError("Unable to determine type for object") - typ = state.objectset.expand_iri(typ) or typ + typ_iri = state.objectset.expand_iri(typ_iri) or typ_iri + + if typ := SHACLObject.CLASSES.get(typ_iri): + if target_type and not issubclass(typ, target_type): + raise TypeError( + f"Type {typ_iri} is not valid where {target_type._TYPE} is expected" + ) + + if typ.IS_ABSTRACT: + raise NotImplementedError( + f"{typ._TYPE} is abstract and cannot be implemented" + ) + + obj = typ() + elif target_type and issubclass(target_type, SHACLExtensibleObject): + obj = target_type(typ_iri) + else: + if is_IRI(typ_iri): + possible = [] + for v in SHACLObject.CLASSES.values(): + if not issubclass(v, SHACLExtensibleObject): + continue + + if v.IS_ABSTRACT: + continue + + possible.append(v) + + possible.sort(key=lambda k: k._TYPE) + + for t in possible: + try: + return SHACLObject.decode(decoder, state, t) + except TypeError: + pass + + raise TypeError( + f"Unable to create object of type '{typ_iri}'. No matching extensible class" + ) - obj = cls._make_object(typ) _id = obj_d.read_object_id(obj.ID_ALIAS) if _id is not None: obj._id = state.expand_iri(_id) or _id @@ -1272,7 +1307,13 @@ class SHACLObject(metaclass=SHACLObjectMeta): raise KeyError(f"Unknown property '{key}'") def _decode_prop(self, decoder: Decoder, key: str, state: DecodeState) -> bool: - if key in ("@id", self.ID_ALIAS): + if key == self.ID_ALIAS: + return True + + if self.ID_ALIAS and key == "@id": + raise KeyError("'@id' not allowed for class with ID alias") + + if key == "@id": return True with decoder.read_property(key) as prop_d: @@ -1390,15 +1431,6 @@ class SHACLExtensibleObject(SHACLObject): def _ext_data(self) -> Dict[str, Any]: return self._extensible["data"] - @classmethod - def _make_object(cls: Type["SHACLExtensibleObject"], typ: str) -> "SHACLObject": - # Check for a known type, and if so, deserialize as that instead - if typ in cls.CLASSES: - return cls.CLASSES[typ]() - - obj = cls(typ) - return obj - def _decode_properties(self, decoder: Decoder, state: DecodeState) -> None: def decode_value(d: Decoder) -> Any: if not d.is_list(): @@ -1859,7 +1891,7 @@ class SHACLObjectSet(object): """Decode objects from the decoder and add them to this set, then link all references.""" self.create_index() for obj_d in decoder.read_list(): - o = SHACLExtensibleObject.decode(obj_d, state) + o = SHACLObject.decode(obj_d, state) self.objects.add(o) self._link() diff --git a/src/shacl2code/lang/templates/python/model.pyi.j2 b/src/shacl2code/lang/templates/python/model.pyi.j2 index 4ac1c65b..fa96a06b 100644 --- a/src/shacl2code/lang/templates/python/model.pyi.j2 +++ b/src/shacl2code/lang/templates/python/model.pyi.j2 @@ -288,12 +288,12 @@ class SHACLObject: self, *, recursive: bool = False, visited: Optional[Set[SHACLObject]] = None ) -> Iterable[SHACLObject]: ... def encode(self, encoder: "Encoder", state: "EncodeState") -> None: ... - @classmethod - def _make_object(cls: Type[T_SHACLObject], typ: str) -> SHACLObject: ... - @classmethod + @staticmethod def decode( - cls: Type[T_SHACLObject], decoder: "Decoder", state: "DecodeState" - ) -> SHACLObject: ... + decoder: Decoder, + state: DecodeState, + target_type: Optional[Type["SHACLObject"]] = None, + ) -> "SHACLObject": ... def link_helper( self, objectset: SHACLObjectSet, From 231839f60133102410876c5912416d97ffe1551d Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 12 Aug 2026 12:23:45 -0600 Subject: [PATCH 3/7] python: Fix line numbers in generated code It is useful to have the line numbers in the jinja file line up with the line numbers in the generated file as much as possible (at least until the block at then end with all the generated classes). Rework the way the version information is generated by Jinja to make sure this is the case. --- src/shacl2code/lang/templates/python/model.py.j2 | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/shacl2code/lang/templates/python/model.py.j2 b/src/shacl2code/lang/templates/python/model.py.j2 index e363679c..a46819b1 100644 --- a/src/shacl2code/lang/templates/python/model.py.j2 +++ b/src/shacl2code/lang/templates/python/model.py.j2 @@ -717,11 +717,8 @@ def is_blank_node(s: Any) -> bool: # fmt: off """Format Guard{{ '"' }}{{ '"' }}{{ '"' }} _USE_SLOTS = {{ use_slots }} -{% if version_str %} -VERSION_STRING = "{{ version_str }}" -{% endif %}{% if version %} -VERSION = {{ version }} -{% endif %} +{% if version_str %}VERSION_STRING = "{{ version_str }}"{% endif %} +{% if version %}VERSION = {{ version }}{% endif %} {{ '"' }}{{ '"' }}{{ '"' }}Format Guard""" # fmt: on From 72389d110e5c12ca1543fc6b172aa5fdb56bd4fe Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 12 Aug 2026 12:27:41 -0600 Subject: [PATCH 4/7] python: Implement custom errors and paths Implements tracking the path for validation errors, and also raising custom exceptions that capture the path. This matches the other bindings and also helps when debugging errors --- .../lang/templates/python/cmd.py.j2 | 12 +- .../lang/templates/python/model.py.j2 | 594 ++++++++++++------ .../lang/templates/python/model.pyi.j2 | 11 + tests/test_python.py | 184 +++--- 4 files changed, 516 insertions(+), 285 deletions(-) diff --git a/src/shacl2code/lang/templates/python/cmd.py.j2 b/src/shacl2code/lang/templates/python/cmd.py.j2 index 82b1dd23..b8f30c77 100644 --- a/src/shacl2code/lang/templates/python/cmd.py.j2 +++ b/src/shacl2code/lang/templates/python/cmd.py.j2 @@ -6,9 +6,10 @@ import argparse from pathlib import Path -from typing import Any, Iterable, List +from typing import Any from .model import ( + DataPath, JSONLDDeserializer, JSONLDSerializer, ListProxy, @@ -17,13 +18,13 @@ from .model import ( ) -def print_tree(objects: Iterable[SHACLObject], all_fields: bool = False) -> None: +def print_tree(objectset: SHACLObjectSet, all_fields: bool = False) -> None: """ Print object tree """ seen = set() - def callback(value: Any, path: List[str]) -> bool: + def callback(value: Any, path: DataPath) -> bool: s = (" " * (len(path) - 1)) + f"{path[-1]}" if isinstance(value, SHACLObject): s += f" {value} ({id(value)})" @@ -47,8 +48,7 @@ def print_tree(objects: Iterable[SHACLObject], all_fields: bool = False) -> None return True - for o in objects: - o.walk(callback) + objectset.walk(callback) def main() -> int: @@ -65,7 +65,7 @@ def main() -> int: d.read(f, objectset) if args.print: - print_tree(objectset.objects) + print_tree(objectset) if args.outfile: with args.outfile.open("wb") as f: diff --git a/src/shacl2code/lang/templates/python/model.py.j2 b/src/shacl2code/lang/templates/python/model.py.j2 index a46819b1..3b8dc8bb 100644 --- a/src/shacl2code/lang/templates/python/model.py.j2 +++ b/src/shacl2code/lang/templates/python/model.py.j2 @@ -45,16 +45,70 @@ from typing import ( T_PropV = TypeVar("T_PropV") -def check_type(obj: Any, types: Union[Type[Any], Tuple[Type[Any], ...]]) -> None: +class DataPath(object): + def __init__(self, p: Optional[List[str]] = None): + if p is None: + self.path = [] + else: + self.path = p + + @contextmanager + def push_path(self, s: str) -> Iterator[DataPath]: + yield DataPath(self.path + [s]) + + @contextmanager + def push_index(self, idx: int) -> Iterator[DataPath]: + yield DataPath(self.path + [f"[{idx}]"]) + + def __str__(self) -> str: + return ".".join(self.path) + + def __len__(self) -> int: + return len(self.path) + + def __iter__(self) -> Iterator[str]: + return iter(self.path) + + def __getitem__(self, idx: int) -> str: + return self.path[idx] + + +class PathError(Exception): + def __init__(self, path, message): + self.path = path + self.message = message + + def __str__(self): + return f"{self.path}: {self.message}" + + +class ValidationError(PathError): + pass + + +class DecodeError(PathError): + pass + + +class EncodeError(PathError): + pass + + +def check_type( + path: DataPath, + obj: Any, + types: Union[Type[Any], Tuple[Type[Any], ...]], +) -> None: """Check if an object is an instance of a type or one of types. Raise a TypeError if not.""" if not isinstance(obj, types): if isinstance(types, (list, tuple)): - raise TypeError( - f"Value must be one of type: {', '.join(t.__name__ for t in types)}. Got {type(obj).__name__}" + raise ValidationError( + path, + f"Value must be one of type: {', '.join(t.__name__ for t in types)}. Got {type(obj).__name__}", ) - raise TypeError( - f"Value must be of type {types.__name__}. Got {type(obj).__name__}" + raise ValidationError( + path, f"Value must be of type {types.__name__}. Got {type(obj).__name__}" ) @@ -74,32 +128,33 @@ class Property(ABC, Generic[T_PropV]): def init(self) -> Optional[T_PropV]: return None - def validate(self, value: Any) -> None: - check_type(value, self.VALID_TYPES) + def validate(self, path: DataPath, value: Any) -> None: + check_type(path, value, self.VALID_TYPES) if self.pattern is not None and not re.search( - self.pattern, self.to_string(value) + self.pattern, self.to_string(path, value) ): - raise ValueError( - f"Value is not correctly formatted. Got '{self.to_string(value)}'" + raise ValidationError( + path, + f"Value is not correctly formatted. Got '{self.to_string(path, value)}'", ) def set(self, value: Any) -> T_PropV: return cast(T_PropV, value) - def check_min_count(self, value: T_PropV, min_count: int) -> bool: + def check_min_count(self, path: DataPath, value: T_PropV, min_count: int) -> bool: return min_count == 1 - def check_max_count(self, value: T_PropV, max_count: int) -> bool: + def check_max_count(self, path: DataPath, value: T_PropV, max_count: int) -> bool: return max_count == 1 - def elide(self, value: T_PropV) -> bool: + def elide(self, path: DataPath, value: T_PropV) -> bool: return value is None def walk( self, value: T_PropV, - callback: Callable[[Any, List[str]], bool], - path: List[str], + callback: Callable[[Any, DataPath], bool], + path: DataPath, ) -> None: callback(value, path) @@ -114,18 +169,23 @@ class Property(ABC, Generic[T_PropV]): objectset: "SHACLObjectSet", missing: Optional[Set[str]], visited: Set["SHACLObject"], + path: DataPath, ) -> Optional[T_PropV]: return value - def to_string(self, value: T_PropV) -> str: + def to_string(self, path: DataPath, value: T_PropV) -> str: return str(value) @abstractmethod - def encode(self, encoder: "Encoder", value: T_PropV, state: "EncodeState") -> None: + def encode( + self, encoder: "Encoder", value: T_PropV, state: "EncodeState", path: DataPath + ) -> None: raise NotImplementedError("Subclasses must implement encode method") @abstractmethod - def decode(self, decoder: "Decoder", state: "DecodeState") -> Optional[T_PropV]: + def decode( + self, decoder: "Decoder", state: "DecodeState", path: DataPath + ) -> Optional[T_PropV]: raise NotImplementedError("Subclasses must implement decode method") @@ -139,20 +199,28 @@ class StringProp(Property[str]): def set(self, value: Any) -> str: return str(value) - def encode(self, encoder: Encoder, value: str, state: EncodeState) -> None: + def encode( + self, encoder: Encoder, value: str, state: EncodeState, path: DataPath + ) -> None: encoder.write_string(value) - def decode(self, decoder: Decoder, state: DecodeState) -> Optional[str]: + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Optional[str]: return decoder.read_string() class AnyURIProp(StringProp): """A string property whose value is encoded as an IRI rather than a plain string.""" - def encode(self, encoder: Encoder, value: str, state: EncodeState) -> None: + def encode( + self, encoder: Encoder, value: str, state: EncodeState, path: DataPath + ) -> None: encoder.write_iri(value) - def decode(self, decoder: Decoder, state: DecodeState) -> Optional[str]: + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Optional[str]: return decoder.read_iri() @@ -170,16 +238,20 @@ class DateTimeProp(Property[datetime]): def set(self, value: datetime) -> datetime: return self._normalize(value) - def encode(self, encoder: Encoder, value: datetime, state: EncodeState) -> None: - encoder.write_datetime(self.to_string(value)) + def encode( + self, encoder: Encoder, value: datetime, state: EncodeState, path: DataPath + ) -> None: + encoder.write_datetime(self.to_string(path, value)) - def decode(self, decoder: Decoder, state: DecodeState) -> Optional[datetime]: + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Optional[datetime]: s = decoder.read_datetime() if s is None: return None if isinstance(s, datetime): return self._normalize(s) - v = self.from_string(s) + v = self.from_string(path, s) return self._normalize(v) def _normalize(self, value: datetime) -> datetime: @@ -204,15 +276,17 @@ class DateTimeProp(Property[datetime]): value = value.replace(microsecond=0) return value - def to_string(self, value: datetime) -> str: + def to_string(self, path: DataPath, value: datetime) -> str: value = self._normalize(value) if value.tzinfo == timezone.utc: return value.strftime(self.UTC_FORMAT_STR) return value.isoformat() - def from_string(self, value: str) -> datetime: + def from_string(self, path: DataPath, value: str) -> datetime: if not re.match(self.REGEX, value): - raise ValueError(f"'{value}' is not a correctly formatted datetime") + raise ValidationError( + path, f"'{value}' is not a correctly formatted datetime" + ) if "Z" in value: d = datetime( *(time.strptime(value, self.UTC_FORMAT_STR)[0:6]), @@ -244,10 +318,14 @@ class IntegerProp(Property[int]): def set(self, value: Any) -> int: return int(value) - def encode(self, encoder: Encoder, value: int, state: EncodeState) -> None: + def encode( + self, encoder: Encoder, value: int, state: EncodeState, path: DataPath + ) -> None: encoder.write_integer(value) - def decode(self, decoder: Decoder, state: DecodeState) -> Optional[int]: + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Optional[int]: return decoder.read_integer() @@ -256,10 +334,10 @@ class PositiveIntegerProp(IntegerProp): __slots__ = () - def validate(self, value: Any) -> None: - super().validate(value) + def validate(self, path: DataPath, value: Any) -> None: + super().validate(path, value) if value < 1: - raise ValueError(f"Value must be >= 1. Got {value}") + raise ValidationError(path, f"Value must be >= 1. Got {value}") class NonNegativeIntegerProp(IntegerProp): @@ -267,10 +345,10 @@ class NonNegativeIntegerProp(IntegerProp): __slots__ = () - def validate(self, value: Any) -> None: - super().validate(value) + def validate(self, path: DataPath, value: Any) -> None: + super().validate(path, value) if value < 0: - raise ValueError(f"Value must be >= 0. Got {value}") + raise ValidationError(path, f"Value must be >= 0. Got {value}") class BooleanProp(Property[bool]): @@ -283,10 +361,14 @@ class BooleanProp(Property[bool]): def set(self, value: Any) -> bool: return bool(value) - def encode(self, encoder: Encoder, value: bool, state: EncodeState) -> None: + def encode( + self, encoder: Encoder, value: bool, state: EncodeState, path: DataPath + ) -> None: encoder.write_bool(value) - def decode(self, decoder: Decoder, state: DecodeState) -> Optional[bool]: + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Optional[bool]: return decoder.read_bool() @@ -301,11 +383,17 @@ class FloatProp(Property[float]): return float(value) def encode( - self, encoder: Encoder, value: Union[float, int], state: EncodeState + self, + encoder: Encoder, + value: Union[float, int], + state: EncodeState, + path: DataPath, ) -> None: encoder.write_float(value) - def decode(self, decoder: Decoder, state: DecodeState) -> Optional[float]: + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Optional[float]: return decoder.read_float() @@ -361,8 +449,8 @@ class ObjectProp(IRIProp[Union[str, "SHACLObject"]]): self.cls = cls self.required = required - def validate(self, value: Any) -> None: - check_type(value, (self.cls, str)) + def validate(self, path: DataPath, value: Any) -> None: + check_type(path, value, (self.cls, str)) def set(self, value: Any) -> Union[str, "SHACLObject"]: if isinstance(value, SHACLObject): @@ -372,8 +460,8 @@ class ObjectProp(IRIProp[Union[str, "SHACLObject"]]): def walk( self, value: Optional[Union[str, "SHACLObject"]], - callback: Callable[[Any, List[str]], bool], - path: List[str], + callback: Callable[[Any, DataPath], bool], + path: DataPath, ) -> None: if value is None: return @@ -401,24 +489,30 @@ class ObjectProp(IRIProp[Union[str, "SHACLObject"]]): yield c def encode( - self, encoder: Encoder, value: Union[str, "SHACLObject"], state: EncodeState + self, + encoder: Encoder, + value: Union[str, "SHACLObject"], + state: EncodeState, + path: DataPath, ) -> None: if value is None: - raise ValueError("Object cannot be None") + raise EncodeError(path, "Object cannot be None") if isinstance(value, str): encoder.write_iri(value, self.compact(value) or state.compact_iri(value)) return - return value.encode(encoder, state) + return value.encode(encoder, state, path) - def decode(self, decoder: Decoder, state: DecodeState) -> Union[str, "SHACLObject"]: + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Union[str, "SHACLObject"]: if decoder.is_object(): - return SHACLObject.decode(decoder, state, self.cls) + return SHACLObject.decode(decoder, state, path, self.cls) iri = decoder.read_iri() if iri is None: - raise TypeError("IRI cannot be None") + raise DecodeError(path, "IRI cannot be None") iri = self.expand(iri) or state.expand_iri(iri) or iri @@ -429,7 +523,7 @@ class ObjectProp(IRIProp[Union[str, "SHACLObject"]]): if obj is None: return iri - self.validate(obj) + self.validate(path, obj) return obj def link_prop( @@ -438,6 +532,7 @@ class ObjectProp(IRIProp[Union[str, "SHACLObject"]]): objectset: "SHACLObjectSet", missing: Optional[Set[str]], visited: Set["SHACLObject"], + path: DataPath, ) -> Optional[Union[str, "SHACLObject"]]: if value is None: return value @@ -445,7 +540,7 @@ class ObjectProp(IRIProp[Union[str, "SHACLObject"]]): if isinstance(value, str): o = objectset.find_by_id(value) if o is not None: - self.validate(o) + self.validate(path, o) return o if missing is not None: @@ -459,9 +554,9 @@ class ObjectProp(IRIProp[Union[str, "SHACLObject"]]): # find_by_id will always return a SHACLObject because we pass value as default. # So we can safely cast here to allow subsequent value.link_helper call to work. value = cast("SHACLObject", objectset.find_by_id(value._id, value)) - self.validate(value) + self.validate(path, value) - value.link_helper(objectset, missing, visited) + value.link_helper(objectset, missing, visited, path) return value @@ -480,11 +575,13 @@ class ListProxy(Generic[T_PropV]): self._prop: Property[T_PropV] = prop def append(self, value: T_PropV) -> None: - self._prop.validate(value) + with DataPath().push_index(len(self._data)) as p: + self._prop.validate(p, value) self._data.append(self._prop.set(value)) def insert(self, idx: int, value: T_PropV) -> None: - self._prop.validate(value) + with DataPath().push_index(idx) as p: + self._prop.validate(p, value) self._data.insert(idx, self._prop.set(value)) def extend(self, items: Iterable[T_PropV]) -> None: @@ -512,13 +609,15 @@ class ListProxy(Generic[T_PropV]): def __setitem__( self, key: Union[int, slice], value: Union[T_PropV, Iterable[T_PropV]] ) -> None: + path = DataPath() if isinstance(key, slice): val_iter = cast(Iterable[T_PropV], value) - for v in val_iter: - self._prop.validate(v) + for idx, v in enumerate(val_iter): + with path.push_index(idx) as p: + self._prop.validate(p, v) self._data[key] = [self._prop.set(v) for v in val_iter] elif isinstance(key, int): - self._prop.validate(value) + self._prop.validate(path, value) self._data[key] = self._prop.set(value) else: raise TypeError( @@ -566,11 +665,12 @@ class ListProp(Property[ListProxy[T_PropV]]): def init(self) -> ListProxy[T_PropV]: return ListProxy(self.prop) - def validate(self, value: Any) -> None: - super().validate(value) + def validate(self, path: DataPath, value: Any) -> None: + super().validate(path, value) - for i in value: - self.prop.validate(i) + for idx, i in enumerate(value): + with path.push_index(idx) as p: + self.prop.validate(p, i) def set( self, @@ -581,29 +681,34 @@ class ListProp(Property[ListProxy[T_PropV]]): return ListProxy(self.prop, [self.prop.set(d) for d in value]) - def check_min_count(self, value: ListProxy[T_PropV], min_count: int) -> bool: - check_type(value, ListProxy) + def check_min_count( + self, path: DataPath, value: ListProxy[T_PropV], min_count: int + ) -> bool: + check_type(path, value, ListProxy) return len(value) >= min_count - def check_max_count(self, value: ListProxy[T_PropV], max_count: int) -> bool: - check_type(value, ListProxy) + def check_max_count( + self, path: DataPath, value: ListProxy[T_PropV], max_count: int + ) -> bool: + check_type(path, value, ListProxy) return len(value) <= max_count - def elide(self, value: ListProxy[T_PropV]) -> bool: - check_type(value, ListProxy) + def elide(self, path: DataPath, value: ListProxy[T_PropV]) -> bool: + check_type(path, value, ListProxy) return len(value) == 0 def walk( self, value: ListProxy[T_PropV], - callback: Callable[[Any, List[str]], bool], - path: List[str], + callback: Callable[[Any, DataPath], bool], + path: DataPath, ) -> None: if value is None: return callback(value, path) for idx, v in enumerate(value): - self.prop.walk(v, callback, path + [f"[{idx}]"]) + with path.push_index(idx) as p: + self.prop.walk(v, callback, p) def iter_objects( self, @@ -623,34 +728,46 @@ class ListProp(Property[ListProxy[T_PropV]]): objectset: "SHACLObjectSet", missing: Optional[Set[str]], visited: Set["SHACLObject"], + path: DataPath, ) -> ListProxy[T_PropV]: if value is None: return ListProxy(self.prop) - data: List[T_PropV] = [ - cast(T_PropV, self.prop.link_prop(v, objectset, missing, visited)) - for v in value - ] + data: List[T_PropV] = [] + for idx, v in enumerate(value): + with path.push_index(idx) as p: + data.append( + cast( + T_PropV, self.prop.link_prop(v, objectset, missing, visited, p) + ) + ) return ListProxy(self.prop, data=data) def encode( - self, encoder: Encoder, value: ListProxy[T_PropV], state: EncodeState + self, + encoder: Encoder, + value: ListProxy[T_PropV], + state: EncodeState, + path: DataPath, ) -> None: - check_type(value, ListProxy) + check_type(path, value, ListProxy) with encoder.write_list() as list_s: - for v in value: - with list_s.write_list_item() as item_s: - self.prop.encode(item_s, v, state) + for idx, v in enumerate(value): + with list_s.write_list_item() as item_s, path.push_index(idx) as p: + self.prop.encode(item_s, v, state, p) - def decode(self, decoder: Decoder, state: DecodeState) -> ListProxy[T_PropV]: + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> ListProxy[T_PropV]: data: List[T_PropV] = [] - for val_d in decoder.read_list(): - v = self.prop.decode(val_d, state) - if v is not None: - self.prop.validate(v) - data.append(v) + for idx, val_d in enumerate(decoder.read_list()): + with path.push_index(idx) as p: + v = self.prop.decode(val_d, state, p) + if v is not None: + self.prop.validate(p, v) + data.append(v) return ListProxy(self.prop, data=data) @@ -670,21 +787,24 @@ class EnumProp(IRIProp[str]): ) -> None: super().__init__(values, pattern=pattern) - def validate(self, value: Any) -> None: - super().validate(value) + def validate(self, path: DataPath, value: Any) -> None: + super().validate(path, value) if value not in self.iri_values(): - raise ValueError( - f"'{value}' is not a valid value. Choose one of {' '.join(self.iri_values())}" + raise ValidationError( + path, + f"'{value}' is not a valid value. Choose one of {' '.join(self.iri_values())}", ) - def encode(self, encoder: Encoder, value: str, state: EncodeState) -> None: + def encode( + self, encoder: Encoder, value: str, state: EncodeState, path: DataPath + ) -> None: encoder.write_enum(value, self, self.compact(value)) - def decode(self, decoder: Decoder, state: DecodeState) -> str: + def decode(self, decoder: Decoder, state: DecodeState, path: DataPath) -> str: v = decoder.read_enum(self) if v is None: - raise TypeError("Enum IRI cannot be None") + raise DecodeError(path, "Enum IRI cannot be None") return self.expand(v) or v @@ -1050,25 +1170,28 @@ class SHACLObject(metaclass=SHACLObjectMeta): def _is_abstract(self) -> bool: return self.__class__.IS_ABSTRACT - def __set(self, p: ClassProp, value: Any) -> None: + def __set(self, p: ClassProp, path: DataPath, value: Any) -> None: if p.iri == "@id": if self.NODE_KIND == NodeKind.BlankNode: if not is_blank_node(value): - raise ValueError( - f"{self.__class__.__name__} ({id(self)}) can only have local reference. Property '{p.iri}' cannot be set to {value!r} and must start with '_:'" + raise ValidationError( + path, + f"{self.__class__.__name__} ({id(self)}) can only have local reference. Property '{p.iri}' cannot be set to {value!r} and must start with '_:'", ) elif self.NODE_KIND == NodeKind.IRI: if not is_IRI(value): - raise ValueError( - f"{self.__class__.__name__} ({id(self)}) can only have an IRI value. Property '{p.iri}' cannot be set to {value!r}" + raise ValidationError( + path, + f"{self.__class__.__name__} ({id(self)}) can only have an IRI value. Property '{p.iri}' cannot be set to {value!r}", ) else: if not is_blank_node(value) and not is_IRI(value): - raise ValueError( - f"{self.__class__.__name__} ({id(self)}) Has invalid Property '{p.iri}' {value!r}. Must be a blank node or IRI" + raise ValidationError( + path, + f"{self.__class__.__name__} ({id(self)}) Has invalid Property '{p.iri}' {value!r}. Must be a blank node or IRI", ) - p.prop.validate(value) + p.prop.validate(path, value) if p.deprecated: warnings.warn( f"{self.__class__.__name__}.{p.pyname} is deprecated", @@ -1095,7 +1218,9 @@ class SHACLObject(metaclass=SHACLObjectMeta): object.__setattr__(self, name, value) return - self.__set(self.__get_attr(name), value) + prop = self.__get_attr(name) + with DataPath().push_path(prop.pyname) as path: + self.__set(prop, path, value) def __getattr__(self, name: str) -> Any: if name == self.ID_ALIAS: @@ -1119,7 +1244,9 @@ class SHACLObject(metaclass=SHACLObjectMeta): return getattr(self, self.__get_key(iri).pyname) def __setitem__(self, iri: str, value: Any) -> None: - self.__set(self.__get_key(iri), value) + key = self.__get_key(iri) + with DataPath().push_path(key.pyname) as path: + self.__set(key, path, value) def __delitem__(self, iri: str) -> None: self.__del(self.__get_key(iri)) @@ -1129,23 +1256,21 @@ class SHACLObject(metaclass=SHACLObjectMeta): def walk( self, - callback: Callable[[Any, List[str]], bool], - path: Optional[List[str]] = None, + callback: Callable[[Any, DataPath], bool], + path: DataPath, ) -> None: """ Walk object tree, invoking the callback for each item Callback has the form: - def walk_callback(object: Any, path: List[str]) -> bool: + def walk_callback(object: Any, path: DataPath) -> bool: ... """ - if path is None: - path = ["."] - if callback(self, path): for p in self._OBJ_PY_PROPS.values(): - p.prop.walk(getattr(self, p.pyname), callback, path + [f".{p.iri}"]) + with path.push_path(p.pyname) as prop_path: + p.prop.walk(getattr(self, p.pyname), callback, prop_path) def property_keys(self) -> Iterator[Tuple[Optional[str], str, Optional[str]]]: """Yield (python_name, iri, compact_iri) tuples for each property defined on this object.""" @@ -1171,12 +1296,13 @@ class SHACLObject(metaclass=SHACLObjectMeta): ): yield c - def encode(self, encoder: Encoder, state: EncodeState) -> None: + def encode(self, encoder: Encoder, state: EncodeState, path: DataPath) -> None: """Encode this object to the given encoder, writing its type, ID, and properties.""" idname = self.ID_ALIAS or "@id" if not self._id and self.NODE_KIND == NodeKind.IRI: - raise ValueError( - f"{self.__class__.__name__} ({id(self)}) must have a IRI for property '{idname}'" + raise EncodeError( + path, + f"{self.__class__.__name__} ({id(self)}) must have a IRI for property '{idname}'", ) _id = state.get_object_id(self) @@ -1195,61 +1321,69 @@ class SHACLObject(metaclass=SHACLObjectMeta): state.compact_iri(_id), bool(self._id) or state.is_refed(self), ) as obj_s: - self._encode_properties(obj_s, state) + self._encode_properties(obj_s, state, path) - def _encode_properties(self, encoder: Encoder, state: EncodeState) -> None: + def _encode_properties( + self, encoder: Encoder, state: EncodeState, path: DataPath + ) -> None: for p in self._OBJ_PY_PROPS.values(): - value = getattr(self, p.pyname) - if p.prop.elide(value): - if p.min_count: - raise ValueError( - f"Property '{p.pyname}' in {self.__class__.__name__} ({id(self)}) is required (currently {value!r})" - ) - continue + with path.push_path(p.pyname) as prop_path: + value = getattr(self, p.pyname) + if p.prop.elide(prop_path, value): + if p.min_count: + raise ValidationError( + prop_path, + f"Property '{p.pyname}' in {self.__class__.__name__} ({id(self)}) is required (currently {value!r})", + ) + continue - if p.min_count is not None: - if not p.prop.check_min_count(value, p.min_count): - raise ValueError( - f"Property '{p.pyname}' in {self.__class__.__name__} ({id(self)}) requires a minimum of {p.min_count} elements" - ) + if p.min_count is not None: + if not p.prop.check_min_count(prop_path, value, p.min_count): + raise ValidationError( + prop_path, + f"Property '{p.pyname}' in {self.__class__.__name__} ({id(self)}) requires a minimum of {p.min_count} elements", + ) - if p.max_count is not None: - if not p.prop.check_max_count(value, p.max_count): - raise ValueError( - f"Property '{p.pyname}' in {self.__class__.__name__} ({id(self)}) requires a maximum of {p.max_count} elements" - ) + if p.max_count is not None: + if not p.prop.check_max_count(prop_path, value, p.max_count): + raise ValidationError( + prop_path, + f"Property '{p.pyname}' in {self.__class__.__name__} ({id(self)}) requires a maximum of {p.max_count} elements", + ) - if p.iri == "@id": - continue + if p.iri == "@id": + continue - with encoder.write_property( - p.iri, p.compact or state.compact_iri(p.iri) - ) as prop_s: - p.prop.encode(prop_s, value, state) + with encoder.write_property( + p.iri, p.compact or state.compact_iri(p.iri) + ) as prop_s: + p.prop.encode(prop_s, value, state, prop_path) @staticmethod def decode( decoder: Decoder, state: DecodeState, + path: DataPath, target_type: Optional[Type["SHACLObject"]] = None, ) -> "SHACLObject": if not decoder.is_object(): - raise TypeError("Object expected") + raise DecodeError(path, "Object expected") typ_iri, obj_d = decoder.read_object() if typ_iri is None: - raise TypeError("Unable to determine type for object") + raise DecodeError(path, "Unable to determine type for object") typ_iri = state.objectset.expand_iri(typ_iri) or typ_iri if typ := SHACLObject.CLASSES.get(typ_iri): if target_type and not issubclass(typ, target_type): - raise TypeError( - f"Type {typ_iri} is not valid where {target_type._TYPE} is expected" + raise ValidationError( + path, + f"Type {typ_iri} is not valid where {target_type._TYPE} is expected", ) if typ.IS_ABSTRACT: - raise NotImplementedError( - f"{typ._TYPE} is abstract and cannot be implemented" + raise DecodeError( + path, f"{typ._TYPE} is abstract and cannot be implemented" ) obj = typ() @@ -1271,12 +1405,13 @@ class SHACLObject(metaclass=SHACLObjectMeta): for t in possible: try: - return SHACLObject.decode(decoder, state, t) - except TypeError: + return SHACLObject.decode(decoder, state, path, t) + except (ValidationError, DecodeError): pass - raise TypeError( - f"Unable to create object of type '{typ_iri}'. No matching extensible class" + raise DecodeError( + path, + f"Unable to create object of type '{typ_iri}'. No matching extensible class", ) _id = obj_d.read_object_id(obj.ID_ALIAS) @@ -1284,43 +1419,50 @@ class SHACLObject(metaclass=SHACLObjectMeta): obj._id = state.expand_iri(_id) or _id if obj.NODE_KIND == NodeKind.IRI and not obj._id: - raise ValueError("Object is missing required IRI") + raise DecodeError(path, "Object is missing required IRI") if obj._id: if obj._id in state.read_objs: return state.read_objs[obj._id] state.read_objs[obj._id] = obj - obj._decode_properties(obj_d, state) + obj._decode_properties(obj_d, state, path) if state.objectset is not None: state.objectset.add_index(obj) return obj - def _decode_properties(self, decoder: Decoder, state: DecodeState) -> None: + def _decode_properties( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> None: for key in decoder.object_keys(): - if not self._decode_prop(decoder, key, state): - raise KeyError(f"Unknown property '{key}'") + if not self._decode_prop(decoder, key, state, path): + raise DecodeError(path, f"Unknown property '{key}'") - def _decode_prop(self, decoder: Decoder, key: str, state: DecodeState) -> bool: + def _decode_prop( + self, decoder: Decoder, key: str, state: DecodeState, path: DataPath + ) -> bool: if key == self.ID_ALIAS: return True if self.ID_ALIAS and key == "@id": - raise KeyError("'@id' not allowed for class with ID alias") + raise DecodeError(path, "'@id' not allowed for class with ID alias") if key == "@id": return True with decoder.read_property(key) as prop_d: if prop_d is None: - raise TypeError(f"Property decoder for key '{key}' cannot be None") + raise DecodeError( + path, f"Property decoder for key '{key}' cannot be None" + ) if _JSS_SIGNATURE and key == _JSS_SIGNATURE: if not prop_d.is_list(): - raise TypeError( - f"Property '{key}' must be a list with signature information", + raise DecodeError( + path, + f"Property '{key}' must be an object with signature information", ) return True @@ -1334,8 +1476,9 @@ class SHACLObject(metaclass=SHACLObjectMeta): else: return False - v = p.prop.decode(prop_d, state) - self.__set(p, v) + with path.push_path(p.pyname) as prop_path: + v = p.prop.decode(prop_d, state, prop_path) + self.__set(p, prop_path, v) return True def link_helper( @@ -1343,6 +1486,7 @@ class SHACLObject(metaclass=SHACLObjectMeta): objectset: SHACLObjectSet, missing: Optional[Set[str]], visited: Set["SHACLObject"], + path: DataPath, ) -> None: """Resolve string IRI references in this object's properties to actual SHACLObject instances.""" if self in visited: @@ -1351,16 +1495,18 @@ class SHACLObject(metaclass=SHACLObjectMeta): visited.add(self) for p in self._OBJ_PY_PROPS.values(): - object.__setattr__( - self, - p.pyname, - p.prop.link_prop( - getattr(self, p.pyname), - objectset, - missing, - visited, - ), - ) + with path.push_path(p.pyname) as prop_path: + object.__setattr__( + self, + p.pyname, + p.prop.link_prop( + getattr(self, p.pyname), + objectset, + missing, + visited, + prop_path, + ), + ) def __str__(self) -> str: parts = [ @@ -1428,38 +1574,51 @@ class SHACLExtensibleObject(SHACLObject): def _ext_data(self) -> Dict[str, Any]: return self._extensible["data"] - def _decode_properties(self, decoder: Decoder, state: DecodeState) -> None: - def decode_value(d: Decoder) -> Any: + def _decode_properties( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> None: + def decode_value(d: Decoder, path: DataPath) -> Any: if not d.is_list(): - return d.read_value() + try: + return d.read_value() + except TypeError as e: + raise DecodeError(path, str(e)) + + result = [] + for idx, val_d in enumerate(d.read_list()): + with path.push_index(idx) as p: + result.append(decode_value(val_d, p)) - return [decode_value(val_d) for val_d in d.read_list()] + return result if self.CLOSED: - super()._decode_properties(decoder, state) + super()._decode_properties(decoder, state, path) return for key in decoder.object_keys(): - if self._decode_prop(decoder, key, state): + if self._decode_prop(decoder, key, state, path): continue if key is None: - raise KeyError("Property key cannot be None") + raise DecodeError(path, "Property key cannot be None") expanded_key = state.expand_iri(key) or key if not is_IRI(expanded_key): - raise KeyError( - f"Extensible object properties must be IRIs. Got '{key}' (expanded to '{expanded_key}')" + raise DecodeError( + path, + f"Extensible object properties must be IRIs. Got '{key}' (expanded to '{expanded_key}')", ) - with decoder.read_property(key) as prop_d: + with decoder.read_property(key) as prop_d, path.push_path(key) as prop_path: if prop_d is None: - raise KeyError(f"Property '{key}' has no value") - self._ext_data[expanded_key] = decode_value(prop_d) + raise DecodeError(path, f"Property '{key}' has no value") + self._ext_data[expanded_key] = decode_value(prop_d, prop_path) - def _encode_properties(self, encoder: Encoder, state: EncodeState) -> None: - super()._encode_properties(encoder, state) + def _encode_properties( + self, encoder: Encoder, state: EncodeState, path: DataPath + ) -> None: + super()._encode_properties(encoder, state, path) if self.CLOSED: return @@ -1467,7 +1626,10 @@ class SHACLExtensibleObject(SHACLObject): if iri in self._OBJ_IRI_PROPS: continue - with encoder.write_property(iri, state.compact_iri(iri)) as prop_s: + with ( + encoder.write_property(iri, state.compact_iri(iri)) as prop_s, + path.push_path(iri) as prop_path, + ): if isinstance(value, list): v = value else: @@ -1484,8 +1646,9 @@ class SHACLExtensibleObject(SHACLObject): elif isinstance(i, float): item_s.write_float(i) else: - raise TypeError( - f"Unsupported serialized type {type(i)} with value {i!r}" + raise EncodeError( + prop_path, + f"Unsupported serialized type {type(i)} with value {i!r}", ) def __getitem__(self, iri: str) -> Any: @@ -1670,11 +1833,14 @@ class SHACLObjectSet(object): visited: Set[SHACLObject] = set() new_objects: Set[SHACLObject] = set() - for o in self.objects: - if o._id: - o = cast(SHACLObject, self.find_by_id(o._id, o)) - o.link_helper(self, self.missing_ids, visited) - new_objects.add(o) + path = DataPath() + + for idx, o in enumerate(self.objects): + with path.push_index(idx) as p: + if o._id: + o = cast(SHACLObject, self.find_by_id(o._id, o)) + o.link_helper(self, self.missing_ids, visited, p) + new_objects.add(o) self.objects = new_objects @@ -1766,6 +1932,14 @@ class SHACLObjectSet(object): return SHACLObjectSet(new_objects, link=True) + def walk(self, callback: Callable[[Any, DataPath], bool]) -> None: + path = DataPath() + for idx, o in enumerate(self.objects): + # Note that every object in the root object set gets at least one + # reference + with path.push_index(idx) as p: + o.walk(callback, p) + def inline_blank_nodes(self) -> None: """ Removes (inlines) blank node objects from the root object set if they @@ -1777,7 +1951,7 @@ class SHACLObjectSet(object): """ ref_counts: Dict[SHACLObject, int] = {} - def walk_callback(value: SHACLObject, path: List[str]) -> bool: + def walk_callback(value: SHACLObject, path: DataPath) -> bool: if not isinstance(value, SHACLObject): return True @@ -1788,10 +1962,9 @@ class SHACLObjectSet(object): return True - for o in self.objects: - # Note that every object in the root object set gets at least one - # reference - o.walk(walk_callback) + # Note that every object in the root object set gets at least one + # reference + self.walk(walk_callback) new_objects: Set[SHACLObject] = set() for o in self.objects: @@ -1822,7 +1995,7 @@ class SHACLObjectSet(object): """ ref_counts: Dict[SHACLObject, int] = {} - def walk_callback(value: SHACLObject, path: List[str]) -> bool: + def walk_callback(value: SHACLObject, path: DataPath) -> bool: if not isinstance(value, SHACLObject): return True @@ -1846,7 +2019,8 @@ class SHACLObjectSet(object): for o in self.objects: if o._id: state.add_refed(o) - o.walk(walk_callback) + + self.walk(walk_callback) use_list = force_list or len(self.objects) > 1 @@ -1862,6 +2036,7 @@ class SHACLObjectSet(object): # getattr, which dominates runtime on large object sets. objects.sort(key=key if key is not None else SHACLObject._sort_key) + path = DataPath() if use_list: # Ensure top level objects are only written in the top level graph # node, and referenced by ID everywhere else. This is done by setting @@ -1875,20 +2050,25 @@ class SHACLObjectSet(object): state.written_objects.add(o) with encoder.write_object_list() as list_s: - for o in objects: + for idx, o in enumerate(objects): # Allow this specific object to be written now state.written_objects.remove(o) - with list_s.write_list_item() as item_s: - o.encode(item_s, state) + with ( + list_s.write_list_item() as item_s, + path.push_index(idx) as obj_path, + ): + o.encode(item_s, state, obj_path) elif objects: - objects[0].encode(encoder, state) + objects[0].encode(encoder, state, path) def decode(self, decoder: Decoder, state: DecodeState) -> None: """Decode objects from the decoder and add them to this set, then link all references.""" self.create_index() - for obj_d in decoder.read_list(): - o = SHACLObject.decode(obj_d, state) + path = DataPath() + for idx, obj_d in enumerate(decoder.read_list()): + with path.push_index(idx) as obj_path: + o = SHACLObject.decode(obj_d, state, obj_path) self.objects.add(o) self._link() diff --git a/src/shacl2code/lang/templates/python/model.pyi.j2 b/src/shacl2code/lang/templates/python/model.pyi.j2 index fa96a06b..cc843a37 100644 --- a/src/shacl2code/lang/templates/python/model.pyi.j2 +++ b/src/shacl2code/lang/templates/python/model.pyi.j2 @@ -31,6 +31,16 @@ from typing import ( T_PropV = TypeVar("T_PropV") +class DataPath(object): + def __init__(self, p: Optional[List[str]] = None): ... + def push_path(self, s: str) -> Iterator[DataPath]: ... + def push_index(self, idx: int) -> Iterator[DataPath]: ... + def __str__(self) -> str: ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[str]: ... + def __getitem__(self, idx: int) -> str: ... + + class Property(ABC, Generic[T_PropV]): VALID_TYPES: ClassVar[Union[Type[Any], Tuple[Type[Any], ...]]] pattern: Optional[str] @@ -350,6 +360,7 @@ class SHACLObjectSet: self, typ: Type[T_SHACLObject], *, match_subclass: bool = True ) -> Iterator[T_SHACLObject]: ... def merge(self, *objectsets: SHACLObjectSet) -> SHACLObjectSet: ... + def walk(self, callback: Callable[[Any, DataPath], bool]) -> None: ... def inline_blank_nodes(self) -> None: ... def expand_iri(self, iri: str, default: Optional[str] = None) -> Optional[str]: ... def compact_iri(self, iri: str, default: Optional[str] = None) -> Optional[str]: ... diff --git a/tests/test_python.py b/tests/test_python.py index fae51da3..bb1b1364 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -40,6 +40,10 @@ MODEL_VERSION = "1.0.0.alpha" +VALIDATION_ERROR = object() + +ENCODE_ERROR = object() + def shacl2code_generate(args, python_args, outfile): p = subprocess.run( @@ -571,21 +575,39 @@ def test_links(filename, name, expect_tag, model, tmp_path, test_context_url): @pytest.mark.parametrize( - "filename,expect", + "filename,expect,match", [ - ("bad-object-type-inline.json", TypeError), - ("bad-object-type-ref-before.json", TypeError), - ("bad-object-type-ref-after.json", TypeError), + pytest.param( + "bad-object-type-inline.json", + VALIDATION_ERROR, + "Type test-class is not valid where", + id="Bad object type for property (inline)", + ), + pytest.param( + "bad-object-type-ref-before.json", + VALIDATION_ERROR, + "Value must be one of type: link_class, str. Got test_class", + id="Bad object type for property (linked by ID before)", + ), + pytest.param( + "bad-object-type-ref-after.json", + VALIDATION_ERROR, + "Value must be one of type: link_class, str. Got test_class", + id="Bad object type for property (linked by ID after)", + ), ], ) -def test_deserialize(filename, expect, model, test_context_url): +def test_deserialize(filename, expect, match, model, test_context_url): objset = model.SHACLObjectSet() deserializer = model.JSONLDDeserializer() with (DATA_DIR / "python" / filename).open("r") as f: d = json.loads(f.read().replace("@CONTEXT_URL@", test_context_url)) + if expect is VALIDATION_ERROR: + expect = model.ValidationError + if issubclass(expect, Exception): - with pytest.raises(expect): + with pytest.raises(expect, match=match): deserializer.deserialize_data(d, objset) else: deserializer.deserialize_data(d, objset) @@ -601,7 +623,7 @@ def test_node_kind_blank(model, test_context_url): ref = model.node_kind_blank() - with pytest.raises(ValueError): + with pytest.raises(model.ValidationError): ref._id = "http://example.com/name" # Blank node assignment is fine but not preserved when serializing @@ -685,11 +707,11 @@ def test_node_kind_iri(model, test_context_url, cls): ref = getattr(model, cls)() - with pytest.raises(ValueError): + with pytest.raises(model.ValidationError): ref._id = "_:blank" # serializing without an ID is not allowed - with pytest.raises(ValueError): + with pytest.raises(model.EncodeError): s.serialize_data(model.SHACLObjectSet([ref])) # Inlining not allowed @@ -862,26 +884,26 @@ def test_set_id_with_alias(model, cls): def type_tests(name, *typ): tests = [ - (name, None, TypeError), - (name, [], TypeError), - (name, object(), TypeError), - (name, lambda model: sum, TypeError), + (name, None, VALIDATION_ERROR), + (name, [], VALIDATION_ERROR), + (name, object(), VALIDATION_ERROR), + (name, lambda model: sum, VALIDATION_ERROR), ] if bool not in typ and int not in typ: - tests.append((name, True, TypeError)) - tests.append((name, False, TypeError)) + tests.append((name, True, VALIDATION_ERROR)) + tests.append((name, False, VALIDATION_ERROR)) if int not in typ: - tests.append((name, 1, TypeError)) + tests.append((name, 1, VALIDATION_ERROR)) if float not in typ: - tests.append((name, 1.0, TypeError)) + tests.append((name, 1.0, VALIDATION_ERROR)) if datetime not in typ: - tests.append((name, datetime(2024, 3, 11, 0, 0, 0), TypeError)), + tests.append((name, datetime(2024, 3, 11, 0, 0, 0), VALIDATION_ERROR)), if str not in typ: - tests.append((name, "foo", TypeError)) + tests.append((name, "foo", VALIDATION_ERROR)) return tests @@ -890,14 +912,14 @@ def type_tests(name, *typ): "prop,value,expect", [ # positive integer - ("test_class_positive_integer_prop", -1, ValueError), - ("test_class_positive_integer_prop", 0, ValueError), + ("test_class_positive_integer_prop", -1, VALIDATION_ERROR), + ("test_class_positive_integer_prop", 0, VALIDATION_ERROR), ("test_class_positive_integer_prop", 1, 1), - ("test_class_positive_integer_prop", False, ValueError), + ("test_class_positive_integer_prop", False, VALIDATION_ERROR), ("test_class_positive_integer_prop", True, 1), *type_tests("test_class_positive_integer_prop", int), # non-negative integer - ("test_class_nonnegative_integer_prop", -1, ValueError), + ("test_class_nonnegative_integer_prop", -1, VALIDATION_ERROR), ("test_class_nonnegative_integer_prop", 0, 0), ("test_class_nonnegative_integer_prop", 1, 1), ("test_class_nonnegative_integer_prop", False, 0), @@ -1050,7 +1072,7 @@ def type_tests(name, *typ): "http://example.org/shacl2code-test/enumType/foo", "http://example.org/shacl2code-test/enumType/foo", ), - ("test_class_enum_prop", "foo", ValueError), + ("test_class_enum_prop", "foo", VALIDATION_ERROR), *type_tests("test_class_enum_prop", str), # Object ("test_class_class_prop", lambda model: model.test_class(), SAME_AS_VALUE), @@ -1059,8 +1081,12 @@ def type_tests(name, *typ): lambda model: model.test_derived_class(), SAME_AS_VALUE, ), - ("test_class_class_prop", lambda model: model.test_another_class(), TypeError), - ("test_class_class_prop", lambda model: model.parent_class(), TypeError), + ( + "test_class_class_prop", + lambda model: model.test_another_class(), + VALIDATION_ERROR, + ), + ("test_class_class_prop", lambda model: model.parent_class(), VALIDATION_ERROR), ("test_class_class_prop", lambda model: model.test_class.named, SAME_AS_VALUE), ("test_class_class_prop", "_:blanknode", "_:blanknode"), ( @@ -1073,9 +1099,9 @@ def type_tests(name, *typ): ("test_class_regex", "foo1", "foo1"), ("test_class_regex", "foo2", "foo2"), ("test_class_regex", "foo2a", "foo2a"), - ("test_class_regex", "bar", ValueError), - ("test_class_regex", "fooa", ValueError), - ("test_class_regex", "afoo1", ValueError), + ("test_class_regex", "bar", VALIDATION_ERROR), + ("test_class_regex", "fooa", VALIDATION_ERROR), + ("test_class_regex", "afoo1", VALIDATION_ERROR), *type_tests("test_class_regex", str), # Pattern validated dateTime ( @@ -1086,23 +1112,23 @@ def type_tests(name, *typ): ( "test_class_regex_datetime", datetime(2024, 3, 11, 0, 0, 0, tzinfo=timezone(-timedelta(hours=6))), - ValueError, + VALIDATION_ERROR, ), ( "test_class_regex_datetime", datetime(2024, 3, 11, 0, 0, 0, tzinfo=timezone.utc), - ValueError, + VALIDATION_ERROR, ), ( "test_class_regex_datetime", datetime(2024, 3, 11, 0, 0, 0), - ValueError, + VALIDATION_ERROR, ), # Pattern validated dateTimeStamp ( "test_class_regex_datetimestamp", datetime(2024, 3, 11, 0, 0, 0, tzinfo=timezone(-timedelta(hours=6))), - ValueError, + VALIDATION_ERROR, ), ( "test_class_regex_datetimestamp", @@ -1136,6 +1162,9 @@ def test_scalar_prop_validation(model, test_timezone, prop, value, expect): for cls in model.test_class, model.test_derived_class: c = cls() + if expect is VALIDATION_ERROR: + expect = model.ValidationError + if isinstance(expect, type) and issubclass(expect, Exception): with pytest.raises(expect): setattr(c, prop, value) @@ -1166,34 +1195,34 @@ def test_derived_property(model): def list_type_tests(name, *typ): tests = [ # Non list types - (name, 1, TypeError), - (name, 1.0, TypeError), - (name, True, TypeError), - (name, "foo", TypeError), - (name, datetime(2024, 3, 11, 0, 0, 0), TypeError), - (name, object(), TypeError), - (name, [object()], TypeError), - (name, lambda model: sum, TypeError), - (name, [sum], TypeError), + (name, 1, VALIDATION_ERROR), + (name, 1.0, VALIDATION_ERROR), + (name, True, VALIDATION_ERROR), + (name, "foo", VALIDATION_ERROR), + (name, datetime(2024, 3, 11, 0, 0, 0), VALIDATION_ERROR), + (name, object(), VALIDATION_ERROR), + (name, [object()], VALIDATION_ERROR), + (name, lambda model: sum, VALIDATION_ERROR), + (name, [sum], VALIDATION_ERROR), # Empty list is always allowed (name, [], []), ] if bool not in typ and int not in typ: - tests.append((name, [True], TypeError)) - tests.append((name, [False], TypeError)) + tests.append((name, [True], VALIDATION_ERROR)) + tests.append((name, [False], VALIDATION_ERROR)) if int not in typ: - tests.append((name, [1], TypeError)) + tests.append((name, [1], VALIDATION_ERROR)) if float not in typ: - tests.append((name, [1.0], TypeError)) + tests.append((name, [1.0], VALIDATION_ERROR)) if datetime not in typ: - tests.append((name, [datetime(2024, 3, 11, 0, 0, 0)], TypeError)), + tests.append((name, [datetime(2024, 3, 11, 0, 0, 0)], VALIDATION_ERROR)), if str not in typ: - tests.append((name, ["foo"], TypeError)) + tests.append((name, ["foo"], VALIDATION_ERROR)) return tests @@ -1280,7 +1309,7 @@ def list_type_tests(name, *typ): "http://example.org/shacl2code-test/enumType/foo", "foo", ], - ValueError, + VALIDATION_ERROR, ), *list_type_tests("test_class_enum_list_prop", str), # Object @@ -1298,12 +1327,12 @@ def list_type_tests(name, *typ): ( "test_class_class_list_prop", lambda model: [model.test_another_class()], - TypeError, + VALIDATION_ERROR, ), ( "test_class_class_list_prop", lambda model: [model.parent_class()], - TypeError, + VALIDATION_ERROR, ), *list_type_tests("test_class_class_list_prop", str), # Pattern validated @@ -1320,9 +1349,9 @@ def list_type_tests(name, *typ): "foo2a", ], ), - ("test_class_regex_list", ["bar"], ValueError), - ("test_class_regex_list", ["fooa"], ValueError), - ("test_class_regex_list", ["afoo1"], ValueError), + ("test_class_regex_list", ["bar"], VALIDATION_ERROR), + ("test_class_regex_list", ["fooa"], VALIDATION_ERROR), + ("test_class_regex_list", ["afoo1"], VALIDATION_ERROR), *list_type_tests("test_class_regex_list", str), # TODO Add more list tests ], @@ -1340,6 +1369,9 @@ def test_list_prop_validation(model, prop, value, expect): for cls in model.test_class, model.test_derived_class: c = cls() + if expect is VALIDATION_ERROR: + expect = model.ValidationError + if isinstance(expect, type) and issubclass(expect, Exception): with pytest.raises(expect): if value is list: @@ -1387,36 +1419,39 @@ def test_list_prop_validation(model, prop, value, expect): @timetests.datetime_decode_tests() def test_datetime_from_string(model, value, expect): p = model.DateTimeProp() + path = model.DataPath() if expect is None: - with pytest.raises(ValueError): - p.from_string(value) + with pytest.raises(model.ValidationError): + p.from_string(path, value) else: - v = p.from_string(value) + v = p.from_string(path, value) assert v == expect @timetests.datetimestamp_decode_tests() def test_datetimestamp_from_string(model, value, expect): p = model.DateTimeStampProp() + path = model.DataPath() if expect is None: - with pytest.raises(ValueError): - p.from_string(value) + with pytest.raises(model.ValidationError): + p.from_string(path, value) else: - v = p.from_string(value) + v = p.from_string(path, value) assert v == expect @timetests.datetime_encode_tests() def test_datetime_to_string(model, value, expect): p = model.DateTimeProp() + path = model.DataPath() if expect is None: - with pytest.raises(expect): - p.to_string(value) + with pytest.raises(model.ValidationError): + p.to_string(path, value) else: - v = p.to_string(value) + v = p.to_string(path, value) assert v == expect assert re.match( model.DateTimeProp.REGEX, v @@ -1499,7 +1534,7 @@ def test_extensible_prop(model, test_context_url, prop, serkey, value, expect): "http://example.org/shacl2code-test/extensible-test-prop", object(), SAME_AS_VALUE, - TypeError, + ENCODE_ERROR, None, ), ( @@ -1513,7 +1548,7 @@ def test_extensible_prop(model, test_context_url, prop, serkey, value, expect): "http://example.org/shacl2code-test/extensible-test-prop", [object()], SAME_AS_VALUE, - TypeError, + ENCODE_ERROR, None, ), ], @@ -1544,6 +1579,11 @@ def test_extensible_iri( objset = model.SHACLObjectSet() objset.add(e) + if ser_data is VALIDATION_ERROR: + ser_data = model.ValidationError + elif ser_data is ENCODE_ERROR: + ser_data = model.EncodeError + if isinstance(ser_data, type) and issubclass(ser_data, Exception): with pytest.raises(ser_data): data = s.serialize_data(objset) @@ -1609,7 +1649,7 @@ class OpenExtension(model.extensible_class): assert obj.get_type() == TEST_TYPE assert obj.get_compact_type() is None - with pytest.raises(KeyError): + with pytest.raises(model.DecodeError): deserialize_extension( { "@type": TEST_TYPE, @@ -1618,7 +1658,7 @@ class OpenExtension(model.extensible_class): } ) - with pytest.raises(KeyError): + with pytest.raises(model.DecodeError): deserialize_extension( { "@type": TEST_TYPE, @@ -1699,28 +1739,28 @@ def base_obj(): c = base_obj() del c.test_class_required_string_scalar_prop with outfile.open("wb") as f: - with pytest.raises(ValueError): + with pytest.raises(model.ValidationError): s.write(model.SHACLObjectSet([c]), f) # Array that is deleted c = base_obj() del c.test_class_required_string_list_prop with outfile.open("wb") as f: - with pytest.raises(ValueError): + with pytest.raises(model.ValidationError): s.write(model.SHACLObjectSet([c]), f) # Array initialized to empty list c = base_obj() c.test_class_required_string_list_prop = [] with outfile.open("wb") as f: - with pytest.raises(ValueError): + with pytest.raises(model.ValidationError): s.write(model.SHACLObjectSet([c]), f) # Array with too many items c = base_obj() c.test_class_required_string_list_prop.append("too many") with outfile.open("wb") as f: - with pytest.raises(ValueError): + with pytest.raises(model.ValidationError): s.write(model.SHACLObjectSet([c]), f) @@ -1976,7 +2016,7 @@ def test_required_abstract_class_property(model, tmp_path): # Attempting to serialize without assigning the property should fail with outfile.open("wb") as f: - with pytest.raises(ValueError): + with pytest.raises(model.ValidationError): s.write(objset, f, indent=4) # Assigning a concrete class should succeed and allow serialization From 954349c9692c1f53b19c5a8cb78ca6552291c476 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 12 Aug 2026 15:00:41 -0600 Subject: [PATCH 5/7] tests: jsonvalidation: Change unknown property to be valid IRI Unknown properties (in extensible classes) should still be valid IRIs. Test should be added to enforce that they are not blank nodes and that context is correctly expanded. --- testfixtures/testfixtures/jsonvalidation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testfixtures/testfixtures/jsonvalidation.py b/testfixtures/testfixtures/jsonvalidation.py index 4fc88049..fadbe7e4 100644 --- a/testfixtures/testfixtures/jsonvalidation.py +++ b/testfixtures/testfixtures/jsonvalidation.py @@ -572,7 +572,7 @@ def node_kind_tests(name, blank, iri): "@context": CONTEXT, "@type": "http://example.com/extended", "extensible-class/required": "foo", - "unknown-prop": "foo", + "http://unknown-prop": "foo", }, id="Extensible class with unknown property", ), @@ -581,7 +581,7 @@ def node_kind_tests(name, blank, iri): { "@context": CONTEXT, "@type": "http://example.com/extended", - "unknown-prop": "foo", + "http://unknown-prop": "foo", }, id="Extensible class with missing required property", ), @@ -629,7 +629,7 @@ def node_kind_tests(name, blank, iri): "link-class-link-prop": { "@type": "http://example.com/extended", "extensible-class/required": "foo", - "unknown-prop": "foo", + "http://unknown-prop": "foo", }, }, id="Nested extensible class with custom unknown property", From 693903a4fcef2014c47a990c1534133f816224d1 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Wed, 12 Aug 2026 14:21:20 -0600 Subject: [PATCH 6/7] tests: python: Run all json validation tests Now that Python conforms with the other code binding implementations, run the entire JSON validation test suite instead of just a subset --- tests/test_python.py | 55 +------------------------------------------- 1 file changed, 1 insertion(+), 54 deletions(-) diff --git a/tests/test_python.py b/tests/test_python.py index bb1b1364..250f781d 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -480,60 +480,7 @@ def test_jsonschema_validation(roundtrip, test_jsonschema): jsonschema.validate(data, schema=test_jsonschema) -# TODO: Make python bindings pass the other JSON validation tests -@pytest.mark.parametrize( - "passes,data", - [ - pytest.param( - True, - { - "@context": jsonvalidation.CONTEXT, - "@graph": [ - { - "@type": "test-class", - }, - ], - "signatures": [], - }, - id="JSS Signature", - ), - pytest.param( - False, - { - "@context": jsonvalidation.CONTEXT, - "@graph": [ - { - "@type": "test-class", - }, - ], - "unknown": {}, - }, - id="Unknown top level property with @graph", - ), - pytest.param( - True, - { - "@context": jsonvalidation.CONTEXT, - "@type": "test-class", - "signatures": [], - }, - id="Inline with signature", - ), - pytest.param( - False, - { - "@context": jsonvalidation.CONTEXT, - "@graph": [ - { - "@type": "test-class", - }, - ], - "signatures": "string", - }, - id="Signature with wrong type", - ), - ], -) +@jsonvalidation.validation_tests() def test_json_validation(passes, data, tmp_path, test_context_url, model_script): jsonvalidation.replace_context(data, test_context_url) From 447d8ca9ca3ae38379eff3a8b96b4600942fee12 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Mon, 17 Aug 2026 13:44:07 -0600 Subject: [PATCH 7/7] tests: python: Add stubtest tests Adds stubtest tests. This ensures that the .pyi files stay in sync with the .py files. Several API are fixed to conform, and an ignore list is provided for the rest (although these can be fixed at a later time). --- src/shacl2code/lang/python.py | 1 + .../lang/templates/python/cmd.pyi.j2 | 21 ++ .../lang/templates/python/model.py.j2 | 4 +- .../lang/templates/python/model.pyi.j2 | 259 +++++++++++++++--- tests/data/stubtest/allow.txt | 83 ++++++ tests/test_python.py | 30 ++ 6 files changed, 361 insertions(+), 37 deletions(-) create mode 100644 src/shacl2code/lang/templates/python/cmd.pyi.j2 create mode 100644 tests/data/stubtest/allow.txt diff --git a/src/shacl2code/lang/python.py b/src/shacl2code/lang/python.py index cad9f41d..ac26bf93 100644 --- a/src/shacl2code/lang/python.py +++ b/src/shacl2code/lang/python.py @@ -152,6 +152,7 @@ def get_file(name): if self.__include_main: yield get_file("cmd.py") + yield get_file("cmd.pyi") yield get_file("__main__.py") def get_extra_env(self): diff --git a/src/shacl2code/lang/templates/python/cmd.pyi.j2 b/src/shacl2code/lang/templates/python/cmd.pyi.j2 new file mode 100644 index 00000000..11293803 --- /dev/null +++ b/src/shacl2code/lang/templates/python/cmd.pyi.j2 @@ -0,0 +1,21 @@ +# {{ disclaimer }} +# SPDX-License-Identifier: {{ spdx_license }} + +import argparse +from pathlib import Path +from typing import Any + +from .model import ( + DataPath, + JSONLDDeserializer, + JSONLDSerializer, + ListProxy, + SHACLObject, + SHACLObjectSet, +) + + +def print_tree(objectset: SHACLObjectSet, all_fields: bool = False) -> None: ... + + +def main() -> int: ... diff --git a/src/shacl2code/lang/templates/python/model.py.j2 b/src/shacl2code/lang/templates/python/model.py.j2 index 3b8dc8bb..6724e49d 100644 --- a/src/shacl2code/lang/templates/python/model.py.j2 +++ b/src/shacl2code/lang/templates/python/model.py.j2 @@ -1016,7 +1016,7 @@ class SHACLObjectMeta(type): SHACLObject.CLASSES[key] = c -register_lock = threading.Lock() +_register_lock = threading.Lock() _ALL_NAMED_INDIVIDUAL_IDS: Set[str] = set() T_SHACLObject = TypeVar("T_SHACLObject", bound="SHACLObject") @@ -1119,7 +1119,7 @@ class SHACLObject(metaclass=SHACLObjectMeta): if self.ONTOLOGY: _warn_ontology(self.ONTOLOGY) - with register_lock: + with _register_lock: cls = self.__class__ if cls._NEEDS_REG: for p in cls._OBJ_PY_PROPS.values(): diff --git a/src/shacl2code/lang/templates/python/model.pyi.j2 b/src/shacl2code/lang/templates/python/model.pyi.j2 index cc843a37..66786d2c 100644 --- a/src/shacl2code/lang/templates/python/model.pyi.j2 +++ b/src/shacl2code/lang/templates/python/model.pyi.j2 @@ -41,22 +41,50 @@ class DataPath(object): def __getitem__(self, idx: int) -> str: ... +class PathError(Exception): + path: DataPath + message: str + + def __init__(self, path: DataPath, message: str) -> None: ... + def __str__(self) -> str: ... + + +class ValidationError(PathError): ... + + +class DecodeError(PathError): ... + + +class EncodeError(PathError): ... + + +def check_type( + path: DataPath, + obj: Any, + types: Union[Type[Any], Tuple[Type[Any], ...]], +) -> None: ... + + class Property(ABC, Generic[T_PropV]): VALID_TYPES: ClassVar[Union[Type[Any], Tuple[Type[Any], ...]]] pattern: Optional[str] def __init__(self, *, pattern: Optional[str] = None) -> None: ... def init(self) -> Optional[T_PropV]: ... - def validate(self, value: Any) -> None: ... + def validate(self, path: DataPath, value: Any) -> None: ... def set(self, value: Any) -> T_PropV: ... - def check_min_count(self, value: T_PropV, min_count: int) -> bool: ... - def check_max_count(self, value: T_PropV, max_count: int) -> bool: ... - def elide(self, value: T_PropV) -> bool: ... + def check_min_count( + self, path: DataPath, value: T_PropV, min_count: int + ) -> bool: ... + def check_max_count( + self, path: DataPath, value: T_PropV, max_count: int + ) -> bool: ... + def elide(self, path: DataPath, value: T_PropV) -> bool: ... def walk( self, value: T_PropV, - callback: Callable[[Any, List[str]], bool], - path: List[str], + callback: Callable[[Any, DataPath], bool], + path: DataPath, ) -> None: ... def iter_objects( self, value: Optional[T_PropV], recursive: bool, visited: Set[SHACLObject] @@ -67,15 +95,34 @@ class Property(ABC, Generic[T_PropV]): objectset: SHACLObjectSet, missing: Optional[Set[str]], visited: Set[SHACLObject], + path: DataPath, + ) -> Optional[T_PropV]: ... + def to_string(self, path: DataPath, value: T_PropV) -> str: ... + def encode( + self, encoder: Encoder, value: T_PropV, state: EncodeState, path: DataPath + ) -> None: ... + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath ) -> Optional[T_PropV]: ... - def to_string(self, value: T_PropV) -> str: ... class StringProp(Property[str]): def set(self, value: Any) -> str: ... + def encode( + self, encoder: Encoder, value: str, state: EncodeState, path: DataPath + ) -> None: ... + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Optional[str]: ... -class AnyURIProp(StringProp): ... +class AnyURIProp(StringProp): + def encode( + self, encoder: Encoder, value: str, state: EncodeState, path: DataPath + ) -> None: ... + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Optional[str]: ... class DateTimeProp(Property[datetime]): @@ -83,8 +130,14 @@ class DateTimeProp(Property[datetime]): REGEX: ClassVar[str] def set(self, value: datetime) -> datetime: ... - def to_string(self, value: datetime) -> str: ... - def from_string(self, value: str) -> datetime: ... + def encode( + self, encoder: Encoder, value: datetime, state: EncodeState, path: DataPath + ) -> None: ... + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Optional[datetime]: ... + def to_string(self, path: DataPath, value: datetime) -> str: ... + def from_string(self, path: DataPath, value: str) -> datetime: ... class DateTimeStampProp(DateTimeProp): @@ -93,22 +146,44 @@ class DateTimeStampProp(DateTimeProp): class IntegerProp(Property[int]): def set(self, value: Any) -> int: ... + def encode( + self, encoder: Encoder, value: int, state: EncodeState, path: DataPath + ) -> None: ... + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Optional[int]: ... class PositiveIntegerProp(IntegerProp): - def validate(self, value: Any) -> None: ... + def validate(self, path: DataPath, value: Any) -> None: ... class NonNegativeIntegerProp(IntegerProp): - def validate(self, value: Any) -> None: ... + def validate(self, path: DataPath, value: Any) -> None: ... class BooleanProp(Property[bool]): def set(self, value: Any) -> bool: ... + def encode( + self, encoder: Encoder, value: bool, state: EncodeState, path: DataPath + ) -> None: ... + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Optional[bool]: ... class FloatProp(Property[float]): def set(self, value: Any) -> float: ... + def encode( + self, + encoder: Encoder, + value: Union[float, int], + state: EncodeState, + path: DataPath, + ) -> None: ... + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Optional[float]: ... class IRIProp(Property[T_PropV]): @@ -135,13 +210,13 @@ class ObjectProp(IRIProp[Union[str, "SHACLObject"]]): required: bool, context: Optional[Tuple[Tuple[str, str], ...]] = None, ) -> None: ... - def validate(self, value: Any) -> None: ... + def validate(self, path: DataPath, value: Any) -> None: ... def set(self, value: Any) -> Union[str, SHACLObject]: ... def walk( self, value: Optional[Union[str, SHACLObject]], - callback: Callable[[Any, List[str]], bool], - path: List[str], + callback: Callable[[Any, DataPath], bool], + path: DataPath, ) -> None: ... def iter_objects( self, @@ -149,12 +224,23 @@ class ObjectProp(IRIProp[Union[str, "SHACLObject"]]): recursive: bool, visited: Set[SHACLObject], ) -> Iterable[SHACLObject]: ... + def encode( + self, + encoder: Encoder, + value: Union[str, SHACLObject], + state: EncodeState, + path: DataPath, + ) -> None: ... + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath + ) -> Union[str, SHACLObject]: ... def link_prop( self, value: Optional[Union[str, SHACLObject]], objectset: SHACLObjectSet, missing: Optional[Set[str]], visited: Set[SHACLObject], + path: DataPath, ) -> Optional[Union[str, SHACLObject]]: ... @@ -186,18 +272,22 @@ class ListProp(Property[ListProxy[T_PropV]]): def __init__(self, prop: Property[T_PropV]) -> None: ... def init(self) -> ListProxy[T_PropV]: ... - def validate(self, value: Any) -> None: ... + def validate(self, path: DataPath, value: Any) -> None: ... def set( self, value: Union[ListProxy[T_PropV], Iterable[T_PropV]] ) -> ListProxy[T_PropV]: ... - def check_min_count(self, value: ListProxy[T_PropV], min_count: int) -> bool: ... - def check_max_count(self, value: ListProxy[T_PropV], max_count: int) -> bool: ... - def elide(self, value: ListProxy[T_PropV]) -> bool: ... + def check_min_count( + self, path: DataPath, value: ListProxy[T_PropV], min_count: int + ) -> bool: ... + def check_max_count( + self, path: DataPath, value: ListProxy[T_PropV], max_count: int + ) -> bool: ... + def elide(self, path: DataPath, value: ListProxy[T_PropV]) -> bool: ... def walk( self, value: ListProxy[T_PropV], - callback: Callable[[Any, List[str]], bool], - path: List[str], + callback: Callable[[Any, DataPath], bool], + path: DataPath, ) -> None: ... def iter_objects( self, @@ -211,6 +301,17 @@ class ListProp(Property[ListProxy[T_PropV]]): objectset: SHACLObjectSet, missing: Optional[Set[str]], visited: Set[SHACLObject], + path: DataPath, + ) -> ListProxy[T_PropV]: ... + def encode( + self, + encoder: Encoder, + value: ListProxy[T_PropV], + state: EncodeState, + path: DataPath, + ) -> None: ... + def decode( + self, decoder: Decoder, state: DecodeState, path: DataPath ) -> ListProxy[T_PropV]: ... @@ -221,7 +322,11 @@ class EnumProp(IRIProp[str]): *, pattern: Optional[str] = None, ) -> None: ... - def validate(self, value: Any) -> None: ... + def validate(self, path: DataPath, value: Any) -> None: ... + def encode( + self, encoder: Encoder, value: str, state: EncodeState, path: DataPath + ) -> None: ... + def decode(self, decoder: Decoder, state: DecodeState, path: DataPath) -> str: ... class NodeKind(Enum): @@ -234,10 +339,19 @@ def is_IRI(s: Any) -> bool: ... def is_blank_node(s: Any) -> bool: ... +# fmt: off +"""Format Guard{{ '"' }}{{ '"' }}{{ '"' }} +{% if version_str %}VERSION_STRING: str{% endif %} +{% if version %}VERSION: tuple[int, ...]{% endif %} +{{ '"' }}{{ '"' }}{{ '"' }}Format Guard""" +# fmt: on + + @dataclass class Ontology: iri: str version: Optional[str] = ... + is_prerelease: bool = ... @dataclass @@ -249,24 +363,88 @@ class ClassProp: max_count: Optional[int] = ... compact: Optional[str] = ... deprecated: bool = ... + prop: Optional[Property[Any]] = ... -class EncodeState: ... +class EncodeState: + ref_objects: Set[SHACLObject] + written_objects: Set[SHACLObject] + blank_objects: Dict[SHACLObject, str] + objectset: SHACLObjectSet + def __init__(self, objectset: SHACLObjectSet) -> None: ... + def get_object_id(self, o: SHACLObject) -> str: ... + def is_refed(self, o: SHACLObject) -> bool: ... + def add_refed(self, o: SHACLObject) -> None: ... + def is_written(self, o: SHACLObject) -> bool: ... + def add_written(self, o: SHACLObject) -> None: ... + def expand_iri(self, iri: str, default: Optional[str] = None) -> Optional[str]: ... + def compact_iri(self, iri: str, default: Optional[str] = None) -> Optional[str]: ... -class DecodeState: ... +class DecodeState: + objectset: SHACLObjectSet + read_objs: Dict[str, SHACLObject] -class Encoder(ABC): ... + def __init__(self, objectset: SHACLObjectSet) -> None: ... + def expand_iri(self, iri: str, default: Optional[str] = None) -> Optional[str]: ... + def compact_iri(self, iri: str, default: Optional[str] = None) -> Optional[str]: ... -class Decoder(ABC): ... +class Encoder(ABC): + def write_string(self, v: str) -> None: ... + def write_datetime(self, v: str) -> None: ... + def write_integer(self, v: int) -> None: ... + def write_iri(self, v: str, compact: Optional[str] = None) -> None: ... + def write_enum( + self, v: str, e: Property[Any], compact: Optional[str] = None + ) -> None: ... + def write_bool(self, v: bool) -> None: ... + def write_float(self, v: float) -> None: ... + def write_object( + self, + typ: str, + compact_type: Optional[str], + id_alias: Optional[str], + _id: str, + compact_id: Optional[str], + needs_id: bool, + ) -> Iterator[Encoder]: ... + def write_property( + self, iri: str, compact: Optional[str] = None + ) -> Iterator[Encoder]: ... + def write_list(self) -> Iterator[Encoder]: ... + def write_list_item(self) -> Iterator[Encoder]: ... + def write_object_list(self) -> Iterator[Encoder]: ... + def write_dict(self) -> Iterator[Encoder]: ... + + +class Decoder(ABC): + def read_value(self) -> Any: ... + def read_string(self) -> Optional[str]: ... + def read_datetime(self) -> Optional[str]: ... + def read_integer(self) -> Optional[int]: ... + def read_iri(self) -> Optional[str]: ... + def read_enum(self, e: EnumProp) -> Optional[str]: ... + def read_bool(self) -> Optional[bool]: ... + def read_float(self) -> Optional[float]: ... + def read_list(self) -> Iterator[Decoder]: ... + def is_list(self) -> bool: ... + def read_object(self) -> Tuple[Any, Decoder]: ... + def read_property(self, key: str) -> Iterator[Optional[Decoder]]: ... + def is_object(self) -> bool: ... + def object_keys(self) -> Iterator[str]: ... + def read_object_id(self, alias: Optional[str] = None) -> Optional[Any]: ... + + +class SHACLObjectMeta(type): + def __new__(cls, name, bases, attrs): ... T_SHACLObject = TypeVar("T_SHACLObject", bound="SHACLObject") -class SHACLObject: +class SHACLObject(metaclass=SHACLObjectMeta): CLASSES: ClassVar[Dict[str, Type[SHACLObject]]] NAMED_INDIVIDUALS: ClassVar[Dict[str, str]] AUTO_NAMED_INDIVIDUALS: ClassVar[bool] @@ -290,18 +468,21 @@ class SHACLObject: def __iter__(self) -> Iterator[str]: ... def walk( self, - callback: Callable[[Any, List[str]], bool], - path: Optional[List[str]] = None, + callback: Callable[[Any, DataPath], bool], + path: DataPath, ) -> None: ... def property_keys(self) -> Iterator[Tuple[Optional[str], str, Optional[str]]]: ... def iter_objects( self, *, recursive: bool = False, visited: Optional[Set[SHACLObject]] = None ) -> Iterable[SHACLObject]: ... - def encode(self, encoder: "Encoder", state: "EncodeState") -> None: ... + def encode( + self, encoder: "Encoder", state: "EncodeState", path: DataPath + ) -> None: ... @staticmethod def decode( decoder: Decoder, state: DecodeState, + path: DataPath, target_type: Optional[Type["SHACLObject"]] = None, ) -> "SHACLObject": ... def link_helper( @@ -309,13 +490,11 @@ class SHACLObject: objectset: SHACLObjectSet, missing: Optional[Set[str]], visited: Set[SHACLObject], + path: DataPath, ) -> None: ... def __hash__(self) -> int: ... def __eq__(self, other: Any) -> bool: ... def __lt__(self, other: Any) -> bool: ... - def __gt__(self, other: Any) -> bool: ... - def __le__(self, other: Any) -> bool: ... - def __ge__(self, other: Any) -> bool: ... class SHACLExtensibleObject(SHACLObject): @@ -365,12 +544,17 @@ class SHACLObjectSet: def expand_iri(self, iri: str, default: Optional[str] = None) -> Optional[str]: ... def compact_iri(self, iri: str, default: Optional[str] = None) -> Optional[str]: ... def encode( - self, encoder: "Encoder", state: "EncodeState", force_at_graph: bool = False + self, + encoder: "Encoder", + state: "EncodeState", + force_list: bool = False, + *, + key: Optional[Callable[[SHACLObject], Any]] = None, ) -> None: ... def decode( self, decoder: "Decoder", - state: Optional["DecodeState"] = None, + state: "DecodeState", ) -> None: ... @@ -408,6 +592,11 @@ def decode_context(decoder: Any, objectset: SHACLObjectSet) -> List[str]: ... # fmt: off """Format Guard{{ '"' }}{{ '"' }}{{ '"' }} +# ONTOLOGIES +{%- for o in ontologies %} +{{ varname(o.name).upper() }}: Ontology +{%- endfor %} + # CLASSES {%- for class in classes %} class {{ varname(*class.clsname) }}( diff --git a/tests/data/stubtest/allow.txt b/tests/data/stubtest/allow.txt new file mode 100644 index 00000000..8e1b73b7 --- /dev/null +++ b/tests/data/stubtest/allow.txt @@ -0,0 +1,83 @@ +pymodel.CONTEXT_URLS +pymodel.TYPE_CHECKING +pymodel.VERSION +pymodel.VERSION_STRING +pymodel.model.CONTEXT_URLS +pymodel.model.ClassProp.__init__ +pymodel.model.Decoder.is_list +pymodel.model.Decoder.is_object +pymodel.model.Decoder.object_keys +pymodel.model.Decoder.read_bool +pymodel.model.Decoder.read_datetime +pymodel.model.Decoder.read_enum +pymodel.model.Decoder.read_float +pymodel.model.Decoder.read_integer +pymodel.model.Decoder.read_iri +pymodel.model.Decoder.read_list +pymodel.model.Decoder.read_object +pymodel.model.Decoder.read_object_id +pymodel.model.Decoder.read_property +pymodel.model.Decoder.read_string +pymodel.model.Decoder.read_value +pymodel.model.Encoder.write_bool +pymodel.model.Encoder.write_datetime +pymodel.model.Encoder.write_dict +pymodel.model.Encoder.write_enum +pymodel.model.Encoder.write_float +pymodel.model.Encoder.write_integer +pymodel.model.Encoder.write_iri +pymodel.model.Encoder.write_list +pymodel.model.Encoder.write_list_item +pymodel.model.Encoder.write_object +pymodel.model.Encoder.write_object_list +pymodel.model.Encoder.write_property +pymodel.model.Encoder.write_string +pymodel.model.JSONLDDecoder +pymodel.model.JSONLDEncoder +pymodel.model.JSONLDInlineEncoder +pymodel.Property.decode +pymodel.model.Property.decode +pymodel.Property.encode +pymodel.model.Property.encode +pymodel.model.RDFDecoder +pymodel.model.RDFDeserializer +pymodel.model.RDFEncoder +pymodel.model.RDFSerializer +pymodel.model.SHACLObject.__ge__ +pymodel.model.SHACLObject.__gt__ +pymodel.model.SHACLObject.__le__ +pymodel.cmd.SHACLObject.__ge__ +pymodel.cmd.SHACLObject.__gt__ +pymodel.cmd.SHACLObject.__le__ +pymodel.SHACLObject.__ge__ +pymodel.SHACLObject.__gt__ +pymodel.SHACLObject.__le__ +pymodel.Decoder.is_list +pymodel.Decoder.is_object +pymodel.Decoder.object_keys +pymodel.Decoder.read_bool +pymodel.Decoder.read_datetime +pymodel.Decoder.read_enum +pymodel.Decoder.read_float +pymodel.Decoder.read_integer +pymodel.Decoder.read_iri +pymodel.Decoder.read_list +pymodel.Decoder.read_object +pymodel.Decoder.read_object_id +pymodel.Decoder.read_property +pymodel.Decoder.read_string +pymodel.Decoder.read_value +pymodel.Encoder.write_bool +pymodel.Encoder.write_datetime +pymodel.Encoder.write_dict +pymodel.Encoder.write_enum +pymodel.Encoder.write_float +pymodel.Encoder.write_integer +pymodel.Encoder.write_iri +pymodel.Encoder.write_list +pymodel.Encoder.write_list_item +pymodel.Encoder.write_object +pymodel.Encoder.write_object_list +pymodel.Encoder.write_property +pymodel.Encoder.write_string + diff --git a/tests/test_python.py b/tests/test_python.py index 250f781d..a4c54b77 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -225,6 +225,36 @@ def test_mypy(self, tmp_path, args, python_args): check=True, ) + def test_stubtest(self, tmp_path, args, python_args): + """ + Mypy stub checks to ensure pyi stubs are in sync with py code + """ + output_dir = tmp_path / "pymodel" + shacl2code_generate(args, python_args, output_dir) + + pythonpath = os.environ.get("PYTHONPATH") + if pythonpath: + pythonpath = os.pathsep.join(str(tmp_path), pythonpath) + else: + pythonpath = str(tmp_path) + + env = os.environ.copy() + env["PYTHONPATH"] = pythonpath + + subprocess.run( + [ + "stubtest", + "pymodel", + "--allow", + DATA_DIR / "stubtest" / "allow.txt", + "--ignore-unused-allowlist", + "--ignore-missing-stub", + ], + encoding="utf-8", + check=True, + env=env, + ) + def test_pyrefly(self, tmp_path, args, python_args): """ Pyrefly static type checking