diff --git a/pyproject.toml b/pyproject.toml index 0e3df18..b20ffaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,6 +90,41 @@ line-length = 120 exclude = [ "tests/od/**", ] +lint.ignore = [ + # Stop gap for getting ruff checks to pass. These should be fixed and removed in the future. + "B015", # Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it. + "B017", # Do not assert blind exception: `Exception` + "BLE001", # Do not catch blind exception: `Exception` + "C400", # Unnecessary generator (rewrite as a list comprehension) + "C401", # Unnecessary generator (rewrite as a set comprehension) + "C408", # Unnecessary `tuple()` call (rewrite as a literal) + "C413", # Unnecessary `list()` call around `sorted()` + "FURB122", # Use of `fo.write` in a for loop + "FURB167", # Use of regular expression alias `re.M` + "I001", # Import block is un-sorted or un-formatted + "PIE810", # Call `startswith` once with a `tuple` + "PLR1711", # Useless `return` statement at end of function + "PLR1736", # List index lookup in `enumerate()` loop + "PLW0127", # Self-assignment of variable + "PLW1510", # `subprocess.run` without explicit `check` argument + "PYI034", # `__iadd__`/`__imod__` methods usually return `self` at runtime + "PYI041", # Use `complex` instead of `int | float | complex` + "RET501", # Do not explicitly `return None` in function if it is the only possible return value + "RUF010", # Use explicit conversion flag + "RUF012", # Mutable default value for class attribute + "RUF059", # Unpacked variable `editors` is never used + "RUF100", # Unused `noqa` directive + "S102", # Use of `exec` detected + "SIM102", # Use a single `if` statement instead of nested `if` statements + "SIM114", # Combine `if` branches using logical `or` operator + "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements + "SIM202", # Use `x == y` instead of `not x != y` + "TRY004", # Prefer `TypeError` exception for invalid type + "UP031", # Use format specifiers instead of percent format + "UP034", # Avoid extraneous parentheses + "UP035", # Import from `collections.abc` instead + "UP037", # Remove quotes from type annotation +] [tool.ruff.lint.per-file-ignores] "tests/*" = [ diff --git a/src/objdictgen/jsonod.py b/src/objdictgen/jsonod.py index dd4ea47..4eb0a9f 100644 --- a/src/objdictgen/jsonod.py +++ b/src/objdictgen/jsonod.py @@ -120,6 +120,7 @@ class ValidationError(Exception): # node.ParamDictionary[index] = { N: { ..dict..}, ..dict.. } FIELDS_PARAMS = {'comment', 'save', 'buffer_size'} FIELDS_PARAMS_PROMOTE = {'callback'} +FIELDS_PARAMS_EACH = {"start_index"} # Fields representing the dictionary value FIELDS_VALUE = {'value'} @@ -421,7 +422,7 @@ def generate_node(contents: str|TODJson, validate: bool = True) -> Node: if validate: validate_fromdict(jd, objtypes_i2s, objtypes_s2i) - return node_fromdict(jd, objtypes_s2i) + return node_fromdict(jd, objtypes_s2i, validate=validate) def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True) -> TODJson: @@ -484,7 +485,6 @@ def node_todict(node: Node, sort=False, rich=True, internal=False, validate=True finally: # Add in a fancyer index (do it here after index is finished being used) if rich: - index = obj["index"] obj["index"] = f'@@"0x{index:04X}" // {index}@@' dictionary.append(obj) @@ -598,9 +598,12 @@ def indexentry_to_jsondict(ientry: TIndexEntry) -> TODObjJson: obj['struct'] = struct obj['sub'] = obj.pop('values', []) # type: ignore[typeddict-item] # values is about to be renamed - # Move subindex[1] to 'each' on objecs that contain 'nbmax' - if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][1]: - obj['each'] = obj['sub'].pop(1) # type: ignore[typeddict-item] + # Move subindex[-1] to 'each' on objecs that contain 'nbmax' + if len(obj['sub']) > 1 and 'nbmax' in obj['sub'][-1]: + obj['each'] = obj['sub'].pop(-1) # type: ignore[typeddict-item] + start_index = len(obj["sub"]) + if start_index > 1: + obj["each"]["start_index"] = start_index # Baseobj should have been emptied if odobj != {}: @@ -892,20 +895,18 @@ def validate_indexentry(ientry: TIndexEntry): nbmaxok = True elif struct in (OD.ARRAY, OD.NARRAY): - if len(nbmax) == 2: + if len(nbmax) == 2: # Array only have length + repeat lenok = True - if sum(nbmax) == 1 and nbmax[1]: + if sum(nbmax) == 1 and nbmax[-1]: nbmaxok = True elif struct in (OD.RECORD, OD.NRECORD): - if sum(nbmax) and len(nbmax) > 1 and nbmax[1]: + if len(nbmax) >= 2: # Record can have more than one item before repeat + lenok = True + if sum(nbmax) == 1 and nbmax[-1]: nbmaxok = True - if len(nbmax) == 2: - lenok = True - elif sum(nbmax) == 0: + if sum(nbmax) == 0: nbmaxok = True - if len(nbmax) > 1: - lenok = True else: raise ValidationError(f"Unknown struct '{struct}'") @@ -915,7 +916,7 @@ def validate_indexentry(ientry: TIndexEntry): raise ValidationError(f"Unexpexted count of subindexes in mapping object, found {len(nbmax)}") -def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node: +def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int], validate: bool = True) -> Node: """ Convert a dict jd into a Node """ # Create the node and fill the most basic data @@ -983,15 +984,16 @@ def node_fromdict(jd: TODJson, objtypes_s2i: dict[str, int]) -> Node: elif 'built-in' in groups: refobj = maps.MAPPING_DICTIONARY.get(index) - diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree') - if diff: - log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index) - for line in diff.pretty().splitlines(): - log.debug(' %s', line) - raise ValidationError( - f"Built-in object index 0x{index:04x} ({index}) " - "does not match against system parameters" - ) + if validate: + diff = deepdiff.DeepDiff(refobj, ientry['object'], view='tree') + if diff: + log.debug("Index 0x%04x (%s) Difference between built-in object and imported:", index, index) + for line in diff.pretty().splitlines(): + log.debug(' %s', line) + raise ValidationError( + f"Built-in object index 0x{index:04x} ({index}) " + "does not match against system parameters" + ) return node @@ -1093,7 +1095,9 @@ def rearrange_for_node(obj: TODObjJson, objtypes_s2i: dict[str, int]) -> TIndexE # Move back the each object if 'each' in obj: - subitems.append(obj.pop('each')) # type: ignore[arg-type] + each = obj.pop('each') + each.pop("start_index", None) # Remove the start_index if present + subitems.append(each) # type: ignore[arg-type] # Check if the object is a repeat object repeat = obj.pop('repeat', False) @@ -1173,7 +1177,7 @@ def validate_fromdict(jsonobj: TODJson, objtypes_i2s: dict[int, str], objtypes_s # Verify that we have the expected members member_compare(jsonobj.keys(), must=FIELDS_DATA_MUST, optional=FIELDS_DATA_OPT) - def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False): + def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False, is_index_each=False): # Validated: (See FIELDS_MAPVAPS_*, FIELDS_PARAMS and FIELDS_VALUE) # ---------- @@ -1209,7 +1213,7 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False): # Set what parameters should be present, optional or not present if idx == -1: # Checking "each" section. No object or value - params = 'no' + params = "each" elif is_repeat: # Object repeat = defined elsewhere. No definition needed. defs = 'no' @@ -1219,10 +1223,9 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False): elif is_var: # VAR type, guaranteed idx==0 here value = 'opt' - elif is_each: # Param have "each". Should never have any defs in idx > 0 - if idx > 0: - defs = 'no' - value = 'must' + elif is_each and idx >= is_index_each: # Param have "each" and the index is in the "each" items + defs = 'no' + value = 'must' else: # All other (not each, not repeat, not VAR) if idx > 0: @@ -1240,6 +1243,8 @@ def _validate_sub(obj, idx=0, is_var=False, is_repeat=False, is_each=False): # must |= FIELDS_PARAMS if params == 'opt': opts |= FIELDS_PARAMS + if params == 'each': + opts |= FIELDS_PARAMS_EACH if value == 'must': must |= FIELDS_VALUE if value == 'opt': @@ -1335,7 +1340,10 @@ def _validate_dictionary(index, obj): for idx, sub in enumerate(subitems): try: is_var = struct in (OD.VAR, OD.NVAR) - _validate_sub(sub, idx, is_var=is_var, is_repeat=is_repeat, is_each='each' in obj) + _validate_sub( + sub, idx, is_var=is_var, is_repeat=is_repeat, is_each="each" in obj, + is_index_each=obj.get("each", {}).get("start_index", 1), + ) except Exception as exc: exc_amend(exc, f"sub[{idx}]: ") raise @@ -1347,9 +1355,11 @@ def _validate_dictionary(index, obj): if struct in (OD.VAR, OD.NVAR): raise ValidationError("Unexpected 'each' use in VAR/NVAR object") - # Having 'each' requires use of only one sub item with 'name' in it - if not (sum(has_name) == 1 and has_name[0]): - raise ValidationError("Unexpected subitems. Subitem 0 must contain name") + # When each is present, the other items must have a name field in them + if sum(has_name) != sub.get("start_index", 1): + raise ValidationError( + f"Unexpected subitems. Subitems 0..{sub['start_index'] - 1} must contain name" + ) try: _validate_sub(sub, idx=-1) diff --git a/src/objdictgen/schema/od.schema.json b/src/objdictgen/schema/od.schema.json index 1f13134..1974829 100644 --- a/src/objdictgen/schema/od.schema.json +++ b/src/objdictgen/schema/od.schema.json @@ -187,6 +187,7 @@ "pdo": { "$ref": "#pdo" }, "nbmin": { "$ref": "#nbmin" }, "nbmax": { "$ref": "#nbmax" }, + "start_index": { "$ref": "#index" }, "default": { "$ref": "#value" } }, "additionalProperties": false, @@ -305,6 +306,11 @@ "type": "integer" }, + "start_index": { + "$id": "#start_index", + "type": "integer" + }, + "pdo": { "$id": "#pdo", "description": "Set if object is usable in a PDO", @@ -411,4 +417,4 @@ } } -} \ No newline at end of file +} diff --git a/src/objdictgen/typing.py b/src/objdictgen/typing.py index 1a801f0..79fe4bb 100644 --- a/src/objdictgen/typing.py +++ b/src/objdictgen/typing.py @@ -198,6 +198,7 @@ def GetDefaultStringSize(self) -> int: "pdo": bool, "nbmin": int, "nbmax": int, + "start_index": int, "default": TODValue, }) """JSON object dictionary "each" type definition.""" diff --git a/tests/conftest.py b/tests/conftest.py index 22c35c9..d105f2e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -100,7 +100,7 @@ class Fn: def diff(a, b, predicate=None, postprocess=None, **kw): """ Diff two files """ if predicate is None: - predicate = lambda x: True # noqa: E731 + predicate = lambda x: True with open(a, 'r', encoding="utf-8") as f: da = [n.rstrip() for n in f if predicate(n)] with open(b, 'r', encoding="utf-8") as f: diff --git a/tests/od/odtest.jsonc b/tests/od/odtest.jsonc new file mode 100644 index 0000000..ebc7832 --- /dev/null +++ b/tests/od/odtest.jsonc @@ -0,0 +1,229 @@ +{ + "$id": "od data", + "$version": "1", + "$description": "Canfestival object dictionary data", + "$schema": "https://raw.githubusercontent.com/Laerdal/python-objdictgen/main/src/objdictgen/schema/od.schema.json", + "$tool": "odg 3.5.5", + "$date": "2026-08-19T16:04:25.134652+02:00", + "name": "odtest", + "description": "OD for testing object types with repeated subindexes and repeated objects.", + "type": "master", + "id": 0, + "profile": "odtest", + "dictionary": [ + { + "index": "0x1002", // 4098 + "name": "Manufacturer Status Register", + "struct": "var", + "group": "built-in", + "mandatory": false, + "sub": [ + { + "name": "Manufacturer Status Register", + "type": "UNSIGNED32", // 7 + "access": "ro", + "pdo": true, + "value": "{True:\"$NODEID+0x%X00\"%(base+2),False:0x80000000}[base<4]", + "comment": "Tests the python expression in the value field." + } + ] + }, + { + "index": "0x1003", // 4099 + "name": "Pre-defined Error Field", + "struct": "array", + "group": "built-in", + "mandatory": false, + "profile_callback": true, + "each": { + "name": "Standard Error Field", + "type": "UNSIGNED32", // 7 + "access": "ro", + "pdo": false, + "nbmin": 1, + "nbmax": 254 + }, + "sub": [ + { + "name": "Number of Errors", + "type": "UNSIGNED8", // 5 + "access": "rw", + "pdo": false, + "comment": "Tests the repeated subobject in ARRAY via each." + }, + { + // "name": "Standard Error Field" + // "type": "UNSIGNED32" // 7 + "value": 1 + }, + { + // "name": "Standard Error Field" + // "type": "UNSIGNED32" // 7 + "value": 2 + }, + { + // "name": "Standard Error Field" + // "type": "UNSIGNED32" // 7 + "value": 3 + } + ] + }, + { + "index": "0x2000", // 8192 + "name": "Example record", + "struct": "record", + "group": "profile", + "mandatory": false, + "each": { + "name": "Restore Manufacturer Defined Default Parameters %d[(sub - 3)]", + "type": "UNSIGNED32", // 7 + "access": "rw", + "pdo": false, + "nbmin": 0, + "nbmax": 124, + "start_index": 3 + }, + "sub": [ + { + "name": "Number of Entries", + "type": "UNSIGNED8", // 5 + "access": "ro", + "pdo": false, + "comment": "Tests repeated subobjects in a RECORD where only the two last items are repeated. The first two items are not repeated and should be copied to the repeated subobjects." + }, + { + "name": "Restore Communication Default Parameters", + "type": "UNSIGNED32", // 7 + "access": "rw", + "pdo": false, + "value": 0 + }, + { + "name": "Restore Application Default Parameters", + "type": "UNSIGNED32", // 7 + "access": "rw", + "pdo": false, + "value": 0 + }, + { + // "name": "Restore Manufacturer Defined Default Parameters 0" + // "type": "UNSIGNED32" // 7 + "value": 1 + }, + { + // "name": "Restore Manufacturer Defined Default Parameters 1" + // "type": "UNSIGNED32" // 7 + "value": 2 + } + ] + }, + { + "index": "0x2001", // 8193 + "name": "Example array", + "struct": "array", + "group": "profile", + "mandatory": false, + "each": { + "name": "Consumer Heartbeat Time", + "type": "UNSIGNED32", // 7 + "access": "rw", + "pdo": false, + "nbmin": 0, + "nbmax": 127 + }, + "sub": [ + { + "name": "Number of Entries", + "type": "UNSIGNED8", // 5 + "access": "ro", + "pdo": false, + "comment": "Test of an empty ARRAY where each is used." + } + ] + }, + { + "index": "0x2010", // 8208 + "name": "Mymap %d [(idx)]", + "struct": "narray", + "group": "profile", + "mandatory": false, + "incr": 1, + "nbmax": 512, + "each": { + "name": "Mymap %d entry %d[(idx,sub)]", + "type": "UNSIGNED32", // 7 + "access": "rw", + "pdo": false, + "nbmin": 0, + "nbmax": 64 + }, + "sub": [ + { + "name": "Number of Entries", + "type": "UNSIGNED8", // 5 + "access": "rw", + "pdo": false, + "comment": "Base object with repeated subobjects and with comments that should not be copied to the next" + }, + { + // "name": "Mymap 1 entry 1" + // "type": "UNSIGNED32" // 7 + "comment": "Item 1", + "value": 100 + }, + { + // "name": "Mymap 1 entry 2" + // "type": "UNSIGNED32" // 7 + "comment": "Item 2", + "value": 101 + }, + { + // "name": "Mymap 1 entry 3" + // "type": "UNSIGNED32" // 7 + "value": 102 + } + ] + }, + { + "index": "0x2011", // 8209 + // "name": "Mymap 2 " + "repeat": true, + "struct": "narray", + "sub": [ + { + // "name": "Number of Entries" + // "type": "UNSIGNED8" // 5 + "comment": "Repeated object with repeated subobject and with more items than the base. The next object is also repeated, but with no contents." + }, + { + // "name": "Mymap 2 entry 1" + // "type": "UNSIGNED32" // 7 + "value": 32 + }, + { + // "name": "Mymap 2 entry 2" + // "type": "UNSIGNED32" // 7 + "comment": "My item 2", + "value": 33 + }, + { + // "name": "Mymap 2 entry 3" + // "type": "UNSIGNED32" // 7 + "value": 34 + }, + { + // "name": "Mymap 2 entry 4" + // "type": "UNSIGNED32" // 7 + "value": 35 + } + ] + }, + { + "index": "0x2012", // 8210 + // "name": "Mymap 3 " + "repeat": true, + "struct": "narray", + "sub": [] + } + ] +} \ No newline at end of file diff --git a/tests/od/odtest.od b/tests/od/odtest.od new file mode 100644 index 0000000..93ef338 --- /dev/null +++ b/tests/od/odtest.od @@ -0,0 +1,417 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/od/odtest.prf b/tests/od/odtest.prf new file mode 100644 index 0000000..3e2a372 --- /dev/null +++ b/tests/od/odtest.prf @@ -0,0 +1,19 @@ +global Mapping, AddMenuEntries + +Mapping = { + 0x2000: {"name": "Example record", "struct": OD.RECORD, "need": False, "values": [ + {"name": "Number of Entries", "type": 0x05, "access": 'ro', "pdo": False}, + {"name": "Restore Communication Default Parameters", "type": 0x07, "access": 'rw', "pdo": False}, + {"name": "Restore Application Default Parameters", "type": 0x07, "access": 'rw', "pdo": False}, + {"name": "Restore Manufacturer Defined Default Parameters %d[(sub - 3)]", "type": 0x07, "access": 'rw', "pdo": False, "nbmin": 0, "nbmax": 0x7C}]}, + + 0x2001: {"name": "Example array", "struct": OD.ARRAY, "need": False, "values": [ + {"name": "Number of Entries", "type": 0x05, "access": 'ro', "pdo": False}, + {"name": "Consumer Heartbeat Time", "type": 0x07, "access": 'rw', "pdo": False, "nbmin": 0, "nbmax": 0x7F}]}, + + 0x2010: {"name": "Mymap %d [(idx)]", "struct": OD.NARRAY, "incr": 1, "nbmax": 0x200, "need": False, "values": [ + {"name": "Number of Entries", "type": 0x05, "access": 'rw', "pdo": False}, + {"name": "Mymap %d entry %d[(idx,sub)]", "type": 0x07, "access": 'rw', "pdo": False, "nbmin": 0, "nbmax": 0x40}]}, +} + +AddMenuEntries = []