diff --git a/SystemReady-band/build-scripts/build-buildroot.sh b/SystemReady-band/build-scripts/build-buildroot.sh index bfdc295a..7909dfc4 100755 --- a/SystemReady-band/build-scripts/build-buildroot.sh +++ b/SystemReady-band/build-scripts/build-buildroot.sh @@ -65,12 +65,6 @@ do_build () cp -r $TOP_DIR/edk2-test-parser root_fs_overlay/usr/bin/ fi cp -r $TOP_DIR/../common/log_parser root_fs_overlay/usr/bin - mkdir -p root_fs_overlay/usr/bin/log_parser/tools - cp $TOP_DIR/../common/tools/acs-results-schema.json \ - $TOP_DIR/../common/tools/suite_registry.json \ - $TOP_DIR/../common/tools/suite_registry.py \ - $TOP_DIR/../common/tools/validate.py \ - root_fs_overlay/usr/bin/log_parser/tools/ cp -r $TOP_DIR/systemready-scripts root_fs_overlay/usr/bin cp $TOP_DIR/ramdisk/linux-bsa/bsa root_fs_overlay/bin/ cp $TOP_DIR/ramdisk/linux-bsa/bsa_acs.ko root_fs_overlay/lib/modules/ diff --git a/SystemReady-devicetree-band/Yocto/build-scripts/get_source.sh b/SystemReady-devicetree-band/Yocto/build-scripts/get_source.sh index 2ad85c93..60254efe 100755 --- a/SystemReady-devicetree-band/Yocto/build-scripts/get_source.sh +++ b/SystemReady-devicetree-band/Yocto/build-scripts/get_source.sh @@ -159,12 +159,6 @@ copy_recipes() cp $TOP_DIR/../../common/linux_scripts/read_write_check_blk_devices.py $TOP_DIR/meta-woden/recipes-acs/install-files/files cp -r $TOP_DIR/../../common/log_parser $TOP_DIR/meta-woden/recipes-acs/install-files/files/ - mkdir -p $TOP_DIR/meta-woden/recipes-acs/install-files/files/log_parser/tools - cp $TOP_DIR/../../common/tools/acs-results-schema.json \ - $TOP_DIR/../../common/tools/suite_registry.json \ - $TOP_DIR/../../common/tools/suite_registry.py \ - $TOP_DIR/../../common/tools/validate.py \ - $TOP_DIR/meta-woden/recipes-acs/install-files/files/log_parser/tools/ popd # copy any patches to linux src files directory cp $COMMON_DIR_PATH/patches/tpm-tis-spi-Add-hardware-wait-polling.patch $TOP_DIR/meta-woden/recipes-kernel/linux/files diff --git a/common/tools/acs-results-schema.json b/common/log_parser/acs-results-schema.json similarity index 100% rename from common/tools/acs-results-schema.json rename to common/log_parser/acs-results-schema.json diff --git a/common/log_parser/enrich_suite_json.py b/common/log_parser/enrich_suite_json.py index 6bf204c6..7cf6bfeb 100644 --- a/common/log_parser/enrich_suite_json.py +++ b/common/log_parser/enrich_suite_json.py @@ -23,15 +23,7 @@ from pathlib import Path BASE_DIR = Path(__file__).resolve().parent -TOOLS_CANDIDATES = ( - BASE_DIR.parent / "tools", - BASE_DIR / "tools", -) -TOOLS_DIR = next( - (path for path in TOOLS_CANDIDATES if (path / "suite_registry.py").is_file()), - TOOLS_CANDIDATES[0], -) -sys.path.insert(0, str(TOOLS_DIR)) +sys.path.insert(0, str(BASE_DIR)) from suite_registry import REGISTRY_PATH, load_registry diff --git a/common/log_parser/main_log_parser.sh b/common/log_parser/main_log_parser.sh index bb285f38..0c1e4a03 100755 --- a/common/log_parser/main_log_parser.sh +++ b/common/log_parser/main_log_parser.sh @@ -24,11 +24,6 @@ BASE_DIR=$(dirname "$(realpath "$0")") # Determine paths SCRIPTS_PATH="$BASE_DIR" -if [ -f "$BASE_DIR/../tools/suite_registry.json" ]; then - TOOLS_PATH=$(realpath "$BASE_DIR/../tools") -else - TOOLS_PATH="$BASE_DIR/tools" -fi # Update this parser release version when publishing a new log parser release. LOG_PARSER_VERSION="1.0.0" @@ -70,9 +65,9 @@ WAIVER_JSON=$4 POST_SCRIPT_LOG="$LOGS_PATH/post-script/post-script.log" if [ $YOCTO_FLAG_PRESENT -eq 1 ]; then - test_category="/usr/bin/log_parser/test_categoryDT.json" + test_category="$BASE_DIR/test_categoryDT.json" else - test_category="/usr/bin/log_parser/test_category.json" + test_category="$BASE_DIR/test_category.json" fi # Check if ACS_CONFIG_PATH is provided @@ -865,7 +860,7 @@ fi if [ ${#JSON_FILES[@]} -gt 0 ]; then python3 "$SCRIPTS_PATH/enrich_suite_json.py" \ - --registry "$TOOLS_PATH/suite_registry.json" \ + --registry "$SCRIPTS_PATH/suite_registry.json" \ --test-category "$test_category" \ "${JSON_FILES[@]}" python3 "$SCRIPTS_PATH/merge_jsons.py" "$MERGED_JSON" "${JSON_FILES[@]}" diff --git a/common/log_parser/merge_jsons.py b/common/log_parser/merge_jsons.py index 498dff44..9bdff197 100755 --- a/common/log_parser/merge_jsons.py +++ b/common/log_parser/merge_jsons.py @@ -22,15 +22,7 @@ from pathlib import Path BASE_DIR = Path(__file__).resolve().parent -TOOLS_CANDIDATES = ( - BASE_DIR.parent / "tools", - BASE_DIR / "tools", -) -TOOLS_DIR = next( - (path for path in TOOLS_CANDIDATES if (path / "suite_registry.py").is_file()), - TOOLS_CANDIDATES[0], -) -sys.path.insert(0, str(TOOLS_DIR)) +sys.path.insert(0, str(BASE_DIR)) from suite_registry import ( normalize_suite_name as registry_normalize_suite_name, @@ -246,9 +238,9 @@ def load_test_category_data(mode, test_category_path=None): """ if not test_category_path: if mode == "DT": - test_category_path = "/usr/bin/log_parser/test_categoryDT.json" + test_category_path = BASE_DIR / "test_categoryDT.json" else: - test_category_path = "/usr/bin/log_parser/test_category.json" + test_category_path = BASE_DIR / "test_category.json" try: with open(test_category_path, "r") as catf: @@ -858,7 +850,7 @@ def main(): parser.add_argument("--selected-suites", default="", help="Comma-separated suite names to include in compliance reporting") parser.add_argument("--test-category", default="", - help="Explicit test category JSON path; legacy installed paths remain the default") + help="Explicit test category JSON path; defaults to the bundled mode-specific file") parser.add_argument("output_file", help="Output merged JSON file") parser.add_argument("json_files", nargs='+', help="List of JSON files to merge (including acs_info.json if present)") diff --git a/common/log_parser/package_standalone.sh b/common/log_parser/package_standalone.sh index f5a5c654..5851af7b 100755 --- a/common/log_parser/package_standalone.sh +++ b/common/log_parser/package_standalone.sh @@ -20,24 +20,18 @@ SCRIPT_DIR=$(dirname "$(realpath "$0")") REPO_ROOT=$(realpath "$SCRIPT_DIR/../..") OUTPUT_PATH=${1:-"$PWD/systemready-log-parser-standalone.tar.gz"} -required_paths=( - "common/log_parser/main_log_parser.sh" - "common/log_parser/standalone_runner.py" - "common/log_parser/requirements.txt" - "common/log_parser/test_category.json" - "common/log_parser/test_categoryDT.json" - "common/tools/acs-results-schema.json" - "common/tools/suite_registry.json" - "common/tools/suite_registry.py" - "common/tools/validate.py" +# Package the complete parser directory so every suite parser and support file +# is included automatically. The remaining paths are partner documentation. +package_paths=( + "common/log_parser" "docs/acs_schema_guide.md" "docs/log_parser_guide.md" "LICENSE.md" ) -for relative_path in "${required_paths[@]}"; do - if [ ! -f "$REPO_ROOT/$relative_path" ]; then - echo "ERROR: Required package file is missing: $REPO_ROOT/$relative_path" >&2 +for relative_path in "${package_paths[@]}"; do + if [ ! -e "$REPO_ROOT/$relative_path" ]; then + echo "ERROR: Required package path is missing: $REPO_ROOT/$relative_path" >&2 exit 1 fi done @@ -63,14 +57,7 @@ tar \ --transform='s,^,systemready-log-parser/,' \ -czf "$TEMP_ARCHIVE" \ -C "$REPO_ROOT" \ - common/log_parser \ - common/tools/acs-results-schema.json \ - common/tools/suite_registry.json \ - common/tools/suite_registry.py \ - common/tools/validate.py \ - docs/acs_schema_guide.md \ - docs/log_parser_guide.md \ - LICENSE.md + "${package_paths[@]}" mv "$TEMP_ARCHIVE" "$OUTPUT_PATH" output_dir=$(dirname "$OUTPUT_PATH") diff --git a/common/log_parser/standalone_runner.py b/common/log_parser/standalone_runner.py index aee66c17..a26bbbb9 100755 --- a/common/log_parser/standalone_runner.py +++ b/common/log_parser/standalone_runner.py @@ -29,15 +29,7 @@ from pathlib import Path BASE_DIR = Path(__file__).resolve().parent -TOOLS_CANDIDATES = ( - BASE_DIR.parent / "tools", - BASE_DIR / "tools", -) -TOOLS_DIR = next( - (path for path in TOOLS_CANDIDATES if (path / "suite_registry.py").is_file()), - TOOLS_CANDIDATES[0], -) -sys.path.insert(0, str(TOOLS_DIR)) +sys.path.insert(0, str(BASE_DIR)) from suite_registry import ( expand_selected_suites, @@ -49,8 +41,8 @@ ) -DEFAULT_REGISTRY = TOOLS_DIR / "suite_registry.json" -SCHEMA_VALIDATOR = TOOLS_DIR / "validate.py" +DEFAULT_REGISTRY = BASE_DIR / "suite_registry.json" +SCHEMA_VALIDATOR = BASE_DIR / "validate.py" DEFAULT_MODE = "SR" MINIMUM_PYTHON = (3, 8) MINIMUM_OUTPUT_FREE_BYTES = 10 * 1024 * 1024 @@ -1325,7 +1317,8 @@ def main(): json_dir = stage / "acs_jsons" html_dir = stage / "html_detailed_summaries" json_dir.mkdir(parents=True) - html_dir.mkdir(parents=True) + if "html" in outputs: + html_dir.mkdir(parents=True) copy_run_configs(stage, args) results = [] diff --git a/common/tools/suite_registry.json b/common/log_parser/suite_registry.json similarity index 100% rename from common/tools/suite_registry.json rename to common/log_parser/suite_registry.json diff --git a/common/tools/suite_registry.py b/common/log_parser/suite_registry.py similarity index 100% rename from common/tools/suite_registry.py rename to common/log_parser/suite_registry.py diff --git a/common/tools/validate.py b/common/log_parser/validate.py similarity index 99% rename from common/tools/validate.py rename to common/log_parser/validate.py index 00c278fa..f28466ee 100755 --- a/common/tools/validate.py +++ b/common/log_parser/validate.py @@ -35,6 +35,7 @@ sys.exit(2) SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIR)) from suite_registry import expand_selected_suites, load_registry diff --git a/docs/acs_schema_guide.md b/docs/acs_schema_guide.md index e3b57256..26638df2 100644 --- a/docs/acs_schema_guide.md +++ b/docs/acs_schema_guide.md @@ -17,21 +17,22 @@ suite JSON files with the single SystemReady schema validator. ## Files and Prerequisites -The schema tools are kept together: +The schema tools are kept beside the parser so a copied `common/log_parser` +directory remains self-contained: | File | Purpose | |---|---| -| `common/tools/validate.py` | Validates merged or raw JSON and formats errors | -| `common/tools/acs-results-schema.json` | Draft 2020-12 merged and suite contracts | -| `common/tools/suite_registry.json` | Maps raw filenames and suites to schema definitions | -| `common/tools/suite_registry.py` | Shared registry lookup helpers | +| `common/log_parser/validate.py` | Validates merged or raw JSON and formats errors | +| `common/log_parser/acs-results-schema.json` | Draft 2020-12 merged and suite contracts | +| `common/log_parser/suite_registry.json` | Maps raw filenames and suites to schema definitions | +| `common/log_parser/suite_registry.py` | Shared registry lookup helpers | Run commands in this guide from the repository root. Python 3 and the `jsonschema` package are required: ```bash python3 -m pip install -r common/log_parser/requirements.txt -common/tools/validate.py --help +common/log_parser/validate.py --help ``` Paths containing spaces must be quoted. @@ -60,11 +61,11 @@ contract. ### Standard Command ```bash -common/tools/validate.py merged \ +common/log_parser/validate.py merged \ "/path/to/acs_results/acs_summary/acs_jsons/merged_results.json" ``` -The default schema is `common/tools/acs-results-schema.json`. +The default schema is `common/log_parser/acs-results-schema.json`. Merged FWTS/SCT wrapper names are band-specific. SystemReady DT uses `Suite_Name: EBBR-FWTS` and `Suite_Name: EBBR-SCT`; SystemReady SR uses @@ -74,7 +75,7 @@ Merged FWTS/SCT wrapper names are band-specific. SystemReady DT uses ### Use a Different Schema ```bash -common/tools/validate.py merged \ +common/log_parser/validate.py merged \ "/path/to/merged_results.json" \ --schema "/path/to/candidate-schema.json" ``` @@ -85,26 +86,26 @@ The complete error count is always retained. This option changes only how many example locations are printed for each grouped issue: ```bash -common/tools/validate.py merged \ +common/log_parser/validate.py merged \ "/path/to/merged_results.json" \ --max-paths 2 ``` ## Validate Raw Suite JSON -Raw validation uses `common/tools/suite_registry.json` to choose the schema +Raw validation uses `common/log_parser/suite_registry.json` to choose the schema definition from each file's basename. ### One Suite ```bash -common/tools/validate.py raw "/path/to/acs_jsons/bsa.json" +common/log_parser/validate.py raw "/path/to/acs_jsons/bsa.json" ``` ### Multiple Suites ```bash -common/tools/validate.py raw \ +common/log_parser/validate.py raw \ "/path/to/acs_jsons/bsa.json" \ "/path/to/acs_jsons/fwts.json" \ "/path/to/acs_jsons/sct.json" @@ -113,7 +114,7 @@ common/tools/validate.py raw \ ### Discover Selected Suites in a Directory ```bash -common/tools/validate.py raw \ +common/log_parser/validate.py raw \ --json-dir "/path/to/acs_jsons" \ --selected-suites BSA,FWTS,SCT ``` @@ -125,13 +126,13 @@ error. List the accepted canonical names with: ```bash -common/tools/suite_registry.py list +common/log_parser/suite_registry.py list ``` ### Validate Every JSON in a Directory ```bash -common/tools/validate.py raw "/path/to/acs_jsons/"*.json +common/log_parser/validate.py raw "/path/to/acs_jsons/"*.json ``` Only filenames registered as raw suite outputs are validated. Files such as @@ -270,14 +271,14 @@ Use the exit code in automation; do not search terminal text for `PASS`. When adding or renaming a suite output: 1. Define or update the suite contract in - `common/tools/acs-results-schema.json`. + `common/log_parser/acs-results-schema.json`. 2. Update the suite entry, output filename, and schema fragment in - `common/tools/suite_registry.json`. + `common/log_parser/suite_registry.json`. 3. Keep parser script paths in the registry relative to `common/log_parser`. 4. Test the raw file with `validate.py raw`. 5. Test a complete merged artifact with `validate.py merged`. -6. Test the standalone package and the installed `log_parser/tools` layout. +6. Test the standalone package and the installed `log_parser` layout. Do not add a second validator for a new suite. Extend the schema and registry so the single validator handles it. diff --git a/docs/log_parser_guide.md b/docs/log_parser_guide.md index f8ab6e3d..ee14a0db 100644 --- a/docs/log_parser_guide.md +++ b/docs/log_parser_guide.md @@ -88,6 +88,39 @@ Confirm that the entry point is available: ./main_log_parser.sh --standalone --help ``` +### Copy Only the Log-Parser Directory + +`common/log_parser` is self-contained for parser execution. The suite registry, +registry helper, schema validator, merged-results schema, category files, parser +scripts, and Python requirements are all stored in this directory. A CI job may +copy or download only `common/log_parser`; it does not need `common/tools`. + +Update existing CI commands to use these paths: + +| Previous path | Current path | +|---|---| +| `common/tools/suite_registry.json` | `common/log_parser/suite_registry.json` | +| `common/tools/suite_registry.py` | `common/log_parser/suite_registry.py` | +| `common/tools/validate.py` | `common/log_parser/validate.py` | +| `common/tools/acs-results-schema.json` | `common/log_parser/acs-results-schema.json` | + +The previous `common/tools` paths are no longer installed or packaged. + +After copying the directory, verify the isolated layout from outside the source +repository: + +```bash +cd /path/to/copied/log_parser +python3 -m pip install -r requirements.txt +./main_log_parser.sh --version +./main_log_parser.sh --standalone --list-suites +./validate.py --help +``` + +These commands load the entry point, registry, and validator directly from the +copied directory. A real parse should then use the same copied directory and the +required Python packages from `requirements.txt`. + ## Choose an Execution Mode | Requirement | Normal parser | Standalone parser | @@ -192,12 +225,13 @@ The normal parser is the original main-branch behavior. It accepts positional arguments, auto-detects mode, and considers every suite applicable to that mode. -The normal flow in `main_log_parser.sh` uses host state and hard-coded suite -paths and does not use the standalone suite registry. The important boundaries -are: +The normal flow in `main_log_parser.sh` uses host state and fixed suite log +paths rather than the registry's standalone input-discovery paths. It still uses +the bundled registry for metadata enrichment and compliance mapping. The +important boundaries are: -- `/mnt/yocto_image.flag` selects normal DT or SR behavior and the installed - category file; +- `/mnt/yocto_image.flag` selects normal DT or SR behavior and the matching + category file bundled beside `main_log_parser.sh`; - the ACS config `Band`, not the host flag by itself, controls whether `acs_info.py` adds BMC firmware or PSCI version metadata; - compliance is calculated for the full requirement set of the detected mode, @@ -279,11 +313,11 @@ The normal parser checks the parser machine: /mnt/yocto_image.flag is absent -> SR mode ``` -It then uses the installed category file: +It then uses the matching category file beside `main_log_parser.sh`: ```text -DT -> /usr/bin/log_parser/test_categoryDT.json -SR -> /usr/bin/log_parser/test_category.json +DT -> /test_categoryDT.json +SR -> /test_category.json ``` Check what the normal parser will select before running it: @@ -423,8 +457,8 @@ sibling directories in the collected run layout. If one suite's logs use a different layout, supply those files directly. If several suites use a different layout, either arrange them in the registered layout or update their standalone paths in -`common/tools/suite_registry.json` before packaging the parser. The normal -parser does not read this registry. +`common/log_parser/suite_registry.json` before packaging the parser. These input +discovery paths do not change the normal parser's fixed log locations. #### Form 2: Direct Log Files @@ -745,7 +779,8 @@ linux/BsaResultsKernel.log ### How the Registry Is Used -`common/tools/suite_registry.json` is the standalone runner's suite directory. +`common/log_parser/suite_registry.json` is the standalone runner's suite +directory. For each suite it defines: - canonical name, accepted aliases, and DT/SR availability; @@ -891,7 +926,7 @@ Use `--test-category` when logs belong to a release with a different category file. The selected file is used by waiver handling, raw metadata enrichment, and merged compliance generation. -The normal parser uses the installed category file selected by its auto-detected +The normal parser uses the bundled category file selected by its auto-detected mode. That file is also applied to individual suite JSON files before they are merged. @@ -1066,8 +1101,8 @@ compliance output. ## Schema Validation -The repository has one schema command, `common/tools/validate.py`. Select what -you are validating with its first argument: +The repository has one schema command, `common/log_parser/validate.py`. Select +what you are validating with its first argument: - `merged` validates a complete normal-parser `merged_results.json`. - `raw` validates one or more individual suite JSON files. @@ -1119,20 +1154,20 @@ registered for their filename. From `common/log_parser`, validate a completed normal-parser merged result with: ```bash -../tools/validate.py merged \ +./validate.py merged \ /path/to/acs_summary/acs_jsons/merged_results.json ``` Validate one raw suite JSON, or several raw suite JSON files, with: ```bash -../tools/validate.py raw /path/to/acs_summary/acs_jsons/bsa.json +./validate.py raw /path/to/acs_summary/acs_jsons/bsa.json -../tools/validate.py raw /path/to/acs_summary/acs_jsons/*.json +./validate.py raw /path/to/acs_summary/acs_jsons/*.json ``` -The validator uses `common/tools/acs-results-schema.json` and -`common/tools/suite_registry.json` by default. In `merged` mode, use +The validator uses `common/log_parser/acs-results-schema.json` and +`common/log_parser/suite_registry.json` by default. In `merged` mode, use `--schema PATH` only when intentionally testing an alternate schema. In `raw` mode, use `--registry PATH` only when intentionally testing alternate raw-file mappings and schema references. The glob form may report `merged_results.json` @@ -1171,10 +1206,15 @@ python3 -m pip install -r requirements.txt The package includes parser scripts, the suite registry, schema validator, category files, schema, the log-parser and schema guides, requirements, and the -license. +license. All runtime support files remain under +`common/log_parser` in the archive; no `common/tools` directory is required. +The complete `common/log_parser` directory is archived as one unit, so +suite-specific `logs_to_json.py` and `json_to_html.py` files do not need a +second, manually maintained package list. The archive is intended for standalone execution. Its normal parser path still -expects the original installed ACS environment and category files. +expects the original ACS runtime environment; category files are resolved from +the extracted `common/log_parser` directory. ## Main Components @@ -1182,12 +1222,12 @@ expects the original installed ACS environment and category files. |---|---| | `main_log_parser.sh` | Owns the parser release version, dispatches standalone, or runs the original normal flow | | `standalone_runner.py` | Portable orchestration and validation | -| `common/tools/suite_registry.json` | Suite names, aliases, modes, paths, inputs, outputs, and schemas | -| `common/tools/suite_registry.py` | Shared registry loading, alias resolution, and suite lookup helpers | +| `common/log_parser/suite_registry.json` | Suite names, aliases, modes, paths, inputs, outputs, and schemas | +| `common/log_parser/suite_registry.py` | Shared registry loading, alias resolution, and suite lookup helpers | | `logs_to_json.py` | Suite-specific log parsing | | `apply_waivers.py` | Existing waiver application | | `enrich_suite_json.py` | Adds category metadata to raw suite JSON | -| `common/tools/validate.py` | Validates merged or raw suite JSON and formats grouped schema errors | +| `common/log_parser/validate.py` | Validates merged or raw suite JSON and formats grouped schema errors | | `json_to_html.py` | Suite-specific HTML generation | | `merge_jsons.py` | Merges results and calculates compliance | | `generate_acs_summary.py` | Generates combined HTML summary |