export_website_technology_csv.py

"""Flatten a Website Technology Lookup Dataset JSON export into CSV."""

from __future__ import annotations

import argparse
import csv
import json
from pathlib import Path
from typing import Any


FIELDS = [
    "domain",
    "status",
    "title",
    "category",
    "technology",
    "version",
    "detectionStatus",
    "isCurrentlyDetected",
    "httpStatusCode",
    "attempts",
    "error",
]


def export_csv(input_path: Path, output_path: Path) -> None:
    results = json.loads(input_path.read_text(encoding="utf-8"))
    if not isinstance(results, list):
        raise ValueError("The input file must contain a JSON array of Dataset items.")

    rows: list[dict[str, Any]] = []
    for result in results:
        technologies = result.get("technologies") or []
        if not technologies:
            rows.append({
                "domain": result.get("domain"),
                "status": result.get("status"),
                "title": result.get("title"),
                "httpStatusCode": result.get("httpStatusCode"),
                "attempts": result.get("attempts"),
                "error": result.get("error"),
            })
            continue

        for technology in technologies:
            rows.append({
                "domain": result.get("domain"),
                "status": result.get("status"),
                "title": result.get("title"),
                "category": technology.get("category"),
                "technology": technology.get("name"),
                "version": technology.get("version"),
                "detectionStatus": technology.get("detectionStatus"),
                "isCurrentlyDetected": technology.get("isCurrentlyDetected"),
                "httpStatusCode": result.get("httpStatusCode"),
                "attempts": result.get("attempts"),
                "error": result.get("error"),
            })

    output_path.parent.mkdir(parents=True, exist_ok=True)
    with output_path.open("w", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
        writer.writeheader()
        writer.writerows(rows)


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("input", type=Path, nargs="?", default=Path("data/sample-output.json"))
    parser.add_argument("output", type=Path, nargs="?", default=Path("data/exported-website-technology.csv"))
    args = parser.parse_args()
    export_csv(args.input, args.output)
    print(f"Wrote {args.output}")


if __name__ == "__main__":
    main()