my-skills/exam/validate.py

#!/usr/bin/env python3
"""Validate a question JSON file against schema/question.schema.json.

Usage:
    python3 validate.py <json_file>
    echo '<json>' | python3 validate.py -

Exit codes: 0 = valid, 1 = errors found, 2 = usage error.
"""
import json
import sys
from pathlib import Path

# Resolve schema path relative to repo root (1 level up from this script)
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parent
SCHEMA_PATH = REPO_ROOT / "schema" / "question.schema.json"


def load_json(path: str) -> dict:
    if path == "-":
        return json.load(sys.stdin)
    with open(path, encoding="utf-8") as f:
        return json.load(f)


def validate(data: dict, schema: dict) -> list[str]:
    """Validate data against schema. Returns list of error strings."""
    errors = []
    try:
        from jsonschema import Draft7Validator
        validator = Draft7Validator(schema)
        for err in sorted(validator.iter_errors(data), key=lambda e: list(e.path)):
            path = ".".join(str(p) for p in err.absolute_path) or "(root)"
            errors.append(f"{path}: {err.message}")
        return errors
    except ImportError:
        pass

    # Fallback: basic manual validation
    required_top = ["topic", "type", "schema_version", "questions"]
    for field in required_top:
        if field not in data:
            errors.append(f"Missing required field: {field}")

    valid_types = [
        "single_choice", "multiple_choice", "true_false",
        "fill_blank", "short_answer", "code_reading", "scenario",
    ]
    if data.get("type") not in valid_types:
        errors.append(f"Invalid type: {data.get('type')}")

    if data.get("schema_version") != "1.0.0":
        errors.append(f"schema_version must be '1.0.0', got '{data.get('schema_version')}'")

    for i, q in enumerate(data.get("questions", [])):
        prefix = f"questions[{i}]"
        for field in ["id", "type", "difficulty", "tags", "question", "explanation"]:
            if field not in q:
                errors.append(f"{prefix}: missing '{field}'")
        if q.get("type") not in valid_types:
            errors.append(f"{prefix}: invalid type '{q.get('type')}'")
        if "difficulty" in q and not (1 <= q["difficulty"] <= 5):
            errors.append(f"{prefix}: difficulty must be 1-5")
        if "id" in q:
            import re
            if not re.match(r"^(sc|mc|tf|fb|sa|cr|sn)-\d{3}$", q["id"]):
                errors.append(f"{prefix}: id '{q['id']}' doesn't match pattern {{type}}-{{000}}")

    return errors


def main():
    if len(sys.argv) != 2:
        print(f"Usage: {sys.argv[0]} <json_file | ->", file=sys.stderr)
        sys.exit(2)

    schema = load_json(str(SCHEMA_PATH))
    data = load_json(sys.argv[1])
    errors = validate(data, schema)

    if errors:
        print(f"❌ {len(errors)} error(s):")
        for e in errors:
            print(f"  • {e}")
        sys.exit(1)
    else:
        print(f"✅ Valid — {len(data.get('questions', []))} question(s)")
        sys.exit(0)


if __name__ == "__main__":
    main()