#!/usr/bin/env python3 """Check that JSON, JSONL and YAML files parse cleanly. Usage: python3 scripts/validate_config.py [PATH ...] [--exclude GLOB ...] PATH Files or directories to check. Defaults to the current directory. --exclude Glob matched against the path relative to the current directory, e.g. "fixtures/broken/**". Can be repeated. What it checks: *.json parses as JSON, and no object has the same key twice *.jsonl every non-blank line parses as JSON (also checks duplicate keys) *.yml / *.yaml every document parses with yaml.safe_load_all Requirements: Python 3.8+. PyYAML (pip install pyyaml) is needed only when YAML files are found. No environment variables are required. When GITHUB_ACTIONS=true errors are also printed as GitHub annotations so they show up on the PR. Exit codes: 0 every file parsed 1 one or more files failed to parse 2 usage error, missing path, or PyYAML not installed while YAML files exist """ import argparse import fnmatch import json import os import sys SKIP_DIRS = {".git", "node_modules", ".venv", "venv", "__pycache__", "dist", "build", "vendor", ".tox", ".mypy_cache"} EXTENSIONS = {".json": "json", ".jsonl": "jsonl", ".yml": "yaml", ".yaml": "yaml"} IN_ACTIONS = os.environ.get("GITHUB_ACTIONS") == "true" class DuplicateKeyError(ValueError): pass def no_duplicates(pairs): seen = {} for key, value in pairs: if key in seen: raise DuplicateKeyError("duplicate key %r" % key) seen[key] = value return seen def collect(paths, excludes): found = [] for root in paths: if os.path.isfile(root): candidates = [root] elif os.path.isdir(root): candidates = [] for dirpath, dirnames, filenames in os.walk(root): dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS) for name in sorted(filenames): candidates.append(os.path.join(dirpath, name)) else: print("error: path not found: %s" % root, file=sys.stderr) sys.exit(2) for path in candidates: ext = os.path.splitext(path)[1].lower() if ext not in EXTENSIONS: continue rel = os.path.relpath(path).replace(os.sep, "/") if any(fnmatch.fnmatch(rel, pattern) for pattern in excludes): continue found.append((rel, EXTENSIONS[ext])) return found def report(path, line, message): location = "%s:%s" % (path, line) if line else path print("FAIL %s: %s" % (location, message)) if IN_ACTIONS: line_part = ",line=%s" % line if line else "" safe = message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") print("::error file=%s%s::%s" % (path, line_part, safe)) def check_json(path): with open(path, encoding="utf-8-sig") as handle: text = handle.read() try: json.loads(text, object_pairs_hook=no_duplicates) except json.JSONDecodeError as exc: report(path, exc.lineno, "%s (column %d)" % (exc.msg, exc.colno)) return False except DuplicateKeyError as exc: report(path, None, str(exc)) return False return True def check_jsonl(path): ok = True with open(path, encoding="utf-8-sig") as handle: for number, line in enumerate(handle, start=1): if not line.strip(): continue try: json.loads(line, object_pairs_hook=no_duplicates) except json.JSONDecodeError as exc: report(path, number, "%s (column %d)" % (exc.msg, exc.colno)) ok = False except DuplicateKeyError as exc: report(path, number, str(exc)) ok = False return ok def check_yaml(path, yaml): with open(path, encoding="utf-8-sig") as handle: try: for _ in yaml.safe_load_all(handle): pass except yaml.YAMLError as exc: mark = getattr(exc, "problem_mark", None) line = mark.line + 1 if mark is not None else None problem = getattr(exc, "problem", None) or str(exc) report(path, line, problem) return False return True def main(): parser = argparse.ArgumentParser(description="Validate JSON, JSONL and YAML files.") parser.add_argument("paths", nargs="*", default=["."]) parser.add_argument("--exclude", action="append", default=[], metavar="GLOB") args = parser.parse_args() files = collect(args.paths, args.exclude) yaml = None if any(kind == "yaml" for _, kind in files): try: import yaml # noqa: F811 except ImportError: print("error: YAML files found but PyYAML is not installed (pip install pyyaml)", file=sys.stderr) return 2 failures = 0 for path, kind in files: try: if kind == "json": ok = check_json(path) elif kind == "jsonl": ok = check_jsonl(path) else: ok = check_yaml(path, yaml) except UnicodeDecodeError as exc: report(path, None, "not valid UTF-8: %s" % exc.reason) ok = False if not ok: failures += 1 print("checked %d file(s), %d failed" % (len(files), failures)) return 1 if failures else 0 if __name__ == "__main__": sys.exit(main())