#!/usr/bin/env python3 """Redact common PII patterns from text before it goes into a prompt, log or dataset. LIMITS, READ FIRST: this is pattern matching, not entity recognition. It catches well-formed emails, phone numbers, US Social Security numbers, payment card numbers (Luhn-checked), IPv4/IPv6 addresses, IBANs and secrets that look like "Bearer ". It does NOT detect names, street addresses, dates of birth, free-text medical or financial details, national IDs other than US SSNs, or anything misspelled or split across lines. Phone matching needs a separator or a leading + and can still hit other grouped digit runs. Treat the output as reduced-risk, not anonymized, and keep a human or a dedicated PII service in the loop for regulated data. Usage: python3 scripts/redact_pii.py [FILE ...] [--in-place] [--numbered] [--only TYPE ...] [--report] [--fail-on-match] cat transcript.txt | python3 scripts/redact_pii.py > clean.txt FILE Files to redact. With no FILE, reads stdin and writes stdout. --in-place Overwrite each FILE with the redacted text. --numbered Use stable numbered placeholders ([EMAIL_1], [EMAIL_2]) so the same value maps to the same placeholder within one run. --only TYPE Only redact these types. Types: EMAIL, PHONE, SSN, CARD, IPV4, IPV6, IBAN, BEARER. Can be repeated. --report Print counts per type to stderr. --fail-on-match Exit 1 if anything was found (useful as a CI or pre-commit check; combine with --report and no --in-place to only check). Requirements: Python 3.8+, no packages, no environment variables. Exit codes: 0 success (or nothing found with --fail-on-match) 1 PII found and --fail-on-match was given 2 usage error (unreadable file, unknown type) """ import argparse import re import sys def luhn_ok(digits): total, double = 0, False for char in reversed(digits): value = int(char) if double: value *= 2 if value > 9: value -= 9 total += value double = not double return total % 10 == 0 def card_ok(match): digits = re.sub(r"\D", "", match.group(0)) return 13 <= len(digits) <= 19 and luhn_ok(digits) def ssn_ok(match): area, group, serial = match.group(1), match.group(2), match.group(3) return area not in ("000", "666") and not area.startswith("9") and group != "00" and serial != "0000" def phone_ok(match): text = match.group(0) if re.fullmatch(r"\d{4}-\d{2}-\d{2}|\d{2}[-.]\d{2}[-.]\d{4}", text): return False # looks like a date if not re.search(r"[ .\-()+]", text): return False # a bare digit run is more likely an ID than a phone number return 7 <= len(re.sub(r"\D", "", text)) <= 15 def ipv4_ok(match): return all(0 <= int(part) <= 255 for part in match.group(0).split(".")) # Order matters: more specific patterns run first so their text is gone before broader ones run. PATTERNS = [ ("BEARER", re.compile(r"(?i)\bbearer\s+[A-Za-z0-9\-._~+/]{16,}=*"), None), ("EMAIL", re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b"), None), ("IBAN", re.compile(r"\b[A-Z]{2}\d{2}(?: ?[A-Z0-9]{4}){2,7}(?: ?[A-Z0-9]{1,4})?\b"), None), ("CARD", re.compile(r"\b(?:\d[ -]?){12,18}\d\b"), card_ok), ("SSN", re.compile(r"\b(\d{3})-(\d{2})-(\d{4})\b"), ssn_ok), ("IPV6", re.compile(r"\b(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}\b|\b(?:[0-9A-Fa-f]{1,4}:){1,7}:(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4}){0,6})?(?![0-9A-Fa-f:])"), None), ("IPV4", re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), ipv4_ok), ("PHONE", re.compile(r"(?