"""Detect files with broken or unusual characters.

Usage:
  python detect_bad_chars.py --root "A:\\WorkProj\\OverPunk\\WikiServ\\Картотека" --out bad_files.json

The script scans text files and reports files containing:
- UTF-8 decode errors
- Unicode replacement character U+FFFD (�)
- C1 control characters (U+0080..U+009F) or other non-printable controls

Outputs a JSON file listing problematic files with reasons and sample snippets.
"""
from __future__ import annotations

import argparse
import json
import re
from pathlib import Path
from typing import List, Dict
import unicodedata

# Regex to find C0/C1 control characters excluding common whitespace (tab,newline,carriage return)
CONTROL_RE = re.compile(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x80-\x9F]")
REPLACEMENT_CHAR = "\uFFFD"

DEFAULT_EXTS = {".txt", ".md", ".rst", ".html", ".htm"}


def analyze_file(path: Path) -> List[Dict[str, str]]:
    issues = []
    # Try read as utf-8
    try:
        text = path.read_text(encoding="utf-8")
    except UnicodeDecodeError as e:
        # record decode error and try to read with replacement to produce snippet
        try:
            text = path.read_text(encoding="utf-8", errors="replace")
        except Exception:
            text = ""
        snippet = text[max(0, e.start - 40): e.start + 40]
        issues.append({"type": "utf8_decode_error", "detail": str(e), "sample": snippet})
        # Also try cp1251 to see if it decodes cleanly
        try:
            cp_text = path.read_text(encoding="cp1251")
            if REPLACEMENT_CHAR not in cp_text and not CONTROL_RE.search(cp_text):
                issues.append({"type": "cp1251_ok", "detail": "Decodes as cp1251 without obvious issues"})
        except Exception:
            pass
        return issues

    # Check for replacement character
    if REPLACEMENT_CHAR in text:
        idx = text.find(REPLACEMENT_CHAR)
        snippet = text[max(0, idx - 40): idx + 40]
        issues.append({"type": "replacement_char", "detail": f"Found {REPLACEMENT_CHAR}", "sample": snippet})

    # Check for control chars
    m = CONTROL_RE.search(text)
    if m:
        idx = m.start()
        snippet = text[max(0, idx - 40): idx + 40]
        hexval = f"0x{ord(m.group(0)):02X}"
        issues.append({"type": "control_char", "detail": hexval, "sample": snippet})

    # Check for unusual characters (not Cyrillic, not Latin, not common punctuation/digits/whitespace)
    def is_common(ch: str) -> bool:
        o = ord(ch)
        # whitespace, digits
        if ch in "\t\n\r " or (0x30 <= o <= 0x39):
            return True
        # Basic Latin letters
        if 0x0041 <= o <= 0x005A or 0x0061 <= o <= 0x007A:
            return True
        # Latin-1 Supplement and Latin Extended ranges
        if 0x00A0 <= o <= 0x024F or 0x1E00 <= o <= 0x1EFF:
            return True
        # Cyrillic ranges
        if (0x0400 <= o <= 0x04FF) or (0x0500 <= o <= 0x052F) or (0x2DE0 <= o <= 0x2DFF) or (0xA640 <= o <= 0xA69F):
            return True
        # Common ASCII punctuation
        if 0x0020 <= o <= 0x007E:
            return True
        # General punctuation
        if 0x2000 <= o <= 0x206F:
            return True
        # Combining marks
        if 0x0300 <= o <= 0x036F:
            return True
        # Currency symbols and common symbols
        if 0x20A0 <= o <= 0x20CF:
            return True
        return False

    unusual_pos = None
    for i, ch in enumerate(text):
        if ch == REPLACEMENT_CHAR:
            continue
        if CONTROL_RE.match(ch):
            continue
        if not is_common(ch):
            unusual_pos = i
            break

    if unusual_pos is not None:
        ch = text[unusual_pos]
        snippet = text[max(0, unusual_pos - 40): unusual_pos + 40]
        try:
            name = unicodedata.name(ch)
        except ValueError:
            name = "<no name>"
        issues.append({"type": "unusual_char", "detail": f"U+{ord(ch):04X} {name}", "sample": snippet})

    return issues


def scan(root: Path, exts: List[str]) -> Dict[str, Dict]:
    result = {}
    for path in root.rglob("*"):
        if not path.is_file():
            continue
        if path.suffix.lower() not in exts:
            continue
        issues = analyze_file(path)
        if issues:
            rel = path.relative_to(root).as_posix()
            result[rel] = {"issues": issues}
    return result


def main() -> int:
    parser = argparse.ArgumentParser(description="Detect files with broken or unusual characters")
    parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parent / "Картотека")
    parser.add_argument("--out", type=Path, default=Path(__file__).resolve().parent / "bad_files.json")
    parser.add_argument("--exts", type=str, default=",".join(sorted(DEFAULT_EXTS)), help="Comma-separated extensions to check")
    args = parser.parse_args()

    root = args.root
    if not root.exists():
        raise SystemExit(f"Root path '{root}' not found.")

    exts = {e if e.startswith('.') else '.' + e for e in [x.strip().lower() for x in args.exts.split(',') if x.strip()]}
    found = scan(root, list(exts))

    args.out.write_text(json.dumps(found, ensure_ascii=False, indent=2), encoding="utf-8")
    print(f"Scanned {root}. Problematic files: {len(found)}. Wrote {args.out}")
    return 0


if __name__ == '__main__':
    raise SystemExit(main())