import sys, os, re, json
sys.stdout.reconfigure(encoding='utf-8')

base_dir = "resources/knowledge/original/원가계산/건설공사_표준품셈"

all_md_files = []
for root, dirs, files in os.walk(base_dir):
    for f in files:
        if f.endswith('.md') and not f.startswith('_') and '개정사항' not in f and '2026년_건설공사_표준품셈.md' not in f:
            all_md_files.append(os.path.join(root, f))

collapsed_tables_all = []

for fpath in sorted(all_md_files):
    rel_path = os.path.relpath(fpath, base_dir)
    with open(fpath, 'r', encoding='utf-8') as f:
        lines = f.readlines()
        
    for line_idx, line in enumerate(lines):
        line_num = line_idx + 1
        line_clean = line.strip()
        if line_clean.startswith('|') and line_clean.endswith('|'):
            cells = [c.strip() for c in line_clean.split('|')[1:-1]]
            # If cell has many tokens (e.g. >= 15 space-separated tokens)
            for c_idx, c in enumerate(cells):
                tokens = c.split()
                if len(tokens) >= 15:
                    collapsed_tables_all.append({
                        "file": rel_path,
                        "line": line_num,
                        "col_idx": c_idx,
                        "token_count": len(tokens),
                        "snippet": c[:120]
                    })
                    break

print(f"Total collapsed table rows found across 45 files: {len(collapsed_tables_all)}")

# Group by file
by_file = {}
for ct in collapsed_tables_all:
    f = ct['file']
    by_file[f] = by_file.get(f, 0) + 1

for f, cnt in sorted(by_file.items()):
    print(f" - {f}: {cnt} collapsed rows")

with open('scratch/all_collapsed_tables.json', 'w', encoding='utf-8') as f:
    json.dump(collapsed_tables_all, f, ensure_ascii=False, indent=2)