import os, sys, re, json

sys.stdout.reconfigure(encoding='utf-8')

base_dir = "resources/knowledge/original/원가계산/건설공사_표준품셈"

# Exclude 8장 (건설기계) and 합본
all_md_files = []
for root, dirs, files in os.walk(base_dir):
    for f in files:
        if f.endswith('.md') and not f.startswith('_') and '개정사항' not in f and '2026년_건설공사_표준품셈.md' not in f:
            if '01_공통부문\\제8장' in os.path.join(root, f) or '01_공통부문/제8장' in os.path.join(root, f):
                continue
            all_md_files.append(os.path.join(root, f))

print(f"Total target files (excluding 8장 and merged): {len(all_md_files)}")

results = []

for fpath in sorted(all_md_files):
    rel_path = os.path.relpath(fpath, base_dir)
    with open(fpath, 'r', encoding='utf-8') as f:
        lines = f.readlines()
        
    for idx, line in enumerate(lines):
        line_num = idx + 1
        line_str = line.strip()
        if not (line_str.startswith('|') and line_str.endswith('|')):
            continue
            
        cells = [c.strip() for c in line_str.split('|')[1:-1]]
        # Skip header separators |---|---|
        if all(re.match(r'^:?-+:?$', c) for c in cells if c):
            continue
            
        # Check for multi-token collapsed cells
        # Conditions:
        # A) cell has >= 8 tokens and contains numbers or spec symbols
        # B) cell contains embedded table headers like '구 분', '규 격', '단 위' etc
        # C) multiple numeric tokens that look like a row data squashed together
        for c_idx, c in enumerate(cells):
            tokens = c.split()
            # If cell has embedded table structure
            is_embedded_table = ('구 분' in c or '구분' in c) and ('단위' in c or '단 위' in c or '수량' in c or '수 량' in c)
            # If cell has many tokens
            is_many_tokens = len(tokens) >= 8 and sum(1 for tok in tokens if any(char.isdigit() for char in tok)) >= 4
            
            if is_embedded_table or is_many_tokens:
                results.append({
                    "file": rel_path,
                    "line": line_num,
                    "col_idx": c_idx,
                    "cell_snippet": c[:120],
                    "token_count": len(tokens),
                    "is_embedded_table": is_embedded_table
                })
                break

print(f"Total candidate collapsed rows found: {len(results)}")

by_file = {}
for r in results:
    f = r['file']
    by_file[f] = by_file.get(f, 0) + 1

for f, cnt in sorted(by_file.items(), key=lambda x: x[1], reverse=True):
    print(f"  {cnt:3d} rows: {f}")

with open('scratch/all_candidates_collapsed_ex8.json', 'w', encoding='utf-8') as f:
    json.dump(results, f, ensure_ascii=False, indent=2)