import os, sys, re, json

sys.stdout.reconfigure(encoding='utf-8')

base_dir = "resources/knowledge/original/원가계산/건설공사_표준품셈"

all_md_files = []
for root, dirs, files in os.walk(base_dir):
    for f in files:
        if f.endswith('.md') and not f.startswith('_') and '개정사항' not in f and '2026년_건설공사_표준품셈.md' not in f:
            if '01_공통부문\\제8장' in os.path.join(root, f) or '01_공통부문/제8장' in os.path.join(root, f):
                continue
            all_md_files.append(os.path.join(root, f))

squashed_tables = []

for fpath in sorted(all_md_files):
    rel_path = os.path.relpath(fpath, base_dir)
    with open(fpath, 'r', encoding='utf-8') as f:
        lines = f.readlines()
        
    for idx, line in enumerate(lines):
        line_num = idx + 1
        line_str = line.strip()
        if line_str.startswith('|'):
            continue
            
        # Check if line contains a squashed table
        # Sign 1: contains both '구 분' (or '구분') and ('단위' or '단 위') and ('수량' or '수 량')
        # Sign 2: contains sec header followed immediately by table elements
        has_table_headers = ('구 분' in line_str or '구분' in line_str) and \
                            ('단 위' in line_str or '단위' in line_str) and \
                            ('수 량' in line_str or '수량' in line_str or '인' in line_str)
                            
        # Also check for tables without '수량' like '규격' '단위'
        has_spec_unit = ('규 격' in line_str or '규격' in line_str) and \
                        ('단 위' in line_str or '단위' in line_str) and \
                        any(term in line_str for term in ['공(인)', '인부', '비고', '시공량'])

        if has_table_headers or has_spec_unit:
            # Check length to avoid simple sentences
            if len(line_str) > 40:
                squashed_tables.append({
                    "file": rel_path,
                    "line": line_num,
                    "length": len(line_str),
                    "line_text": line_str
                })

print(f"Total squashed table lines in non-table text: {len(squashed_tables)}")

for st in squashed_tables:
    print(f"\n{st['file']} L{st['line']} (len: {st['length']}):")
    print(f"  {st['line_text'][:120]}")

with open('scratch/squashed_tables_in_text.json', 'w', encoding='utf-8') as f:
    json.dump(squashed_tables, f, ensure_ascii=False, indent=2)