import os, sys, re

sys.stdout.reconfigure(encoding='utf-8')

target_files = [
    # 02_토목부문 (전체 9개 파일)
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제1장_도로포장공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제2장_하천공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제3장_터널공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제4장_궤도공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제5장_강구조공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제6장_관부설및접합공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제7장_항만공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제8장_지반조사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제9장_측량.md',
    # 05_유지관리부문 (전체 4개 파일)
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제1장_공통.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제2장_토목.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제3장_건축.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제4장_기계설비.md',
    # 04_기계설비부문 제13장
    'resources/knowledge/original/원가계산/건설공사_표준품셈/04_기계설비부문/제13장_플랜트설비공사.md',
]

print("=== 1. Checking for duplicated or overlapped table rows ===")

candidates = []

for fpath in target_files:
    fname = os.path.basename(fpath)
    with open(fpath, 'r', encoding='utf-8') as f:
        lines = f.readlines()
        
    tables = []
    current_table = []
    current_start = 0
    
    for idx, line in enumerate(lines):
        line_str = line.strip()
        if line_str.startswith('|') and line_str.endswith('|'):
            if not current_table:
                current_start = idx + 1
            current_table.append((idx + 1, line_str))
        else:
            if current_table:
                tables.append((current_start, current_table))
                current_table = []
    if current_table:
        tables.append((current_start, current_table))
        
    # Check each table
    for start_line, t_rows in tables:
        # Separate header and data rows
        data_rows = []
        for r_idx, (l_no, r_str) in enumerate(t_rows):
            cells = [c.strip() for c in r_str.split('|')[1:-1]]
            if all(re.match(r'^:?-+:?$', c) for c in cells if c):
                continue
            if r_idx == 0:
                header_cells = cells
                continue
            data_rows.append((l_no, cells, r_str))
            
        # Check if later data rows duplicate earlier data rows or contain words of earlier rows
        earlier_first_cells = []
        earlier_words = set()
        
        for l_no, cells, r_str in data_rows:
            if not cells:
                continue
            c0 = cells[0]
            # Check if this row is a collapsed multi-item row
            # Split words in c0
            c0_words = [w for w in c0.split() if len(w) > 1 and w not in ["-", "·", "ㆍ"]]
            
            # Check overlap
            overlap_words = [w for w in c0_words if w in earlier_words]
            if len(overlap_words) >= 2:
                candidates.append({
                    'file': fname,
                    'line': l_no,
                    'type': 'word_overlap_in_table',
                    'overlap': overlap_words,
                    'row': r_str[:80],
                    'earlier_first_cells': earlier_first_cells[-5:]
                })
            elif len(earlier_first_cells) > 0 and c0 in earlier_first_cells:
                # Same first cell appears again - could be normal (e.g. different spec) or duplicate
                # Check if entire row or multiple cells are identical
                pass
                
            earlier_first_cells.append(c0)
            for w in c0_words:
                earlier_words.add(w)

print(f"Total word overlap candidates in tables: {len(candidates)}")
for c in candidates:
    print(f"[{c['file']}:L{c['line']}] {c['type']} - overlap: {c['overlap']}")
    print(f"   row: {c['row']}")
    print(f"   earlier: {c['earlier_first_cells']}")