import os, sys, re

sys.stdout.reconfigure(encoding='utf-8')

target_files = [
    # 02_토목부문 (전체 9개 파일)
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제1장_도로포장공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제2장_하천공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제3장_터널공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제4장_궤도공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제5장_강구조공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제6장_관부설및접합공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제7장_항만공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제8장_지반조사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제9장_측량.md',
    # 05_유지관리부문 (전체 4개 파일)
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제1장_공통.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제2장_토목.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제3장_건축.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제4장_기계설비.md',
    # 04_기계설비부문 제13장
    'resources/knowledge/original/원가계산/건설공사_표준품셈/04_기계설비부문/제13장_플랜트설비공사.md',
]

print("=== Scanning Tables for Trailing Duplicate/Overlapped Rows ===")

tail_duplicates = []

for fpath in target_files:
    fname = os.path.basename(fpath)
    with open(fpath, 'r', encoding='utf-8') as f:
        lines = f.readlines()
        
    tables = []
    cur_t = []
    cur_start = 0
    for idx, l in enumerate(lines):
        line_str = l.strip()
        if line_str.startswith('|') and line_str.endswith('|'):
            if not cur_t:
                cur_start = idx + 1
            cur_t.append((idx + 1, line_str))
        else:
            if cur_t:
                tables.append((cur_start, cur_t))
                cur_t = []
    if cur_t:
        tables.append((cur_start, cur_t))
        
    for t_start, t_rows in tables:
        # Separate divider
        data_rows = []
        for lno, r_str in t_rows:
            cells = [c.strip() for c in r_str.split('|')[1:-1]]
            if all(re.match(r'^:?-+:?$', c) for c in cells if c):
                continue
            if cells and cells[0] in ["비고", "비 고", "주", "[주]"]:
                continue
            data_rows.append((lno, r_str, cells))
            
        if len(data_rows) < 4:
            continue
            
        # Check every row against all PRECEDING rows in the table
        # We look for:
        # 1. Exact duplicate rows (not consecutive 〃 or standard repeats)
        # 2. Rows whose cells contain multiple tokens from distinct preceding rows
        
        seen_cells_set = [] # list of (lno, cells, text_tokens, num_tokens)
        
        for r_idx, (lno, r_str, cells) in enumerate(data_rows):
            # Tokens
            text_tokens = set(re.findall(r'[가-힣A-Za-z]+', r_str))
            text_tokens = {t for t in text_tokens if len(t) > 1 and t not in ["인부", "보통인부", "특별인부", "플랜트", "설비공", "용접공", "배관공"]}
            num_tokens = set(re.findall(r'\b\d+(?:\.\d+)?\b', r_str))
            
            # Check 1: Exact duplicate of an earlier row that is NOT adjacent
            for prev_lno, prev_cells, prev_tt, prev_nt in seen_cells_set[:-1]:
                non_empty = [c for c in cells if c not in ["", "-", "〃"]]
                if len(non_empty) >= 2 and cells == prev_cells:
                    tail_duplicates.append({
                        'file': fname,
                        'line': lno,
                        'type': 'exact_duplicate_of_earlier_row',
                        'dup_of_line': prev_lno,
                        'row': r_str[:80]
                    })
                    break
                    
            # Check 2: Squashed composite row that combines tokens of 2+ distinct preceding rows
            matching_earlier = []
            for prev_lno, prev_cells, prev_tt, prev_nt in seen_cells_set:
                if len(prev_tt) >= 2 and prev_tt.issubset(text_tokens) and len(prev_nt.intersection(num_tokens)) >= 1:
                    matching_earlier.append(prev_lno)
                    
            if len(matching_earlier) >= 2:
                tail_duplicates.append({
                    'file': fname,
                    'line': lno,
                    'type': 'composite_squashed_duplicate',
                    'dup_of_lines': matching_earlier,
                    'row': r_str[:80]
                })
                
            seen_cells_set.append((lno, cells, text_tokens, num_tokens))

print(f"Total potential trailing duplicates found: {len(tail_duplicates)}")
for td in tail_duplicates:
    if td['type'] == 'exact_duplicate_of_earlier_row':
        print(f"[{td['file']}:L{td['line']}] EXACT DUP of L{td['dup_of_line']}: {td['row']}")
    else:
        print(f"[{td['file']}:L{td['line']}] COMPOSITE DUP of {td['dup_of_lines']}: {td['row']}")