import os, sys, re

sys.stdout.reconfigure(encoding='utf-8')

target_files = [
    # 02_토목부문
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제1장_도로포장공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제2장_하천공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제3장_터널공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제4장_궤도공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제5장_강구조공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제6장_관부설및접합공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제7장_항만공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제8장_지반조사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제9장_측량.md',
    # 05_유지관리부문
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제1장_공통.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제2장_토목.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제3장_건축.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제4장_기계설비.md',
    # 04_기계설비부문 제13장
    'resources/knowledge/original/원가계산/건설공사_표준품셈/04_기계설비부문/제13장_플랜트설비공사.md',
]

print("=== Scanning 14 target files for duplicated/overlapped rows ===")

exact_duplicates = []
squashed_overlaps = []
header_repeats = []

for fpath in target_files:
    fname = os.path.basename(fpath)
    with open(fpath, 'r', encoding='utf-8') as f:
        lines = f.readlines()
        
    # Track recent data rows (sliding window of 40 rows)
    recent_rows = [] # list of (line_no, line_str, cells, numbers_set, text_words_set)
    recent_headers = []
    
    in_table = False
    
    for idx, line in enumerate(lines):
        line_str = line.strip()
        l_no = idx + 1
        
        if not (line_str.startswith('|') and line_str.endswith('|')):
            in_table = False
            # If leaving table, keep recent_rows for a bit or clear?
            # A duplicate table might start right after!
            continue
            
        cells = [c.strip() for c in line_str.split('|')[1:-1]]
        is_divider = all(re.match(r'^:?-+:?$', c) for c in cells if c)
        if is_divider:
            continue
            
        # Is it a header row?
        is_header = any(c in ["구 분", "구분", "품 명", "품명", "규 격", "규격", "단 위", "단위", "수 량", "수량", "직 종", "직종"] for c in cells)
        if is_header:
            # Check if identical header was seen recently (within 50 lines)
            for h_lno, h_cells in recent_headers[-5:]:
                if h_cells == cells:
                    header_repeats.append({
                        'file': fname,
                        'line': l_no,
                        'prev_line': h_lno,
                        'header': cells
                    })
            recent_headers.append((l_no, cells))
            continue
            
        # Skip notes
        if cells and cells[0] in ["비고", "비 고", "주", "[주]"]:
            continue
            
        # Extract numbers and words
        row_numbers = set(re.findall(r'\b\d+(?:\.\d+)?\b', line_str))
        row_words = set(w for w in re.findall(r'[가-힣A-Za-z0-9]+', line_str) if len(w) > 1)
        
        # 1. Check exact duplicate in sliding window
        for r_lno, r_str, r_cells, r_nums, r_words in recent_rows[-30:]:
            # If cells match exactly, and not just all empty or standard dashes
            non_empty = [c for c in cells if c not in ["", "-", "〃"]]
            if len(non_empty) >= 2 and cells == r_cells:
                exact_duplicates.append({
                    'file': fname,
                    'line': l_no,
                    'prev_line': r_lno,
                    'row': line_str[:80]
                })
                break
                
        # 2. Check squashed overlap:
        # If this row contains multiple words/numbers that appeared across MULTIPLE preceding separate rows
        # E.g., row_words has words from at least 3 distinct preceding rows, or numbers from 3 distinct rows
        matches_per_row = []
        for r_lno, r_str, r_cells, r_nums, r_words in recent_rows[-20:]:
            common_nums = row_numbers.intersection(r_nums)
            common_words = row_words.intersection(r_words)
            if len(common_nums) >= 2 or len(common_words) >= 2:
                matches_per_row.append((r_lno, common_words, common_nums))
                
        if len(matches_per_row) >= 3:
            # This row overlaps with 3 or more distinct preceding rows!
            squashed_overlaps.append({
                'file': fname,
                'line': l_no,
                'matched_prev_lines': [m[0] for m in matches_per_row],
                'row': line_str[:90]
            })
            
        recent_rows.append((l_no, line_str, cells, row_numbers, row_words))
        if len(recent_rows) > 40:
            recent_rows.pop(0)

print(f"\n1. Exact duplicate rows in window: {len(exact_duplicates)}")
for d in exact_duplicates:
    print(f"  [{d['file']}:L{d['line']} (prev L{d['prev_line']})] {d['row']}")

print(f"\n2. Squashed overlaps (matching 3+ preceding rows): {len(squashed_overlaps)}")
for s in squashed_overlaps:
    print(f"  [{s['file']}:L{s['line']} (matched {s['matched_prev_lines']})] {s['row']}")

print(f"\n3. Header repeats (same table header repeated within 50 lines): {len(header_repeats)}")
for h in header_repeats:
    print(f"  [{h['file']}:L{h['line']} (prev L{h['prev_line']})] {h['header']}")