import os, sys, re

sys.stdout.reconfigure(encoding='utf-8')

target_files = [
    # 02_토목부문
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제1장_도로포장공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제2장_하천공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제3장_터널공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제4장_궤도공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제5장_강구조공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제6장_관부설및접합공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제7장_항만공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제8장_지반조사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제9장_측량.md',
    # 05_유지관리부문
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제1장_공통.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제2장_토목.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제3장_건축.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제4장_기계설비.md',
    # 04_기계설비부문 제13장
    'resources/knowledge/original/원가계산/건설공사_표준품셈/04_기계설비부문/제13장_플랜트설비공사.md',
]

print("=== Scanning by Section for Overlapping / Redundant Table Rows ===")

findings = []

for fpath in target_files:
    fname = os.path.basename(fpath)
    with open(fpath, 'r', encoding='utf-8') as f:
        content = f.read()
        
    # Split content by section pattern: e.g. \n\d+-\d+(?:-\d+)?
    # Or line by line tracking current section
    lines = content.split('\n')
    current_sec = "TOP"
    sec_lines = []
    
    sections = [] # (sec_name, start_lno, lines)
    for idx, line in enumerate(lines):
        # Section header match e.g. "1-2-4 ", "## 1-2-4", etc.
        m = re.match(r'^(?:#+\s*)?(\d+-\d+(?:-\d+)?)\s+', line.strip())
        if m:
            if sec_lines:
                sections.append((current_sec, idx - len(sec_lines) + 1, sec_lines))
            current_sec = m.group(1)
            sec_lines = [line]
        else:
            sec_lines.append(line)
    if sec_lines:
        sections.append((current_sec, len(lines) - len(sec_lines) + 1, sec_lines))
        
    for sec_name, start_lno, s_lines in sections:
        # Extract tables in this section
        tables = []
        cur_t = []
        cur_t_start = 0
        for s_idx, sl in enumerate(s_lines):
            l_str = sl.strip()
            if l_str.startswith('|') and l_str.endswith('|'):
                if not cur_t:
                    cur_t_start = start_lno + s_idx
                cur_t.append((start_lno + s_idx, l_str))
            else:
                if cur_t:
                    tables.append((cur_t_start, cur_t))
                    cur_t = []
        if cur_t:
            tables.append((cur_t_start, cur_t))
            
        # Check within each table or across tables in the same section
        # Collect all rows in this section
        all_rows = []
        for t_start, t_rows in tables:
            for r_lno, r_str in t_rows:
                cells = [c.strip() for c in r_str.split('|')[1:-1]]
                if all(re.match(r'^:?-+:?$', c) for c in cells if c):
                    continue
                if cells and cells[0] in ["비고", "비 고", "주", "[주]"]:
                    continue
                # Skip header rows
                if any(c in ["구 분", "구분", "품 명", "품명", "규 격", "규격", "단 위", "단위", "수 량", "수량", "직 종", "직종"] for c in cells):
                    continue
                all_rows.append((r_lno, r_str, cells))
                
        # Now check if later rows duplicate earlier rows in the same section!
        # Two types:
        # Type A: Exact same data in 3 or more consecutive rows
        # Type B: A single row that squashes 2 or more preceding rows from the same section
        for i in range(len(all_rows)):
            r_lno, r_str, cells = all_rows[i]
            
            # Check if this row is a squashed version of preceding rows
            # Extract distinct tokens from this row
            row_tokens = set(re.findall(r'[가-힣A-Za-z0-9.]+', r_str))
            row_tokens = {t for t in row_tokens if len(t) > 1 and t not in ["인부", "보통인부", "특별인부", "0.0", "1.0"]}
            
            matching_preceding = []
            for j in range(0, i):
                prev_lno, prev_str, prev_cells = all_rows[j]
                prev_tokens = set(re.findall(r'[가-힣A-Za-z0-9.]+', prev_str))
                prev_tokens = {t for t in prev_tokens if len(t) > 1 and t not in ["인부", "보통인부", "특별인부", "0.0", "1.0"]}
                
                # Check how much prev_tokens is contained in row_tokens
                if len(prev_tokens) >= 3 and prev_tokens.issubset(row_tokens):
                    matching_preceding.append(prev_lno)
                    
            if len(matching_preceding) >= 2:
                findings.append({
                    'file': fname,
                    'section': sec_name,
                    'line': r_lno,
                    'squashed_row': r_str[:80],
                    'duplicates_of_lines': matching_preceding
                })

print(f"Total findings of squashed rows duplicating preceding rows in same section: {len(findings)}")
for f in findings:
    print(f"[{f['file']}:{f['section']}:L{f['line']}] duplicates lines {f['duplicates_of_lines']}")
    print(f"   Squashed row: {f['squashed_row']}")