import os, sys, re

sys.stdout.reconfigure(encoding='utf-8')

target_files = [
    # 02_토목부문
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제1장_도로포장공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제2장_하천공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제3장_터널공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제4장_궤도공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제5장_강구조공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제6장_관부설및접합공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제7장_항만공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제8장_지반조사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제9장_측량.md',
    # 05_유지관리부문
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제1장_공통.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제2장_토목.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제3장_건축.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제4장_기계설비.md',
    # 04_기계설비부문 제13장
    'resources/knowledge/original/원가계산/건설공사_표준품셈/04_기계설비부문/제13장_플랜트설비공사.md',
]

print("=== Strict Audit of Duplicate Substantive Data Rows across 14 Target Chapters ===")

strict_candidates = []

for fpath in target_files:
    fname = os.path.basename(fpath)
    with open(fpath, 'r', encoding='utf-8') as f:
        lines = f.readlines()
        
    # Collect all table rows
    all_rows = []
    for idx, l in enumerate(lines):
        line_str = l.strip()
        if not (line_str.startswith('|') and line_str.endswith('|')):
            continue
        cells = [c.strip() for c in line_str.split('|')[1:-1]]
        if all(re.match(r'^:?-+:?$', c) for c in cells if c):
            continue
        if cells and cells[0] in ["비고", "비 고", "주", "[주]"]:
            continue
        if any(c in ["구 분", "구분", "품 명", "품명", "규 격", "규격", "단 위", "단위", "수 량", "수량", "직 종", "직종"] for c in cells):
            continue
            
        # Is it a substantive row?
        # Must have at least one specific item name, spec, or multi-values
        # Filter out generic worker rows with no spec (e.g. '| 보통인부 | 인 | 1 | |')
        non_empty = [c for c in cells if c and c not in ["-", "〃"]]
        if len(non_empty) <= 1:
            continue
            
        # Check if it's just a generic worker row
        if len(non_empty) <= 3 and any(w in line_str for w in ["보통인부", "특별인부", "인부", "기계설비공", "용접공"]) and not any(re.search(r'\d+\.\d{2,}', c) for c in cells):
            # Check if all numbers are small ints like 1, 2, 3
            nums = re.findall(r'\b\d+(?:\.\d+)?\b', line_str)
            if all(n in ["1", "2", "3", "4", "5", "10", "20"] for n in nums):
                continue
                
        all_rows.append((idx + 1, line_str, cells))
        
    # Now check for duplicates within distance of 80 rows
    for i in range(len(all_rows)):
        lno_i, str_i, cells_i = all_rows[i]
        for j in range(i + 1, min(len(all_rows), i + 80)):
            lno_j, str_j, cells_j = all_rows[j]
            
            # Exact match of non-trivial row
            if cells_i == cells_j:
                strict_candidates.append({
                    'file': fname,
                    'line_1': lno_i,
                    'line_2': lno_j,
                    'type': 'exact_duplicate_substantive',
                    'row': str_i[:90]
                })
            # Check if str_j is a squashed row containing str_i
            # E.g. cells_j has multiple values per cell and covers cells_i
            else:
                words_i = set(re.findall(r'[가-힣A-Za-z0-9.]+', str_i))
                words_j = set(re.findall(r'[가-힣A-Za-z0-9.]+', str_j))
                # Remove generic tokens
                clean_i = {w for w in words_i if w not in ["인부", "보통인부", "특별인부", "인", "대", "개", "m", "㎡", "㎥", "ton"]}
                if len(clean_i) >= 4 and clean_i.issubset(words_j) and len(words_j) > len(words_i) + 4:
                    strict_candidates.append({
                        'file': fname,
                        'line_1': lno_i,
                        'line_2': lno_j,
                        'type': 'squashed_superset_of_line_1',
                        'row_1': str_i[:80],
                        'row_2': str_j[:80]
                    })

print(f"Total strict candidates found: {len(strict_candidates)}")
for sc in strict_candidates:
    print(f"\n[{sc['file']}] L{sc['line_1']} & L{sc['line_2']} - {sc['type']}")
    if sc['type'] == 'exact_duplicate_substantive':
        print(f"   Row: {sc['row']}")
    else:
        print(f"   Row 1: {sc['row_1']}")
        print(f"   Row 2: {sc['row_2']}")