import os, sys, re

sys.stdout.reconfigure(encoding='utf-8')

target_files = [
    # 02_토목부문
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제1장_도로포장공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제2장_하천공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제3장_터널공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제4장_궤도공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제5장_강구조공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제6장_관부설및접합공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제7장_항만공사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제8장_지반조사.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제9장_측량.md',
    # 05_유지관리부문
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제1장_공통.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제2장_토목.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제3장_건축.md',
    'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제4장_기계설비.md',
    # 04_기계설비부문 제13장
    'resources/knowledge/original/원가계산/건설공사_표준품셈/04_기계설비부문/제13장_플랜트설비공사.md',
]

print("=== Scanning Tables for Exact Numeric/Text Subsets or Duplicates ===")

true_overlaps = []

for fpath in target_files:
    fname = os.path.basename(fpath)
    with open(fpath, 'r', encoding='utf-8') as f:
        lines = f.readlines()
        
    tables = []
    cur_t = []
    cur_start = 0
    for idx, l in enumerate(lines):
        line_str = l.strip()
        if line_str.startswith('|') and line_str.endswith('|'):
            if not cur_t:
                cur_start = idx + 1
            cur_t.append((idx + 1, line_str))
        else:
            if cur_t:
                tables.append((cur_start, cur_t))
                cur_t = []
    if cur_t:
        tables.append((cur_start, cur_t))
        
    for t_start, t_rows in tables:
        data_rows = []
        for lno, r_str in t_rows:
            cells = [c.strip() for c in r_str.split('|')[1:-1]]
            if all(re.match(r'^:?-+:?$', c) for c in cells if c):
                continue
            if cells and cells[0] in ["비고", "비 고", "주", "[주]"]:
                continue
            data_rows.append((lno, r_str, cells))
            
        if len(data_rows) < 3:
            continue
            
        # For each row, extract list of numbers (excluding trivial 0, 1, 2)
        row_num_tuples = []
        for lno, r_str, cells in data_rows:
            nums = re.findall(r'\b\d+(?:\.\d+)?\b', r_str)
            # filter out non-informative
            substantive_nums = [n for n in nums if n not in ["1", "2", "3", "4", "5", "1.0", "2.0"]]
            row_num_tuples.append((lno, r_str, cells, substantive_nums))
            
        for i in range(len(row_num_tuples)):
            lno_i, str_i, cells_i, nums_i = row_num_tuples[i]
            if len(nums_i) < 2:
                continue
                
            for j in range(i + 1, len(row_num_tuples)):
                lno_j, str_j, cells_j, nums_j = row_num_tuples[j]
                if len(nums_j) < 2:
                    continue
                    
                # Check if nums_i is identical to nums_j
                if nums_i == nums_j and len(nums_i) >= 3:
                    # Same 3+ substantive numbers!
                    # Check text
                    t_i = set(re.findall(r'[가-힣A-Za-z]+', str_i))
                    t_j = set(re.findall(r'[가-힣A-Za-z]+', str_j))
                    common_t = t_i.intersection(t_j)
                    if len(common_t) >= 1:
                        true_overlaps.append({
                            'file': fname,
                            'line_1': lno_i,
                            'line_2': lno_j,
                            'type': 'identical_substantive_numbers',
                            'nums': nums_i,
                            'common_text': list(common_t),
                            'row_1': str_i[:80],
                            'row_2': str_j[:80]
                        })
                # Check if nums_j contains nums_i and more (squashed combination)
                elif set(nums_i).issubset(set(nums_j)) and len(nums_i) >= 3 and len(nums_j) > len(nums_i):
                    t_i = set(re.findall(r'[가-힣A-Za-z]+', str_i))
                    t_j = set(re.findall(r'[가-힣A-Za-z]+', str_j))
                    if t_i.intersection(t_j):
                        true_overlaps.append({
                            'file': fname,
                            'line_1': lno_i,
                            'line_2': lno_j,
                            'type': 'subset_squashed_numbers',
                            'nums_subset': nums_i,
                            'nums_superset': nums_j,
                            'row_1': str_i[:80],
                            'row_2': str_j[:80]
                        })

print(f"Total True Overlaps Found: {len(true_overlaps)}")
for to in true_overlaps:
    print(f"\n[{to['file']}] Lines L{to['line_1']} & L{to['line_2']} - {to['type']}")
    print(f"   Row 1: {to['row_1']}")
    print(f"   Row 2: {to['row_2']}")