120 lines
5.7 KiB
Python
120 lines
5.7 KiB
Python
import os, sys, re
|
|
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
target_files = [
|
|
# 02_토목부문
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제1장_도로포장공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제2장_하천공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제3장_터널공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제4장_궤도공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제5장_강구조공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제6장_관부설및접합공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제7장_항만공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제8장_지반조사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제9장_측량.md',
|
|
# 05_유지관리부문
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제1장_공통.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제2장_토목.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제3장_건축.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제4장_기계설비.md',
|
|
# 04_기계설비부문 제13장
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/04_기계설비부문/제13장_플랜트설비공사.md',
|
|
]
|
|
|
|
print("=== Scanning Tables for Exact Numeric/Text Subsets or Duplicates ===")
|
|
|
|
true_overlaps = []
|
|
|
|
for fpath in target_files:
|
|
fname = os.path.basename(fpath)
|
|
with open(fpath, 'r', encoding='utf-8') as f:
|
|
lines = f.readlines()
|
|
|
|
tables = []
|
|
cur_t = []
|
|
cur_start = 0
|
|
for idx, l in enumerate(lines):
|
|
line_str = l.strip()
|
|
if line_str.startswith('|') and line_str.endswith('|'):
|
|
if not cur_t:
|
|
cur_start = idx + 1
|
|
cur_t.append((idx + 1, line_str))
|
|
else:
|
|
if cur_t:
|
|
tables.append((cur_start, cur_t))
|
|
cur_t = []
|
|
if cur_t:
|
|
tables.append((cur_start, cur_t))
|
|
|
|
for t_start, t_rows in tables:
|
|
data_rows = []
|
|
for lno, r_str in t_rows:
|
|
cells = [c.strip() for c in r_str.split('|')[1:-1]]
|
|
if all(re.match(r'^:?-+:?$', c) for c in cells if c):
|
|
continue
|
|
if cells and cells[0] in ["비고", "비 고", "주", "[주]"]:
|
|
continue
|
|
data_rows.append((lno, r_str, cells))
|
|
|
|
if len(data_rows) < 3:
|
|
continue
|
|
|
|
# For each row, extract list of numbers (excluding trivial 0, 1, 2)
|
|
row_num_tuples = []
|
|
for lno, r_str, cells in data_rows:
|
|
nums = re.findall(r'\b\d+(?:\.\d+)?\b', r_str)
|
|
# filter out non-informative
|
|
substantive_nums = [n for n in nums if n not in ["1", "2", "3", "4", "5", "1.0", "2.0"]]
|
|
row_num_tuples.append((lno, r_str, cells, substantive_nums))
|
|
|
|
for i in range(len(row_num_tuples)):
|
|
lno_i, str_i, cells_i, nums_i = row_num_tuples[i]
|
|
if len(nums_i) < 2:
|
|
continue
|
|
|
|
for j in range(i + 1, len(row_num_tuples)):
|
|
lno_j, str_j, cells_j, nums_j = row_num_tuples[j]
|
|
if len(nums_j) < 2:
|
|
continue
|
|
|
|
# Check if nums_i is identical to nums_j
|
|
if nums_i == nums_j and len(nums_i) >= 3:
|
|
# Same 3+ substantive numbers!
|
|
# Check text
|
|
t_i = set(re.findall(r'[가-힣A-Za-z]+', str_i))
|
|
t_j = set(re.findall(r'[가-힣A-Za-z]+', str_j))
|
|
common_t = t_i.intersection(t_j)
|
|
if len(common_t) >= 1:
|
|
true_overlaps.append({
|
|
'file': fname,
|
|
'line_1': lno_i,
|
|
'line_2': lno_j,
|
|
'type': 'identical_substantive_numbers',
|
|
'nums': nums_i,
|
|
'common_text': list(common_t),
|
|
'row_1': str_i[:80],
|
|
'row_2': str_j[:80]
|
|
})
|
|
# Check if nums_j contains nums_i and more (squashed combination)
|
|
elif set(nums_i).issubset(set(nums_j)) and len(nums_i) >= 3 and len(nums_j) > len(nums_i):
|
|
t_i = set(re.findall(r'[가-힣A-Za-z]+', str_i))
|
|
t_j = set(re.findall(r'[가-힣A-Za-z]+', str_j))
|
|
if t_i.intersection(t_j):
|
|
true_overlaps.append({
|
|
'file': fname,
|
|
'line_1': lno_i,
|
|
'line_2': lno_j,
|
|
'type': 'subset_squashed_numbers',
|
|
'nums_subset': nums_i,
|
|
'nums_superset': nums_j,
|
|
'row_1': str_i[:80],
|
|
'row_2': str_j[:80]
|
|
})
|
|
|
|
print(f"Total True Overlaps Found: {len(true_overlaps)}")
|
|
for to in true_overlaps:
|
|
print(f"\n[{to['file']}] Lines L{to['line_1']} & L{to['line_2']} - {to['type']}")
|
|
print(f" Row 1: {to['row_1']}")
|
|
print(f" Row 2: {to['row_2']}")
|