135 lines
5.0 KiB
Python
135 lines
5.0 KiB
Python
import sys, os, re, json
|
|
import fitz
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
base_dir = "resources/knowledge/original/원가계산/건설공사_표준품셈"
|
|
merged_file = os.path.join(base_dir, "2026년_건설공사_표준품셈.md")
|
|
|
|
with open(merged_file, 'r', encoding='utf-8') as f:
|
|
merged_text = f.read()
|
|
|
|
all_split_files = []
|
|
for root, dirs, files in os.walk(base_dir):
|
|
for f in files:
|
|
if f.endswith('.md') and not f.startswith('_') and '개정사항' not in f and '2026년_건설공사_표준품셈.md' not in f:
|
|
all_split_files.append(os.path.join(root, f))
|
|
|
|
print(f"Total split files: {len(all_split_files)}")
|
|
|
|
# Function to extract all markdown tables from text
|
|
def extract_tables(text, file_label):
|
|
lines = text.split('\n')
|
|
tables = []
|
|
curr_table = []
|
|
start_line = 0
|
|
preceding_header = ""
|
|
|
|
for idx, line in enumerate(lines):
|
|
line_num = idx + 1
|
|
l_strip = line.strip()
|
|
|
|
# Track last seen section header
|
|
m_head = re.match(r'^(?:#+\s*)?(\d+-\d+(?:-\d+)?\s+[^\n]+|\(\d{4}\)\s+[^\n]+)', l_strip)
|
|
if m_head:
|
|
preceding_header = m_head.group(1).strip()
|
|
|
|
if l_strip.startswith('|') and l_strip.endswith('|'):
|
|
if not curr_table:
|
|
start_line = line_num
|
|
curr_table.append((line_num, l_strip))
|
|
else:
|
|
if curr_table:
|
|
# Check if it has a divider row
|
|
has_divider = False
|
|
for _, r in curr_table:
|
|
if re.match(r'^\|(?:\s*:?-+:?\s*\|)+$', r):
|
|
has_divider = True
|
|
break
|
|
if has_divider:
|
|
tables.append({
|
|
"file": file_label,
|
|
"start_line": start_line,
|
|
"end_line": start_line + len(curr_table) - 1,
|
|
"header_hint": preceding_header,
|
|
"rows": curr_table,
|
|
"row_count": len(curr_table),
|
|
"col_count": len(curr_table[0][1].split('|')) - 2,
|
|
"raw_header": curr_table[0][1]
|
|
})
|
|
curr_table = []
|
|
|
|
if curr_table:
|
|
has_divider = any(re.match(r'^\|(?:\s*:?-+:?\s*\|)+$', r) for _, r in curr_table)
|
|
if has_divider:
|
|
tables.append({
|
|
"file": file_label,
|
|
"start_line": start_line,
|
|
"end_line": start_line + len(curr_table) - 1,
|
|
"header_hint": preceding_header,
|
|
"rows": curr_table,
|
|
"row_count": len(curr_table),
|
|
"col_count": len(curr_table[0][1].split('|')) - 2,
|
|
"raw_header": curr_table[0][1]
|
|
})
|
|
return tables
|
|
|
|
merged_tables = extract_tables(merged_text, "합본")
|
|
print(f"Total tables in 합본: {len(merged_tables)}")
|
|
|
|
split_tables = []
|
|
for fpath in sorted(all_split_files):
|
|
rel = os.path.relpath(fpath, base_dir)
|
|
with open(fpath, 'r', encoding='utf-8') as f:
|
|
txt = f.read()
|
|
t_list = extract_tables(txt, rel)
|
|
split_tables.extend(t_list)
|
|
|
|
print(f"Total tables across 45 분할본: {len(split_tables)}")
|
|
|
|
# Now compare:
|
|
# 1. Tables present in split but missing in merged
|
|
# We match tables by header_hint or raw_header + first data row
|
|
def get_table_signature(t):
|
|
# take header hint, col_count, and first 30 chars of row 0
|
|
h_hint = t['header_hint'].split('(')[0].strip()
|
|
r0 = t['rows'][0][1][:40]
|
|
return f"{h_hint} || {t['col_count']} cols || {r0}"
|
|
|
|
merged_signatures = set()
|
|
for mt in merged_tables:
|
|
merged_signatures.add(get_table_signature(mt))
|
|
|
|
split_signatures = set()
|
|
for st in split_tables:
|
|
split_signatures.add(get_table_signature(st))
|
|
|
|
only_in_split = []
|
|
for st in split_tables:
|
|
sig = get_table_signature(st)
|
|
if sig not in merged_signatures:
|
|
only_in_split.append(st)
|
|
|
|
only_in_merged = []
|
|
for mt in merged_tables:
|
|
sig = get_table_signature(mt)
|
|
if sig not in split_signatures:
|
|
only_in_merged.append(mt)
|
|
|
|
print(f"\nTables only in 분할본 (potential missing/collapsed in 합본): {len(only_in_split)}")
|
|
for t in only_in_split[:15]:
|
|
print(f" - {t['file']} (L{t['start_line']}): hint='{t['header_hint']}', cols={t['col_count']}, rows={t['row_count']}")
|
|
|
|
print(f"\nTables only in 합본 (potential missing/collapsed in 분할본): {len(only_in_merged)}")
|
|
for t in only_in_merged[:15]:
|
|
print(f" - {t['file']} (L{t['start_line']}): hint='{t['header_hint']}', cols={t['col_count']}, rows={t['row_count']}")
|
|
|
|
with open('scratch/tables_diff.json', 'w', encoding='utf-8') as f:
|
|
json.dump({
|
|
"merged_table_count": len(merged_tables),
|
|
"split_table_count": len(split_tables),
|
|
"only_in_split_count": len(only_in_split),
|
|
"only_in_split": [{k: v for k, v in t.items() if k != 'rows'} for t in only_in_split],
|
|
"only_in_merged_count": len(only_in_merged),
|
|
"only_in_merged": [{k: v for k, v in t.items() if k != 'rows'} for t in only_in_merged]
|
|
}, f, ensure_ascii=False, indent=2)
|