82 lines
2.8 KiB
Python
82 lines
2.8 KiB
Python
import json
|
|
import sys
|
|
import re
|
|
from pathlib import Path
|
|
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
# Load enriched tables from scratch
|
|
sys.path.insert(0, '.')
|
|
from scratch.enrich_tables import tables
|
|
|
|
with open('resources/data_work_item_master/work_item_master_2026-01-01.json', 'r', encoding='utf-8') as f:
|
|
master = json.load(f)
|
|
|
|
master_tables = {}
|
|
for wi in master['work_items']:
|
|
for t in wi.get('tables', []):
|
|
tid = t.get('pum_table_id')
|
|
if tid:
|
|
master_tables[tid] = t
|
|
|
|
# Compare MD notes vs Master notes for all 476 tables
|
|
truncated_or_missing_notes = []
|
|
empty_in_master_has_in_md = []
|
|
fewer_lines_in_master = []
|
|
|
|
for t in tables:
|
|
tid = t['table_id']
|
|
mt = master_tables.get(tid)
|
|
if not mt:
|
|
continue
|
|
|
|
md_notes = t.get('notes', [])
|
|
master_notes = mt.get('notes', [])
|
|
|
|
# Clean up md_notes
|
|
# Filter out empty or pure markdown formatting if any
|
|
md_clean = [n.strip() for n in md_notes if n.strip()]
|
|
m_clean = [n.strip() for n in master_notes if n.strip()]
|
|
|
|
if md_clean and not m_clean:
|
|
empty_in_master_has_in_md.append({
|
|
'table_id': tid,
|
|
'chapter': t['chapter'],
|
|
'section': t['section'],
|
|
'line': t['start_line'],
|
|
'md_notes_count': len(md_clean),
|
|
'md_notes': md_clean
|
|
})
|
|
elif len(md_clean) > len(m_clean):
|
|
# Master has fewer lines than MD. Check what was lost!
|
|
lost_lines = []
|
|
for line in md_clean:
|
|
# Check if this line is in master_notes
|
|
if not any(line in mn or mn in line for mn in m_clean):
|
|
lost_lines.append(line)
|
|
if lost_lines:
|
|
fewer_lines_in_master.append({
|
|
'table_id': tid,
|
|
'chapter': t['chapter'],
|
|
'section': t['section'],
|
|
'line': t['start_line'],
|
|
'md_count': len(md_clean),
|
|
'master_count': len(m_clean),
|
|
'lost_lines': lost_lines
|
|
})
|
|
|
|
print(f"Tables where MD has notes but Master is completely EMPTY: {len(empty_in_master_has_in_md)}")
|
|
print(f"Tables where Master lost lines compared to MD: {len(fewer_lines_in_master)}")
|
|
|
|
print("\n--- 1. Tables where MD has notes but Master is EMPTY (First 15) ---")
|
|
for it in empty_in_master_has_in_md[:15]:
|
|
print(f"[{it['table_id']} | line {it['line']}] {it['chapter']} | {it['section']} (MD {it['md_notes_count']} lines)")
|
|
for n in it['md_notes'][:3]:
|
|
print(f" {n}")
|
|
|
|
print("\n--- 2. Tables where Master lost lines (Sample 10) ---")
|
|
for it in fewer_lines_in_master[:10]:
|
|
print(f"[{it['table_id']} | line {it['line']}] {it['chapter']} | {it['section']} (MD: {it['md_count']} vs Master: {it['master_count']})")
|
|
for l in it['lost_lines'][:3]:
|
|
print(f" LOST: {l}")
|