import json
import sys
import re
from pathlib import Path

sys.stdout.reconfigure(encoding='utf-8')

# Load enriched tables from scratch
sys.path.insert(0, '.')
from scratch.enrich_tables import tables

with open('resources/data_work_item_master/work_item_master_2026-01-01.json', 'r', encoding='utf-8') as f:
    master = json.load(f)

master_tables = {}
for wi in master['work_items']:
    for t in wi.get('tables', []):
        tid = t.get('pum_table_id')
        if tid:
            master_tables[tid] = t

# Compare MD notes vs Master notes for all 476 tables
truncated_or_missing_notes = []
empty_in_master_has_in_md = []
fewer_lines_in_master = []

for t in tables:
    tid = t['table_id']
    mt = master_tables.get(tid)
    if not mt:
        continue
        
    md_notes = t.get('notes', [])
    master_notes = mt.get('notes', [])
    
    # Clean up md_notes
    # Filter out empty or pure markdown formatting if any
    md_clean = [n.strip() for n in md_notes if n.strip()]
    m_clean = [n.strip() for n in master_notes if n.strip()]
    
    if md_clean and not m_clean:
        empty_in_master_has_in_md.append({
            'table_id': tid,
            'chapter': t['chapter'],
            'section': t['section'],
            'line': t['start_line'],
            'md_notes_count': len(md_clean),
            'md_notes': md_clean
        })
    elif len(md_clean) > len(m_clean):
        # Master has fewer lines than MD. Check what was lost!
        lost_lines = []
        for line in md_clean:
            # Check if this line is in master_notes
            if not any(line in mn or mn in line for mn in m_clean):
                lost_lines.append(line)
        if lost_lines:
            fewer_lines_in_master.append({
                'table_id': tid,
                'chapter': t['chapter'],
                'section': t['section'],
                'line': t['start_line'],
                'md_count': len(md_clean),
                'master_count': len(m_clean),
                'lost_lines': lost_lines
            })

print(f"Tables where MD has notes but Master is completely EMPTY: {len(empty_in_master_has_in_md)}")
print(f"Tables where Master lost lines compared to MD: {len(fewer_lines_in_master)}")

print("\n--- 1. Tables where MD has notes but Master is EMPTY (First 15) ---")
for it in empty_in_master_has_in_md[:15]:
    print(f"[{it['table_id']} | line {it['line']}] {it['chapter']} | {it['section']} (MD {it['md_notes_count']} lines)")
    for n in it['md_notes'][:3]:
        print(f"     {n}")

print("\n--- 2. Tables where Master lost lines (Sample 10) ---")
for it in fewer_lines_in_master[:10]:
    print(f"[{it['table_id']} | line {it['line']}] {it['chapter']} | {it['section']} (MD: {it['md_count']} vs Master: {it['master_count']})")
    for l in it['lost_lines'][:3]:
        print(f"     LOST: {l}")