Audit a name-mention cooccurrence graph

from pathlib import Path
import base64, gzip, hashlib, json, platform
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from IPython.display import display, HTML
import re
from collections import Counter
import networkx as nx
from IPython.display import display as _display
def display(value):
    if isinstance(value,pd.DataFrame):
        table=value.to_html(max_rows=12,escape=True)
        style="<style>.vibeit-readable-table{max-width:100%;overflow-x:auto}.vibeit-readable-table table{width:max-content;max-width:none;border-collapse:collapse}.vibeit-readable-table td,.vibeit-readable-table th{white-space:nowrap!important;word-break:normal!important;overflow-wrap:normal!important}</style>"
        return _display(HTML(style+'<div class="vibeit-readable-table">'+table+'</div>'))
    return _display(value)
RECIPE_ID='hu04-mention-network'
OUTPUT_DIR=Path.cwd()/"results"/RECIPE_ID
plt.rcParams.update({"figure.dpi":120,"font.size":10,"axes.spines.top":False,"axes.spines.right":False,
                      "axes.prop_cycle":plt.cycler(color=["#18785f","#416aa6","#bc6541","#9974a5"])})
print("Python:",platform.python_version(),"; NumPy:",np.__version__,"; Pandas:",pd.__version__)
print("Working folder:",Path.cwd())
print("Core data mode: embedded snapshot; no network requests")

Python: 3.13.13 ; NumPy: 2.5.3 ; Pandas: 3.0.6
Working folder: /Users/zhiluo/Developer/vibeit-cookbook/cookbook/downloads
Core data mode: embedded snapshot; no network requests

def load_snapshot(recipe_id, expected_sha256, snapshot_key):
    """Read embedded data from this notebook, including a renamed copy."""
    for path in sorted(Path.cwd().glob("*.ipynb")):
        try:
            notebook = json.loads(path.read_text(encoding="utf-8"))
            metadata = notebook.get("metadata", {}).get("vibeit_cookbook", {})
            blob = metadata.get("snapshots", {}).get(snapshot_key)
            if metadata.get("recipe_id") != recipe_id or not blob:
                continue
            if blob.get("sha256") != expected_sha256:
                continue
            raw = gzip.decompress(base64.b64decode(blob["gzip_base64"], validate=True))
            if hashlib.sha256(raw).hexdigest() != expected_sha256:
                raise ValueError("Embedded snapshot checksum mismatch: " + snapshot_key)
            return json.loads(raw)
        except (OSError, json.JSONDecodeError):
            continue
    raise FileNotFoundError(
        "Save/open the complete .ipynb in the active working folder. "
        "The embedded snapshot could not be located for " + recipe_id)

SNAPSHOT_HASHES={'novels': 'c03763abb21b2380d2934b088f1c22a6d5a4675b17e23534e3f7343dc90084ec'}
def snapshot(key):
    return load_snapshot(RECIPE_ID,SNAPSHOT_HASHES[key],key)
print("Embedded snapshots:",list(SNAPSHOT_HASHES))

Embedded snapshots: ['novels']

def strip_ebook_wrapper(raw_text):
    """Return the analysis body while the unmodified source/complete license remain in metadata."""
    text=raw_text.replace('\r\n','\n').replace('\r','\n')
    start=re.search(r'\*\*\* START[^\n]*\*\*\*',text,re.I)
    end=re.search(r'\*\*\* END[^\n]*\*\*\*',text,re.I)
    if not start or not end or end.start()<=start.end():raise ValueError('Ebook start/end markers not found')
    return text[start.end():end.start()]

print("Method ready: strip_ebook_wrapper")

Method ready: strip_ebook_wrapper

def split_novel_chapters(body):
    """Declared two-ebook rule: repeated labels resolve to last heading, excluding their TOCs."""
    headings=list(re.finditer(r'(?im)^\s*chapter\s+([ivxlcdm]+|\d+)[.\]]*\s*$',body))
    if not headings:raise ValueError('No supported chapter headings')
    last={match.group(1).upper():match for match in headings}
    chosen=sorted(last.values(),key=lambda match:match.start());chapters=[]
    for j,match in enumerate(chosen):
        text=body[match.end():chosen[j+1].start() if j+1<len(chosen) else len(body)]
        text=re.sub(r'\[Illustration\b[^\]]*\]',' ',text,flags=re.I|re.S)
        text=text.replace('_','').replace('’',"'").strip()
        if len(text)<300:raise ValueError('Unexpected short chapter after declared TOC rule')
        chapters.append(dict(label=match.group(1).upper(),order=j+1,text=text))
    return chapters

print("Method ready: split_novel_chapters")

Method ready: split_novel_chapters

def tokenize_words(text):
    """Transparent English-letter tokenizer; punctuation/case decisions are part of the method."""
    return re.findall(r"[a-z]+(?:'[a-z]+)?",text.lower().replace('’',"'"))

print("Method ready: tokenize_words")

Method ready: tokenize_words

def mention_cooccurrence(paragraphs,patterns,minimum_count=3):
    """Paragraph-level surface-pattern cooccurrence; preserve all declared entity nodes."""
    if minimum_count<1:raise ValueError('Minimum cooccurrence count must be positive')
    labels=list(patterns);compiled={name:re.compile(pattern,re.I) for name,pattern in patterns.items()}
    matrix=np.zeros((len(labels),len(labels)),dtype=int);presence=np.zeros(len(labels),dtype=int)
    for paragraph in paragraphs:
        found=np.array([bool(compiled[name].search(paragraph)) for name in labels]);presence+=found
        matrix+=np.outer(found,found).astype(int)
    np.fill_diagonal(matrix,0);graph=nx.Graph();graph.add_nodes_from(labels)
    for i in range(len(labels)):
        for j in range(i+1,len(labels)):
            if matrix[i,j]>=minimum_count:graph.add_edge(labels[i],labels[j],paragraph_count=int(matrix[i,j]))
    return dict(graph=graph,labels=labels,matrix=matrix,paragraph_presence=presence)

print("Method ready: mention_cooccurrence")

Method ready: mention_cooccurrence

novels=snapshot("novels");corpus={};chapter_rows=[]
for book in novels["books"]:
 chapters=split_novel_chapters(strip_ebook_wrapper(book["raw_text"]))
 tokens=tokenize_words(" ".join(chapter["text"] for chapter in chapters))
 corpus[book["id"]]={"title":book["title"],"author":book["author"],"chapters":chapters,"tokens":tokens}
 for chapter in chapters:chapter_rows.append({"book_id":book["id"],"book":book["title"],"chapter":chapter["order"],"source_label":chapter["label"],"characters":len(chapter["text"]),"tokens":len(tokenize_words(chapter["text"])),"text":chapter["text"]})
chapter_table=pd.DataFrame(chapter_rows)
assert len(corpus[1342]["chapters"])==61 and len(corpus[84]["chapters"])==24
print(novels["source_notice"])
print("Declared chapter counts:",{b:len(corpus[b]["chapters"]) for b in corpus})
display(chapter_table.drop(columns="text").head(8))
patterns={"Elizabeth / Lizzy":r"\b(?:elizabeth|lizzy|eliza)\b","Jane":r"\bjane\b","Darcy surname":r"\bdarcy\b","Bingley surname":r"\bbingley\b","Mr Bennet":r"\bmr\.?\s+bennet\b","Mrs Bennet":r"\bmrs\.?\s+bennet\b","Lydia":r"\blydia\b","Wickham":r"\bwickham\b"}
paragraphs=[part for chapter in corpus[1342]["chapters"] for part in re.split(r"\n\s*\n",chapter["text"]) if part.strip()]
MINIMUM=3;network=mention_cooccurrence(paragraphs,patterns,MINIMUM);graph=network["graph"]
print("Processed paragraphs:",len(paragraphs),"; declared entities:",len(patterns),"; minimum shared paragraphs:",MINIMUM)
display(pd.DataFrame({"pattern":list(patterns),"regex":list(patterns.values()),"paragraph_presence":network["paragraph_presence"]}))

These source ebooks are available free of charge. Complete unmodified source texts and their licenses are included in this snapshot. Project Gutenberg is acknowledged as the transcription source; its name is not used as a course title or endorsement.
Declared chapter counts: {1342: 61, 84: 24}

Processed paragraphs: 2096 ; declared entities: 8 ; minimum shared paragraphs: 3

position=nx.spring_layout(graph,seed=42,weight="paragraph_count")
fig,ax=plt.subplots(figsize=(9,6));width=[1+np.log1p(graph[a][b]["paragraph_count"]) for a,b in graph.edges()]
nx.draw_networkx(graph,pos=position,ax=ax,node_size=1500,font_size=8,width=width,node_color="#b4d4ed")
ax.set_title("Surface-pattern cooccurrence; threshold = 3 shared paragraphs");ax.axis("off");plt.tight_layout();plt.show()

fig,ax=plt.subplots(figsize=(8,6));im=ax.imshow(network["matrix"],cmap="Blues")
ax.set_xticks(range(8),network["labels"],rotation=55,ha="right",fontsize=8);ax.set_yticks(range(8),network["labels"],fontsize=8)
fig.colorbar(im,ax=ax,label="Shared processed paragraphs");ax.set_title("Declared alias-rule counts");plt.tight_layout();plt.show()

sensitivity=[]
for minimum in [1,3,5,10]:
 trial=mention_cooccurrence(paragraphs,patterns,minimum)["graph"]
 sensitivity.append({"minimum_shared_paragraphs":minimum,"nodes":trial.number_of_nodes(),"edges":trial.number_of_edges(),"isolates":len(list(nx.isolates(trial)))})
sensitivity=pd.DataFrame(sensitivity)
fig,ax=plt.subplots(figsize=(7,4));ax.plot(sensitivity.minimum_shared_paragraphs,sensitivity.edges,"o-")
ax.set(xlabel="Minimum shared paragraphs",ylabel="Retained edges",title="Threshold-sensitive cooccurrence representation");plt.tight_layout();plt.show()

assert graph.number_of_edges()>0 and network["paragraph_presence"].sum()>0
assert np.array_equal(network["matrix"],network["matrix"].T) and np.all(np.diag(network["matrix"])==0)
assert set(graph)==set(patterns) and sensitivity.nodes.eq(8).all() and np.all(np.diff(sensitivity.edges)<=0)
OUTPUT_DIR.mkdir(parents=True,exist_ok=True);nx.write_graphml(graph,OUTPUT_DIR/"surface-mention-cooccurrence.graphml")
pd.DataFrame(network["matrix"],index=network["labels"],columns=network["labels"]).to_csv(OUTPUT_DIR/"shared-paragraph-counts.csv")
sensitivity.to_csv(OUTPUT_DIR/"threshold-sensitivity.csv",index=False)
(OUTPUT_DIR/"alias-rules.json").write_text(json.dumps(patterns,indent=2))
print("Count/node checks passed; interpretation remains conditional on alias rules")

Count/node checks passed; interpretation remains conditional on alias rules

RUN_ONLINE_UPDATE=False
if RUN_ONLINE_UPDATE:
    import requests
    try:
        response=requests.get('https://www.gutenberg.org/ebooks/1342',timeout=20)
        response.raise_for_status()
        print("Source:",response.url)
        print(response.text[:700])
    except requests.RequestException as error:
        print("Online extension unavailable:",type(error).__name__,str(error)[:160])
else:
    print("Online extension skipped; embedded core data unchanged")

Online extension skipped; embedded core data unchanged