Compare length-controlled lexical diversity

from pathlib import Path
import base64, gzip, hashlib, json, platform
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from IPython.display import display, HTML
import re
from collections import Counter
from IPython.display import display as _display
def display(value):
    if isinstance(value,pd.DataFrame):
        table=value.to_html(max_rows=12,escape=True)
        style="<style>.vibeit-readable-table{max-width:100%;overflow-x:auto}.vibeit-readable-table table{width:max-content;max-width:none;border-collapse:collapse}.vibeit-readable-table td,.vibeit-readable-table th{white-space:nowrap!important;word-break:normal!important;overflow-wrap:normal!important}</style>"
        return _display(HTML(style+'<div class="vibeit-readable-table">'+table+'</div>'))
    return _display(value)
RECIPE_ID='hu02-lexical-diversity'
OUTPUT_DIR=Path.cwd()/"results"/RECIPE_ID
plt.rcParams.update({"figure.dpi":120,"font.size":10,"axes.spines.top":False,"axes.spines.right":False,
                      "axes.prop_cycle":plt.cycler(color=["#18785f","#416aa6","#bc6541","#9974a5"])})
print("Python:",platform.python_version(),"; NumPy:",np.__version__,"; Pandas:",pd.__version__)
print("Working folder:",Path.cwd())
print("Core data mode: embedded snapshot; no network requests")

Python: 3.13.13 ; NumPy: 2.5.3 ; Pandas: 3.0.6
Working folder: /Users/zhiluo/Developer/vibeit-cookbook/cookbook/downloads
Core data mode: embedded snapshot; no network requests

def load_snapshot(recipe_id, expected_sha256, snapshot_key):
    """Read embedded data from this notebook, including a renamed copy."""
    for path in sorted(Path.cwd().glob("*.ipynb")):
        try:
            notebook = json.loads(path.read_text(encoding="utf-8"))
            metadata = notebook.get("metadata", {}).get("vibeit_cookbook", {})
            blob = metadata.get("snapshots", {}).get(snapshot_key)
            if metadata.get("recipe_id") != recipe_id or not blob:
                continue
            if blob.get("sha256") != expected_sha256:
                continue
            raw = gzip.decompress(base64.b64decode(blob["gzip_base64"], validate=True))
            if hashlib.sha256(raw).hexdigest() != expected_sha256:
                raise ValueError("Embedded snapshot checksum mismatch: " + snapshot_key)
            return json.loads(raw)
        except (OSError, json.JSONDecodeError):
            continue
    raise FileNotFoundError(
        "Save/open the complete .ipynb in the active working folder. "
        "The embedded snapshot could not be located for " + recipe_id)

SNAPSHOT_HASHES={'novels': 'c03763abb21b2380d2934b088f1c22a6d5a4675b17e23534e3f7343dc90084ec'}
def snapshot(key):
    return load_snapshot(RECIPE_ID,SNAPSHOT_HASHES[key],key)
print("Embedded snapshots:",list(SNAPSHOT_HASHES))

Embedded snapshots: ['novels']

def strip_ebook_wrapper(raw_text):
    """Return the analysis body while the unmodified source/complete license remain in metadata."""
    text=raw_text.replace('\r\n','\n').replace('\r','\n')
    start=re.search(r'\*\*\* START[^\n]*\*\*\*',text,re.I)
    end=re.search(r'\*\*\* END[^\n]*\*\*\*',text,re.I)
    if not start or not end or end.start()<=start.end():raise ValueError('Ebook start/end markers not found')
    return text[start.end():end.start()]

print("Method ready: strip_ebook_wrapper")

Method ready: strip_ebook_wrapper

def split_novel_chapters(body):
    """Declared two-ebook rule: repeated labels resolve to last heading, excluding their TOCs."""
    headings=list(re.finditer(r'(?im)^\s*chapter\s+([ivxlcdm]+|\d+)[.\]]*\s*$',body))
    if not headings:raise ValueError('No supported chapter headings')
    last={match.group(1).upper():match for match in headings}
    chosen=sorted(last.values(),key=lambda match:match.start());chapters=[]
    for j,match in enumerate(chosen):
        text=body[match.end():chosen[j+1].start() if j+1<len(chosen) else len(body)]
        text=re.sub(r'\[Illustration\b[^\]]*\]',' ',text,flags=re.I|re.S)
        text=text.replace('_','').replace('’',"'").strip()
        if len(text)<300:raise ValueError('Unexpected short chapter after declared TOC rule')
        chapters.append(dict(label=match.group(1).upper(),order=j+1,text=text))
    return chapters

print("Method ready: split_novel_chapters")

Method ready: split_novel_chapters

def tokenize_words(text):
    """Transparent English-letter tokenizer; punctuation/case decisions are part of the method."""
    return re.findall(r"[a-z]+(?:'[a-z]+)?",text.lower().replace('’',"'"))

print("Method ready: tokenize_words")

Method ready: tokenize_words

novels=snapshot("novels");corpus={};chapter_rows=[]
for book in novels["books"]:
 chapters=split_novel_chapters(strip_ebook_wrapper(book["raw_text"]))
 tokens=tokenize_words(" ".join(chapter["text"] for chapter in chapters))
 corpus[book["id"]]={"title":book["title"],"author":book["author"],"chapters":chapters,"tokens":tokens}
 for chapter in chapters:chapter_rows.append({"book_id":book["id"],"book":book["title"],"chapter":chapter["order"],"source_label":chapter["label"],"characters":len(chapter["text"]),"tokens":len(tokenize_words(chapter["text"])),"text":chapter["text"]})
chapter_table=pd.DataFrame(chapter_rows)
assert len(corpus[1342]["chapters"])==61 and len(corpus[84]["chapters"])==24
print(novels["source_notice"])
print("Declared chapter counts:",{b:len(corpus[b]["chapters"]) for b in corpus})
display(chapter_table.drop(columns="text").head(8))
summary=pd.DataFrame([{ "book":entry["title"],"tokens":len(entry["tokens"]),"types":len(set(entry["tokens"])),"whole_TTR":len(set(entry["tokens"]))/len(entry["tokens"])} for entry in corpus.values()])
display(summary)
fig,axes=plt.subplots(1,2,figsize=(10,4));axes[0].bar(summary.book,summary.tokens);axes[0].set(ylabel="Processed tokens",title="Unequal corpus lengths")
axes[1].bar(summary.book,summary.whole_TTR);axes[1].set(ylabel="Whole-corpus TTR",title="Length-dependent ratio")
for ax in axes:ax.tick_params(axis="x",rotation=15)
plt.tight_layout();plt.show()

These source ebooks are available free of charge. Complete unmodified source texts and their licenses are included in this snapshot. Project Gutenberg is acknowledged as the transcription source; its name is not used as a course title or endorsement.
Declared chapter counts: {1342: 61, 84: 24}

	book_id	book	chapter	source_label	characters	tokens
0	1342	Pride and Prejudice	1	I	4490	853
1	1342	Pride and Prejudice	2	II	4293	799
2	1342	Pride and Prejudice	3	III	9560	1704
3	1342	Pride and Prejudice	4	IV	5931	1064
4	1342	Pride and Prejudice	5	V	5246	962
5	1342	Pride and Prejudice	6	VI	13023	2353
6	1342	Pride and Prejudice	7	VII	11263	2000
7	1342	Pride and Prejudice	8	VIII	10993	1936

	book	tokens	types	whole_TTR
0	Pride and Prejudice	122185	6319	0.051717
1	Frankenstein	69610	6703	0.096294

rng=np.random.default_rng(42);lengths=[100,500,1000,2000,5000];records=[]
for book_id,entry in corpus.items():
 tokens=entry["tokens"]
 for length in lengths:
  for repeat in range(30):
   start=int(rng.integers(0,len(tokens)-length+1));window=tokens[start:start+length]
   records.append({"book":entry["title"],"window_tokens":length,"start_token":start,"TTR":len(set(window))/length})
windows=pd.DataFrame(records)
fig,ax=plt.subplots(figsize=(8,5))
for book,part in windows.groupby("book"):
 grouped=part.groupby("window_tokens").TTR.agg(["mean","std"]);ax.errorbar(grouped.index,grouped["mean"],yerr=grouped["std"],fmt="o-",label=book)
ax.set(xscale="log",xlabel="Contiguous window tokens",ylabel="TTR; bars = window SD",title="Length-controlled corpus comparison");ax.legend();plt.tight_layout();plt.show()

fig,ax=plt.subplots(figsize=(8,5))
for entry in corpus.values():
 frequency=np.array(sorted(Counter(entry["tokens"]).values(),reverse=True));ax.loglog(np.arange(1,len(frequency)+1),frequency/len(entry["tokens"]),label=entry["title"])
ax.set(xlabel="Within-corpus frequency rank",ylabel="Relative token frequency",title="Descriptive normalized rank-frequency curves");ax.legend();plt.tight_layout();plt.show()

assert windows.TTR.between(0,1,inclusive="right").all()
assert windows.groupby(["book","window_tokens"]).size().eq(30).all()
OUTPUT_DIR.mkdir(parents=True,exist_ok=True);windows.to_csv(OUTPUT_DIR/"all-matched-windows.csv",index=False)
summary.to_csv(OUTPUT_DIR/"length-dependent-corpus-counts.csv",index=False)
print("Matched-window checks passed; exports:",OUTPUT_DIR)

Matched-window checks passed; exports: /Users/zhiluo/Developer/vibeit-cookbook/cookbook/downloads/results/hu02-lexical-diversity

RUN_ONLINE_UPDATE=False
if RUN_ONLINE_UPDATE:
    import requests
    try:
        response=requests.get('https://www.gutenberg.org/ebooks/1342',timeout=20)
        response.raise_for_status()
        print("Source:",response.url)
        print(response.text[:700])
    except requests.RequestException as error:
        print("Online extension unavailable:",type(error).__name__,str(error)[:160])
else:
    print("Online extension skipped; embedded core data unchanged")

Online extension skipped; embedded core data unchanged