Read keyword frequencies in context

from pathlib import Path
import base64, gzip, hashlib, json, platform
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from IPython.display import display, HTML
import re
from collections import Counter
from IPython.display import display as _display
def display(value):
    if isinstance(value,pd.DataFrame):
        table=value.to_html(max_rows=12,escape=True)
        style="<style>.vibeit-readable-table{max-width:100%;overflow-x:auto}.vibeit-readable-table table{width:max-content;max-width:none;border-collapse:collapse}.vibeit-readable-table td,.vibeit-readable-table th{white-space:nowrap!important;word-break:normal!important;overflow-wrap:normal!important}</style>"
        return _display(HTML(style+'<div class="vibeit-readable-table">'+table+'</div>'))
    return _display(value)
RECIPE_ID='hu03-keyword-context'
OUTPUT_DIR=Path.cwd()/"results"/RECIPE_ID
plt.rcParams.update({"figure.dpi":120,"font.size":10,"axes.spines.top":False,"axes.spines.right":False,
                      "axes.prop_cycle":plt.cycler(color=["#18785f","#416aa6","#bc6541","#9974a5"])})
print("Python:",platform.python_version(),"; NumPy:",np.__version__,"; Pandas:",pd.__version__)
print("Working folder:",Path.cwd())
print("Core data mode: embedded snapshot; no network requests")

Python: 3.13.13 ; NumPy: 2.5.3 ; Pandas: 3.0.6
Working folder: /Users/zhiluo/Developer/vibeit-cookbook/cookbook/downloads
Core data mode: embedded snapshot; no network requests

def load_snapshot(recipe_id, expected_sha256, snapshot_key):
    """Read embedded data from this notebook, including a renamed copy."""
    for path in sorted(Path.cwd().glob("*.ipynb")):
        try:
            notebook = json.loads(path.read_text(encoding="utf-8"))
            metadata = notebook.get("metadata", {}).get("vibeit_cookbook", {})
            blob = metadata.get("snapshots", {}).get(snapshot_key)
            if metadata.get("recipe_id") != recipe_id or not blob:
                continue
            if blob.get("sha256") != expected_sha256:
                continue
            raw = gzip.decompress(base64.b64decode(blob["gzip_base64"], validate=True))
            if hashlib.sha256(raw).hexdigest() != expected_sha256:
                raise ValueError("Embedded snapshot checksum mismatch: " + snapshot_key)
            return json.loads(raw)
        except (OSError, json.JSONDecodeError):
            continue
    raise FileNotFoundError(
        "Save/open the complete .ipynb in the active working folder. "
        "The embedded snapshot could not be located for " + recipe_id)

SNAPSHOT_HASHES={'novels': 'c03763abb21b2380d2934b088f1c22a6d5a4675b17e23534e3f7343dc90084ec'}
def snapshot(key):
    return load_snapshot(RECIPE_ID,SNAPSHOT_HASHES[key],key)
print("Embedded snapshots:",list(SNAPSHOT_HASHES))

Embedded snapshots: ['novels']

def strip_ebook_wrapper(raw_text):
    """Return the analysis body while the unmodified source/complete license remain in metadata."""
    text=raw_text.replace('\r\n','\n').replace('\r','\n')
    start=re.search(r'\*\*\* START[^\n]*\*\*\*',text,re.I)
    end=re.search(r'\*\*\* END[^\n]*\*\*\*',text,re.I)
    if not start or not end or end.start()<=start.end():raise ValueError('Ebook start/end markers not found')
    return text[start.end():end.start()]

print("Method ready: strip_ebook_wrapper")

Method ready: strip_ebook_wrapper

def split_novel_chapters(body):
    """Declared two-ebook rule: repeated labels resolve to last heading, excluding their TOCs."""
    headings=list(re.finditer(r'(?im)^\s*chapter\s+([ivxlcdm]+|\d+)[.\]]*\s*$',body))
    if not headings:raise ValueError('No supported chapter headings')
    last={match.group(1).upper():match for match in headings}
    chosen=sorted(last.values(),key=lambda match:match.start());chapters=[]
    for j,match in enumerate(chosen):
        text=body[match.end():chosen[j+1].start() if j+1<len(chosen) else len(body)]
        text=re.sub(r'\[Illustration\b[^\]]*\]',' ',text,flags=re.I|re.S)
        text=text.replace('_','').replace('’',"'").strip()
        if len(text)<300:raise ValueError('Unexpected short chapter after declared TOC rule')
        chapters.append(dict(label=match.group(1).upper(),order=j+1,text=text))
    return chapters

print("Method ready: split_novel_chapters")

Method ready: split_novel_chapters

def tokenize_words(text):
    """Transparent English-letter tokenizer; punctuation/case decisions are part of the method."""
    return re.findall(r"[a-z]+(?:'[a-z]+)?",text.lower().replace('’',"'"))

print("Method ready: tokenize_words")

Method ready: tokenize_words

def keyword_contexts(tokens,query,window=7):
    query=query.lower()
    if not re.fullmatch(r"[a-z]+(?:'[a-z]+)?",query) or window<0:raise ValueError('Expected one normalized word and nonnegative window')
    rows=[]
    for index,word in enumerate(tokens):
        if word==query:
            rows.append(dict(token_index=index,before=' '.join(tokens[max(0,index-window):index]),keyword=word,
                after=' '.join(tokens[index+1:index+1+window])))
    return rows

print("Method ready: keyword_contexts")

Method ready: keyword_contexts

novels=snapshot("novels");corpus={};chapter_rows=[]
for book in novels["books"]:
 chapters=split_novel_chapters(strip_ebook_wrapper(book["raw_text"]))
 tokens=tokenize_words(" ".join(chapter["text"] for chapter in chapters))
 corpus[book["id"]]={"title":book["title"],"author":book["author"],"chapters":chapters,"tokens":tokens}
 for chapter in chapters:chapter_rows.append({"book_id":book["id"],"book":book["title"],"chapter":chapter["order"],"source_label":chapter["label"],"characters":len(chapter["text"]),"tokens":len(tokenize_words(chapter["text"])),"text":chapter["text"]})
chapter_table=pd.DataFrame(chapter_rows)
assert len(corpus[1342]["chapters"])==61 and len(corpus[84]["chapters"])==24
print(novels["source_notice"])
print("Declared chapter counts:",{b:len(corpus[b]["chapters"]) for b in corpus})
display(chapter_table.drop(columns="text").head(8))
queries={1342:"pride",84:"creature"};hits=[];counts=[]
for book_id,entry in corpus.items():
 for chapter in entry["chapters"]:
  tokens=tokenize_words(chapter["text"]);contexts=keyword_contexts(tokens,queries[book_id],7)
  for row in contexts:hits.append({"book_id":book_id,"book":entry["title"],"chapter":chapter["order"],**row})
  counts.append({"book_id":book_id,"book":entry["title"],"chapter":chapter["order"],"query":queries[book_id],"hits":len(contexts),"tokens":len(tokens),"per_1000_tokens":1000*len(contexts)/len(tokens)})
kwic=pd.DataFrame(hits);chapter_counts=pd.DataFrame(counts)
display(kwic[["book","chapter","before","keyword","after"]].head(10))

These source ebooks are available free of charge. Complete unmodified source texts and their licenses are included in this snapshot. Project Gutenberg is acknowledged as the transcription source; its name is not used as a course title or endorsement.
Declared chapter counts: {1342: 61, 84: 24}

fig,ax=plt.subplots(figsize=(10,4))
for book_id,part in chapter_counts.groupby("book_id"):ax.plot(part.chapter,part.per_1000_tokens,"o-",ms=3,label=part.book.iloc[0]+": "+queries[book_id])
ax.set(xlabel="Declared chapter order",ylabel="Query hits per 1000 tokens",title="Length-normalized exact-token occurrences");ax.legend(fontsize=8);plt.tight_layout();plt.show()

fig,axes=plt.subplots(2,1,figsize=(10,4))
for ax,(book_id,entry) in zip(axes,corpus.items()):
 indices=[i for i,w in enumerate(entry["tokens"]) if w==queries[book_id]]
 ax.scatter(np.asarray(indices)/len(entry["tokens"]),np.zeros(len(indices)),marker="|",s=100)
 ax.set(xlim=(0,1),yticks=[],xlabel="Relative processed-corpus position",title=entry["title"]+": "+queries[book_id])
plt.tight_layout();plt.show()

stop=set("the a an and to of in i he she it was is that with for".split())
fig,axes=plt.subplots(1,2,figsize=(11,4))
for ax,book_id in zip(axes,corpus):
 subset=kwic[kwic.book_id.eq(book_id)];neighbor=Counter()
 for row in subset.itertuples():neighbor.update(w for w in (row.before+" "+row.after).split() if w not in stop and w!=queries[book_id])
 top=neighbor.most_common(10);ax.barh([w for w,n in top][::-1],[n for w,n in top][::-1]);ax.set(xlabel="Counts in retrieved windows",title=corpus[book_id]["title"])
plt.tight_layout();plt.show()

assert chapter_counts.hits.sum()==len(kwic)
for book_id,entry in corpus.items():assert kwic.book_id.eq(book_id).sum()==entry["tokens"].count(queries[book_id])
OUTPUT_DIR.mkdir(parents=True,exist_ok=True);kwic.to_csv(OUTPUT_DIR/"all-keyword-contexts.csv",index=False)
chapter_counts.to_csv(OUTPUT_DIR/"normalized-chapter-query-counts.csv",index=False)
print("Exact match totals reconciled; all contexts exported:",len(kwic))

Exact match totals reconciled; all contexts exported: 86

RUN_ONLINE_UPDATE=False
if RUN_ONLINE_UPDATE:
    import requests
    try:
        response=requests.get('https://www.gutenberg.org/ebooks/1342',timeout=20)
        response.raise_for_status()
        print("Source:",response.url)
        print(response.text[:700])
    except requests.RequestException as error:
        print("Online extension unavailable:",type(error).__name__,str(error)[:160])
else:
    print("Online extension skipped; embedded core data unchanged")

Online extension skipped; embedded core data unchanged