Before you begin
- Python 3.12 or newer
Set up your example
- Save the complete code as html-word-frequency.py in a working folder.
- Save the provided sample files in that same folder; run commands from that folder.
Sample files in this download:
- sample.html
- style.css
Download the ZIP, extract it, and open a terminal inside the extracted folder. The source file is html-word-frequency.py.
Run the script
python "html-word-frequency.py" --input "sample.html" --top 10 --min-length 4 --output "words.json"If your system uses python3 instead of python, use that command for Python examples. Keep paths containing spaces in quotes. Required inputs and optional settings are described in the code and setup steps.
What to expect
Rank word-like tokens from page text by frequency. useful and project appear repeatedly in the sample.
The complete script
#!/usr/bin/env python3
'Count frequent words in saved HTML'
import argparse
import json
from pathlib import Path
def write_json(path, value):
data = json.dumps(value, indent=2, ensure_ascii=False, allow_nan=False) + "\n"
with Path(path).open("x", encoding="utf-8", newline="") as handle:
handle.write(data)
from html.parser import HTMLParser
import re
import math
import urllib.parse
from collections import Counter, defaultdict
VOID={"area","base","br","col","embed","hr","img","input","link","meta","param","source","track","wbr"}
class Document(HTMLParser):
def __init__(self):
super().__init__(convert_charrefs=True)
self.nodes=[{"tag":"#document","attrs":{},"line":1,"parent":None,"content":[]}]
self.stack=[0]
self.comments=[]
def handle_starttag(self,tag,attrs):
if len(self.nodes)>=50000 or len(self.stack)>=256: raise ValueError("HTML node/depth limit exceeded.")
index=len(self.nodes)
self.nodes.append({"tag":tag,"attrs":dict(attrs),"line":self.getpos()[0],"parent":self.stack[-1],"content":[]})
self.nodes[self.stack[-1]]["content"].append(index)
if tag not in VOID: self.stack.append(index)
def handle_startendtag(self,tag,attrs):
self.handle_starttag(tag,attrs)
if tag not in VOID: self.stack.pop()
def handle_endtag(self,tag):
for pos in range(len(self.stack)-1,0,-1):
if self.nodes[self.stack[pos]]["tag"]==tag:
del self.stack[pos:]
break
def handle_data(self,data):
self.nodes[self.stack[-1]]["content"].append(data)
def handle_comment(self,data):
self.comments.append({"line":self.getpos()[0],"comment":data})
def read_document(path):
if Path(path).stat().st_size>5*1024*1024: raise ValueError("HTML input exceeds 5 MiB.")
doc=Document()
doc.feed(Path(path).read_text(encoding="utf-8-sig"))
doc.close()
return doc
def text(doc,index=0,skip=frozenset({"script","style","template"})):
node=doc.nodes[index]
if node["tag"] in skip: return ""
return " ".join(text(doc,p,skip) if isinstance(p,int) else p for p in node["content"])
def clean(value):
return " ".join(value.split())
def selected(doc,tags):
return [(i,n) for i,n in enumerate(doc.nodes) if n["tag"] in tags]
def ancestor(doc,index,tag):
parent=doc.nodes[index]["parent"]
while parent is not None:
if doc.nodes[parent]["tag"]==tag: return parent
parent=doc.nodes[parent]["parent"]
return None
def visible_text(doc):
bodies=selected(doc,{"body"})
return clean(text(doc,bodies[0][0] if bodies else 0, frozenset({"head","script","style","template"})))
def resources(doc):
result=[]
for i,n in enumerate(doc.nodes):
attrs=n["attrs"]
if n["tag"] in {"img","script","iframe","audio","video","source","track","embed","input"} and attrs.get("src"):
result.append({"tag":n["tag"],"attribute":"src","url":attrs["src"],"line":n["line"]})
if n["tag"]=="link" and "stylesheet" in (attrs.get("rel") or "").lower().split() and attrs.get("href"):
result.append({"tag":"link","attribute":"href","url":attrs["href"],"line":n["line"]})
if n["tag"]=="video" and attrs.get("poster"):
result.append({"tag":"video","attribute":"poster","url":attrs["poster"],"line":n["line"]})
return result
def main():
parser = argparse.ArgumentParser(description='Count frequent words in saved HTML')
parser.add_argument("--input",required=True,help="Local UTF-8 HTML file")
parser.add_argument("--output",required=True,help="New result path")
parser.add_argument("--top",type=int,default=25)
parser.add_argument("--min-length",type=int,default=3)
args = parser.parse_args()
doc=read_document(args.input)
if not 1<=args.top<=1000 or not 1<=args.min_length<=100: raise ValueError("Use top 1..1000 and min-length 1..100.")
words=[w.casefold() for w in re.findall(r"\b\w+\b",visible_text(doc)) if len(w)>=args.min_length]
write_json(args.output,[{"word":w,"count":n} for w,n in Counter(words).most_common(args.top)])
if __name__ == "__main__":
try:
main()
except (OSError, ValueError, KeyError, TypeError, ArithmeticError, IndexError, LookupError) as exc:
raise SystemExit(str(exc))
Open source as plain text · Download source and sample files
How it works
- The script parses only a saved local HTML file; scripts, styles, and external resources are never executed or fetched.
- Casefold merges capitalization; stop words are retained and the result is not an SEO ranking recommendation.
- useful and project appear repeatedly in the sample.
Make it your own
--min-length
Use --min-length 5 to omit shorter tokens.
Adapt the processing
Edit main's words comprehension to exclude an explicit stop-word set.
Change one setting at a time, run it again on sample data, and compare the output. The customization guide walks through this process.
If something goes wrong
Common words dominate.
Add a declared stop-word set only when it serves your analysis.
The destination already exists.
Choose a new --output path. The script deliberately refuses to replace an existing result.
Reference & next steps
These official references explain the underlying tools. The Scriptage example and guide are original instructional content. See verification notes for what has been checked and the limits of those checks.
Copies a template for your own notes. It does not send anything.