113 lines
3.4 KiB
Python
113 lines
3.4 KiB
Python
"""
|
|
Remove inline `#` comments from app/, keeping docstrings and directives.
|
|
|
|
Uses `tokenize` rather than a regular expression. A regex cannot tell a comment
|
|
from a `#` inside a string literal, and this codebase has plenty of both —
|
|
colour codes, URL fragments, f-strings. Getting that wrong corrupts a file
|
|
silently, and "silently" is the whole problem with mass edits.
|
|
|
|
Docstrings are untouched: they are STRING tokens, not COMMENT tokens, so the
|
|
tokenizer never offers them up. That also means the 62 endpoint descriptions in
|
|
openapi.json survive, because those come from handler docstrings.
|
|
|
|
Kept, because they are instructions to a tool rather than prose:
|
|
#! shebang
|
|
# noqa flake8 / ruff
|
|
# pragma: coverage
|
|
# type: mypy
|
|
# fmt: black
|
|
# isort: isort
|
|
# -*- coding source encoding
|
|
"""
|
|
|
|
import ast
|
|
import io
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
import tokenize
|
|
|
|
DIRECTIVE = re.compile(
|
|
r"^#\s*(!|noqa|pragma:|type:\s*ignore|fmt:\s*(on|off)|isort:|nosec|mypy:|"
|
|
r"pylint:|ruff:|-\*-\s*coding|coding[:=])",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
ROOT = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else "app")
|
|
|
|
removed = kept = touched = 0
|
|
failures = []
|
|
|
|
for path in sorted(ROOT.rglob("*.py")):
|
|
if "__pycache__" in str(path):
|
|
continue
|
|
|
|
original = path.read_text(encoding="utf-8")
|
|
lines = original.splitlines(keepends=True)
|
|
|
|
# (row, col, text) for every comment, so edits can be applied right-to-left
|
|
# and bottom-up without invalidating positions.
|
|
comments = []
|
|
try:
|
|
for tok in tokenize.generate_tokens(io.StringIO(original).readline):
|
|
if tok.type == tokenize.COMMENT:
|
|
comments.append(tok)
|
|
except tokenize.TokenError as exc:
|
|
failures.append(f"{path}: tokenizer: {exc}")
|
|
continue
|
|
|
|
if not comments:
|
|
continue
|
|
|
|
drop_whole_line = set()
|
|
for tok in sorted(comments, key=lambda t: (t.start[0], t.start[1]), reverse=True):
|
|
if DIRECTIVE.match(tok.string.strip()):
|
|
kept += 1
|
|
continue
|
|
|
|
row, col = tok.start
|
|
line = lines[row - 1]
|
|
before = line[:col]
|
|
|
|
if before.strip() == "":
|
|
# The line is nothing but a comment — take the line out entirely
|
|
# rather than leaving a blank behind.
|
|
drop_whole_line.add(row)
|
|
else:
|
|
# Trailing comment: keep the code, drop the comment and the
|
|
# whitespace that was only there to separate them.
|
|
newline = "\n" if line.endswith("\n") else ""
|
|
lines[row - 1] = before.rstrip() + newline
|
|
removed += 1
|
|
|
|
if drop_whole_line:
|
|
lines = [ln for i, ln in enumerate(lines, 1) if i not in drop_whole_line]
|
|
|
|
updated = "".join(lines)
|
|
|
|
# Removing a comment block can leave a run of blank lines behind. Two is
|
|
# the most PEP 8 ever calls for.
|
|
updated = re.sub(r"\n{4,}", "\n\n\n", updated)
|
|
|
|
if updated == original:
|
|
continue
|
|
|
|
try:
|
|
compile(updated, str(path), "exec")
|
|
ast.parse(updated)
|
|
except SyntaxError as exc:
|
|
failures.append(f"{path}: would not parse after strip: {exc}")
|
|
continue
|
|
|
|
path.write_text(updated, encoding="utf-8")
|
|
touched += 1
|
|
|
|
print(f"files rewritten : {touched}")
|
|
print(f"comments removed : {removed}")
|
|
print(f"directives kept : {kept}")
|
|
if failures:
|
|
print(f"\nFAILURES ({len(failures)}) — these files were left alone:")
|
|
for f in failures:
|
|
print(" ", f)
|
|
sys.exit(1)
|