1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199 | #!/usr/bin/env python3
"""Generates a static source browser for a published repo.
gen-browse.py --repo <path> --app <name> --out <dir> \
[--clone-url <url>] [--css <url>]
Reads HEAD of --repo (tracked content only — the same bytes the published
mirror serves) and writes <out>/index.html + one page per file:
* code pages are syntax-highlighted with Pygments (pre-colored static
HTML — no JS); plain escaped <pre> when Pygments is missing
* markdown files are rendered to HTML (python-markdown); a directory's
README.md is also rendered inline under its listing, GitHub-style
* binary, oversized (>200 KB) and lockfile pages are stubs pointing at
the git clone
Styling comes from one shared stylesheet (--css, default the ecosystem's
CDN) so every repo browser looks the same and pages stay tiny.
"""
import argparse
import html
import posixpath
import re
import subprocess
from pathlib import Path
try:
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import TextLexer, guess_lexer_for_filename
from pygments.util import ClassNotFound
FORMATTER = HtmlFormatter(linenos="table")
except ImportError: # pragma: no cover
highlight = None
try:
import markdown
def md_html(text: str) -> str:
return markdown.markdown(
text,
extensions=["fenced_code", "tables", "codehilite"],
extension_configs={"codehilite": {"css_class": "highlight",
"guess_lang": False}},
)
except ImportError: # pragma: no cover
md_html = None
LINK_RE = re.compile(r'(<a\s[^>]*href=")([^"#?]+)([^"]*")')
def relink_md(rendered: str, base_dir: str, tracked: set) -> str:
"""Point relative links at browse pages: a markdown link to a tracked
file (README.md, src/foo.clj) 404s as-is — only <path>.html exists."""
def fix(m):
url = m.group(2)
if "://" in url or url.startswith(("/", "mailto:", "#")):
return m.group(0)
target = posixpath.normpath(posixpath.join(base_dir, url))
if target in tracked:
return m.group(1) + url + ".html" + m.group(3)
if (target + "/README.md" in tracked) or any(
t.startswith(target + "/") for t in tracked
): # directory link -> its listing
return m.group(1) + url.rstrip("/") + "/index.html" + m.group(3)
return m.group(0)
return LINK_RE.sub(fix, rendered)
GENERATED = {"package-lock.json", "yarn.lock", "pnpm-lock.yaml", "go.sum"}
MAX_RENDER = 200_000 # bytes; bigger files get a stub page
ap = argparse.ArgumentParser()
ap.add_argument("--repo", required=True)
ap.add_argument("--app", required=True)
ap.add_argument("--out", required=True)
ap.add_argument("--clone-url", default=None)
ap.add_argument("--css", default="https://cdn.cod-sursa.ro/browse.v1.css")
args = ap.parse_args()
APP = args.app
OUT = Path(args.out)
CLONE = args.clone_url or f"https://ird.cod-sursa.ro/{APP}.git"
HEAD_TAG = (
f'<!doctype html><meta charset="utf-8">'
f'<meta name="viewport" content="width=device-width,initial-scale=1">'
f'<link rel="stylesheet" href="{args.css}">'
)
def sh(*a: str) -> bytes:
return subprocess.check_output(a, cwd=args.repo)
def crumbs(path: str, is_file: bool) -> str:
parts = path.split("/") if path else []
dirparts = parts[:-1] if is_file else parts
depth = len(dirparts)
out = [f'<a href="{"../" * depth or "./"}index.html">{html.escape(APP)}</a>']
for i, p in enumerate(dirparts):
out.append(f'<a href="{"../" * (depth - 1 - i)}index.html">{html.escape(p)}</a>')
if is_file and parts:
out.append(html.escape(parts[-1]))
return " / ".join(out)
def footer(depth: int) -> str:
up = "../" * depth
return (
f'<p class="muted">static mirror of HEAD · <a href="{up}../index.html">about</a>'
f' · clone: <code>git clone {html.escape(CLONE)}</code></p>'
)
files = [] # (path, size)
for line in sh("git", "ls-tree", "-r", "--long", "HEAD").decode().splitlines():
meta, path = line.split("\t", 1)
size = meta.split()[3]
files.append((path, int(size) if size.isdigit() else 0))
tracked = {p for p, _ in files}
dirs: dict[str, list] = {"": []}
for path, size in files:
parts = path.split("/")
for i in range(len(parts) - 1):
d = "/".join(parts[: i + 1])
parent = "/".join(parts[:i])
dirs.setdefault(d, [])
if ("d", parts[i]) not in [(t, n) for t, n, *_ in dirs[parent]]:
dirs[parent].append(("d", parts[i]))
dirs["/".join(parts[:-1])].append(("f", parts[-1], size, path))
for d, items in dirs.items():
out = OUT / d / "index.html"
out.parent.mkdir(parents=True, exist_ok=True)
depth = d.count("/") + (1 if d else 0)
rows = []
if d:
rows.append('<li><a href="../index.html">../</a></li>')
readme = None
for it in sorted(items, key=lambda x: (x[0] != "d", x[1].lower())):
if it[0] == "d":
rows.append(f'<li><a href="{html.escape(it[1])}/index.html">{html.escape(it[1])}/</a></li>')
else:
_, name, size, p = it
rows.append(
f'<li><a href="{html.escape(name)}.html">{html.escape(name)}'
f'<span class="sz">{size:,} B</span></a></li>'
)
if name.lower() == "readme.md":
readme = p
inline_md = ""
if readme and md_html is not None:
text = sh("git", "show", f"HEAD:{readme}").decode("utf-8", "replace")
inline_md = ('<div class="md" style="margin-top:16px">'
+ relink_md(md_html(text), d, tracked) + '</div>')
title = html.escape(d or APP)
out.write_text(
f"{HEAD_TAG}<title>{title}</title><main>"
f'<div class="crumb">{crumbs(d, False)}</div>'
f'<h1>{title}/</h1><ul class="tree">{"".join(rows)}</ul>'
f"{inline_md}{footer(depth)}</main>"
)
for path, size in files:
out = OUT / (path + ".html")
out.parent.mkdir(parents=True, exist_ok=True)
depth = path.count("/")
blob = sh("git", "show", f"HEAD:{path}")
name = path.rsplit("/", 1)[-1]
if name in GENERATED:
body = f'<p class="muted">generated file · {size:,} bytes · not rendered — see the git clone</p>'
elif b"\0" in blob[:8000]:
body = f'<p class="muted">binary file · {size:,} bytes</p>'
elif size > MAX_RENDER:
body = f'<p class="muted">file too large to render ({size:,} bytes) — see the git clone</p>'
else:
text = blob.decode("utf-8", "replace")
if name.lower().endswith((".md", ".markdown")) and md_html is not None:
base = path.rsplit('/', 1)[0] if '/' in path else ''
body = '<div class="md">' + relink_md(md_html(text), base, tracked) + '</div>'
elif highlight is not None:
try:
lexer = guess_lexer_for_filename(path, text)
except ClassNotFound:
lexer = TextLexer()
body = highlight(text, lexer, FORMATTER)
else:
body = f'<pre class="plain">{html.escape(text)}</pre>'
out.write_text(
f"{HEAD_TAG}<title>{html.escape(name)}</title><main>"
f'<div class="crumb">{crumbs(path, True)}</div>{body}{footer(depth)}</main>'
)
print(f"browse[{APP}]: {len(files)} files, {len(dirs)} dirs -> {OUT}")
|