Update build scripts and enhance development workflow: Revise package.json to streamline build commands, replacing deprecated scripts with new chapter-specific build commands. Update CONTRIBUTING.md to clarify local development instructions and improve chapter sync processes. Enhance README.md with updated serving and build commands for better user guidance. Add new types for Bun in devDependencies and update the feed and sitemap to reflect recent content changes.

This commit is contained in:
Hananoshika Yomaru 2026-05-22 11:42:00 -07:00
parent 61963350e7
commit 5ac8e330ca
19 changed files with 501 additions and 125 deletions

View file

@ -0,0 +1,137 @@
import { existsSync } from "node:fs";
import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
import { basename, dirname, join } from "node:path";
import { build } from "vite";
import solid from "vite-plugin-solid";
import { fingerprintPaths } from "./build-change-detector";
const ROOT = join(import.meta.dir, "..");
const CONTENT = join(ROOT, "content");
const OUTPUT_ROOT = join(ROOT, "themes/openfreebooks/static/js/chapter-widgets");
const CACHE_PATH = join(ROOT, "data/_generated/chapter-widgets-cache.json");
type Chapter = { subject: string; slug: string; dir: string; key: string };
type CacheFile = { version: 1; chapters: Record<string, string> };
async function discoverChapters(): Promise<Chapter[]> {
const output: Chapter[] = [];
const subjects = await readdir(CONTENT, { withFileTypes: true });
for (const subjectEntry of subjects) {
if (!subjectEntry.isDirectory() || subjectEntry.name.startsWith(".")) continue;
const subject = subjectEntry.name;
const subjectDir = join(CONTENT, subject);
const chapterEntries = await readdir(subjectDir, { withFileTypes: true });
for (const chapterEntry of chapterEntries) {
if (!chapterEntry.isDirectory() || chapterEntry.name.startsWith(".")) continue;
const slug = chapterEntry.name;
const dir = join(subjectDir, slug);
const indexPath = join(dir, "_index.md");
if (!existsSync(indexPath)) continue;
const indexRaw = await readFile(indexPath, "utf8");
if (!/template\s*=\s*["']chapter\.html["']/.test(indexRaw)) continue;
output.push({ subject, slug, dir, key: `${subject}/${slug}` });
}
}
return output;
}
async function listWidgets(chapter: Chapter): Promise<string[]> {
const widgetsDir = join(chapter.dir, "widgets");
if (!existsSync(widgetsDir)) return [];
const entries = await readdir(widgetsDir, { withFileTypes: true });
return entries
.filter((entry) => entry.isFile() && entry.name.endsWith(".tsx") && !entry.name.startsWith("_"))
.map((entry) => basename(entry.name, ".tsx"))
.sort();
}
async function chapterFingerprint(chapter: Chapter): Promise<string> {
return fingerprintPaths(ROOT, [join(chapter.dir, "widgets")]);
}
async function readCache(): Promise<CacheFile> {
if (!existsSync(CACHE_PATH)) return { version: 1, chapters: {} };
try {
const parsed = JSON.parse(await readFile(CACHE_PATH, "utf8")) as CacheFile;
if (parsed.version !== 1) return { version: 1, chapters: {} };
return parsed;
} catch {
return { version: 1, chapters: {} };
}
}
async function buildWidget(chapter: Chapter, widget: string) {
const entry = join(chapter.dir, "widgets", `${widget}.tsx`);
const outDir = join(OUTPUT_ROOT, chapter.subject, chapter.slug);
await mkdir(outDir, { recursive: true });
await build({
configFile: false,
plugins: [solid()],
publicDir: false,
resolve: {
alias: {
"@ofb/katex": join(ROOT, "frontend/src/lib/katex.ts"),
},
},
build: {
outDir,
emptyOutDir: false,
lib: {
entry,
formats: ["es"],
fileName: () => `${widget}.js`,
},
rollupOptions: {
treeshake: false,
output: {
chunkFileNames: "assets/[name]-[hash].js",
assetFileNames: "assets/[name]-[hash][extname]",
},
},
},
});
}
function parseTargets(args: string[]): { full: boolean; chapterKeys: string[] } {
const full = args.includes("--full");
const chapterKeys = args.filter((arg) => !arg.startsWith("--"));
return { full, chapterKeys };
}
async function main() {
const { full, chapterKeys } = parseTargets(process.argv.slice(2));
const chapters = await discoverChapters();
const chapterByKey = new Map(chapters.map((chapter) => [chapter.key, chapter]));
const targets =
chapterKeys.length === 0
? chapters
: chapterKeys
.map((key) => chapterByKey.get(key))
.filter((chapter): chapter is Chapter => Boolean(chapter));
const prev = await readCache();
const next: CacheFile = { version: 1, chapters: { ...prev.chapters } };
for (const chapter of targets) {
const fingerprint = await chapterFingerprint(chapter);
const unchanged = !full && prev.chapters[chapter.key] === fingerprint;
if (unchanged) {
console.log(`Unchanged widgets ${chapter.key} (skipped)`);
continue;
}
const widgets = await listWidgets(chapter);
const chapterOutDir = join(OUTPUT_ROOT, chapter.subject, chapter.slug);
await rm(chapterOutDir, { recursive: true, force: true });
for (const widget of widgets) {
await buildWidget(chapter, widget);
}
next.chapters[chapter.key] = fingerprint;
console.log(`Built widgets ${chapter.key} (${widgets.length})`);
}
await mkdir(dirname(CACHE_PATH), { recursive: true });
await writeFile(CACHE_PATH, `${JSON.stringify(next, null, 2)}\n`);
}
main();

186
scripts/build-chapter.ts Normal file
View file

@ -0,0 +1,186 @@
import { spawn } from "node:child_process";
import { existsSync } from "node:fs";
import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { createInterface } from "node:readline/promises";
import { stdin as input, stdout as output } from "node:process";
import { $ } from "bun";
import { fingerprintPaths } from "./build-change-detector";
const ROOT = join(import.meta.dir, "..");
const CONTENT = join(ROOT, "content");
const BUNDLE_PATH = join(ROOT, "themes/openfreebooks/static/js/bundle.js");
const BUILD_CACHE_PATH = join(ROOT, "data/_generated/build-chapter-cache.json");
type Chapter = { key: string; subject: string; slug: string };
type BuildCache = { version: 1; shellFingerprint: string };
async function runStep(command: string) {
const result = await $`bash -lc ${command}`;
if (result.exitCode !== 0) process.exit(result.exitCode || 1);
}
async function discoverChapters(): Promise<Chapter[]> {
const output: Chapter[] = [];
const subjects = await readdir(CONTENT, { withFileTypes: true });
for (const subjectEntry of subjects) {
if (!subjectEntry.isDirectory() || subjectEntry.name.startsWith(".")) continue;
const subject = subjectEntry.name;
const subjectDir = join(CONTENT, subject);
const chapterEntries = await readdir(subjectDir, { withFileTypes: true });
for (const chapterEntry of chapterEntries) {
if (!chapterEntry.isDirectory() || chapterEntry.name.startsWith(".")) continue;
const slug = chapterEntry.name;
const indexPath = join(subjectDir, slug, "_index.md");
if (!existsSync(indexPath)) continue;
const raw = await readFile(indexPath, "utf8");
if (!/template\s*=\s*["']chapter\.html["']/.test(raw)) continue;
output.push({ key: `${subject}/${slug}`, subject, slug });
}
}
return output.sort((a, b) => a.key.localeCompare(b.key));
}
function patternToRegex(pattern: string): RegExp {
const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
return new RegExp(`^${escaped}$`);
}
function resolvePatterns(chapters: Chapter[], patterns: string[]): Chapter[] {
const results = new Map<string, Chapter>();
for (const pattern of patterns) {
const regex = patternToRegex(pattern);
const matches = chapters.filter((chapter) => regex.test(chapter.key));
for (const chapter of matches) {
results.set(chapter.key, chapter);
}
}
return [...results.values()].sort((a, b) => a.key.localeCompare(b.key));
}
async function changedChapterKeys(chapterKeys: Set<string>): Promise<string[]> {
const trackedDiff = await $`git --no-pager diff --name-only HEAD -- content`.text();
const untracked = await $`git --no-pager ls-files --others --exclude-standard -- content`.text();
const files = new Set(
[...trackedDiff.split("\n"), ...untracked.split("\n")]
.map((line) => line.trim())
.filter((line) => line.length > 0)
);
const keys = new Set<string>();
for (const file of files) {
const match = file.match(/^content\/([^/]+)\/([^/]+)\//);
if (!match) continue;
const key = `${match[1]}/${match[2]}`;
if (chapterKeys.has(key)) keys.add(key);
}
return [...keys].sort();
}
async function confirmDetectedChanges(chapters: Chapter[], frontendChanged: boolean): Promise<boolean> {
if (chapters.length > 0) {
console.log("[build:chapter] Auto-detected changed chapters:");
for (const chapter of chapters) {
console.log(` - ${chapter.key}`);
}
}
if (frontendChanged) {
console.log("[build:chapter] Auto-detected frontend/site-shell changes.");
}
const rl = createInterface({ input, output });
const answer = (await rl.question("Proceed? [y/N] ")).trim().toLowerCase();
rl.close();
return answer === "y" || answer === "yes";
}
async function readBuildCache(): Promise<BuildCache | null> {
if (!existsSync(BUILD_CACHE_PATH)) return null;
try {
const parsed = JSON.parse(await readFile(BUILD_CACHE_PATH, "utf8")) as BuildCache;
if (parsed.version !== 1) return null;
return parsed;
} catch {
return null;
}
}
async function writeBuildCache(cache: BuildCache) {
await mkdir(dirname(BUILD_CACHE_PATH), { recursive: true });
await writeFile(BUILD_CACHE_PATH, `${JSON.stringify(cache, null, 2)}\n`);
}
async function shellFingerprint(): Promise<string> {
return fingerprintPaths(ROOT, [join(ROOT, "frontend"), join(ROOT, "vite.config.ts")]);
}
function runPagefindInBackground() {
const command =
'pagefind --site public --glob "**/index.html" && rm -rf static/pagefind && cp -R public/pagefind static/pagefind';
const child = spawn("bash", ["-lc", command], {
cwd: ROOT,
detached: true,
stdio: "ignore",
});
child.unref();
console.log("[build:chapter] Pagefind indexing started in background.");
}
async function main() {
const patterns = process.argv.slice(2).filter((arg) => !arg.startsWith("--"));
const chapters = await discoverChapters();
const chapterByKey = new Map(chapters.map((chapter) => [chapter.key, chapter]));
const previousBuild = await readBuildCache();
const currentShellFingerprint = await shellFingerprint();
const frontendChanged =
!existsSync(BUNDLE_PATH) || previousBuild?.shellFingerprint !== currentShellFingerprint;
let targets: Chapter[] = [];
if (patterns.length > 0) {
targets = resolvePatterns(chapters, patterns);
if (targets.length === 0) {
console.error(`[build:chapter] No chapters matched patterns: ${patterns.join(", ")}`);
process.exit(1);
}
} else {
if (!input.isTTY || !output.isTTY) {
console.error("[build:chapter] Non-interactive mode requires explicit chapter arguments.");
process.exit(1);
}
const changedKeys = await changedChapterKeys(new Set(chapters.map((chapter) => chapter.key)));
targets = changedKeys.map((key) => chapterByKey.get(key)).filter((c): c is Chapter => Boolean(c));
if (targets.length === 0 && !frontendChanged) {
console.log("[build:chapter] No changed chapters or frontend changes detected.");
return;
}
const confirmed = await confirmDetectedChanges(targets, frontendChanged);
if (!confirmed) {
console.log("[build:chapter] Cancelled.");
return;
}
}
if (targets.length > 0) {
await runStep("bun scripts/sync-chapters.ts");
await runStep(`bun scripts/build-chapter-widgets.ts ${targets.map((t) => t.key).join(" ")}`);
}
const needsShellBuild = !existsSync(BUNDLE_PATH) || frontendChanged;
if (needsShellBuild) {
await runStep("rm -f themes/openfreebooks/static/main.css public/main.css");
await runStep("vite build");
}
await runStep("bun scripts/generate-site-metadata.ts");
await runStep("rm -rf public");
await runStep("zola build");
runPagefindInBackground();
await writeBuildCache({ version: 1, shellFingerprint: currentShellFingerprint });
const summaryTargets = targets.length > 0 ? targets.map((target) => target.key).join(", ") : "(frontend only)";
console.log(`[build:chapter] Core build done: ${summaryTargets}`);
}
void main();

25
scripts/build-site.ts Normal file
View file

@ -0,0 +1,25 @@
import { $ } from "bun";
async function runStep(command: string) {
const result = await $`bash -lc ${command}`;
if (result.exitCode !== 0) process.exit(result.exitCode || 1);
}
async function main() {
await runStep("bash scripts/check-static-html.sh");
await runStep("rm -f themes/openfreebooks/static/main.css public/main.css");
if (process.env.GITHUB_TOKEN?.trim()) {
await runStep("bun scripts/fetch-github-sponsors.ts");
} else {
console.log("[build:site] GITHUB_TOKEN not set; skipping sponsor sync.");
}
await runStep("bun scripts/sync-chapters.ts --full");
await runStep("bun scripts/build-chapter-widgets.ts --full");
await runStep("vite build");
await runStep("bun scripts/generate-site-metadata.ts");
await runStep("rm -rf public");
await runStep("zola build");
await runStep('pagefind --site public --glob "**/index.html" && rm -rf static/pagefind && cp -R public/pagefind static/pagefind');
}
void main();

View file

@ -1,21 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
cleanup() {
if [[ -n "${JS_PID:-}" ]] && kill -0 "${JS_PID}" 2>/dev/null; then
kill "${JS_PID}"
fi
}
trap cleanup EXIT INT TERM
if [[ "${1:-}" == "--full" ]]; then
bun run sync:chapters -- --full
else
bun run sync:chapters
fi
bun run dev:js &
JS_PID=$!
zola serve

View file

@ -123,6 +123,13 @@ async function copyDir(srcDir: string, destDir: string) {
}
}
async function writeIfChanged(path: string, content: string) {
const current = existsSync(path) ? await readFile(path, "utf8") : null;
if (current === content) return;
await mkdir(dirname(path), { recursive: true });
await writeFile(path, content);
}
async function compileChapterScss(
subject: string,
slug: string,
@ -361,18 +368,14 @@ function buildWidgetRegistry(
const lines: string[] = [
"// Auto-generated by scripts/sync-chapters.ts — do not edit.",
"",
"import type { Component } from \"solid-js\";",
"",
"export type ChapterWidgetModule = { default: Component };",
"",
"export const chapterWidgetLoaders: Record<string, () => Promise<ChapterWidgetModule>> = {",
"export const chapterWidgetUrls: Record<string, string> = {",
];
for (const { key, widgets, dir } of entries) {
const relFromGenerated = relative(dirname(GENERATED_WIDGETS), dir).replace(/\\/g, "/");
for (const { key, widgets } of entries) {
const [subject, slug] = key.split("/");
for (const widget of widgets) {
const importPath = `${relFromGenerated}/widgets/${widget}`;
lines.push(` "${key}/${widget}": () => import("${importPath}"),`);
const importPath = `/js/chapter-widgets/${subject}/${slug}/${widget}.js`;
lines.push(` "${key}/${widget}": "${importPath}",`);
}
}
@ -411,14 +414,11 @@ async function main() {
widgetEntries.push({ key, widgets, dir: chapter.dir });
}
await mkdir(dirname(GENERATED_META), { recursive: true });
await writeFile(GENERATED_META, `${JSON.stringify(metaFile, null, 2)}\n`);
await mkdir(dirname(GENERATED_WIDGETS), { recursive: true });
await writeFile(GENERATED_WIDGETS, buildWidgetRegistry(widgetEntries));
await writeIfChanged(GENERATED_META, `${JSON.stringify(metaFile, null, 2)}\n`);
await writeIfChanged(GENERATED_WIDGETS, buildWidgetRegistry(widgetEntries));
const chapterNav = await buildChapterNav(chapters);
await writeFile(GENERATED_NAV, `${JSON.stringify(chapterNav, null, 2)}\n`);
await writeIfChanged(GENERATED_NAV, `${JSON.stringify(chapterNav, null, 2)}\n`);
const subjectEntries: Array<{ id: string; dir: string; hasJs: boolean }> = [];
const subjectsMeta: SubjectsMetaFile = { subjects: {} };
@ -446,9 +446,8 @@ async function main() {
subjectsMeta.subjects[subject.id] = meta;
subjectEntries.push({ id: subject.id, dir: subject.dir, hasJs: meta.hasJs });
}
await writeFile(GENERATED_SUBJECTS_META, `${JSON.stringify(subjectsMeta, null, 2)}\n`);
await mkdir(dirname(GENERATED_SUBJECT_MODULES), { recursive: true });
await writeFile(GENERATED_SUBJECT_MODULES, buildSubjectModuleRegistry(subjectEntries));
await writeIfChanged(GENERATED_SUBJECTS_META, `${JSON.stringify(subjectsMeta, null, 2)}\n`);
await writeIfChanged(GENERATED_SUBJECT_MODULES, buildSubjectModuleRegistry(subjectEntries));
await mkdir(dirname(SYNC_CACHE_PATH), { recursive: true });
await writeFile(SYNC_CACHE_PATH, `${JSON.stringify(nextCache, null, 2)}\n`);

View file

@ -0,0 +1,15 @@
import { describe, expect, it } from "bun:test";
import { validateCurriculum } from "./validate-curriculum";
describe("validate-curriculum", () => {
it("has no curriculum validation errors", async () => {
const result = await validateCurriculum();
if (result.errors.length > 0) {
const details = result.errors
.map((error) => `[${error.file}] ${error.message}`)
.join("\n");
throw new Error(`Curriculum validation failed:\n${details}`);
}
expect(result.errors).toHaveLength(0);
});
});

View file

@ -97,14 +97,24 @@ async function readJsonFile<T>(path: string): Promise<T | null> {
}
}
async function main() {
export async function validateCurriculum(): Promise<{
curriculumFileCount: number;
errors: ValidationError[];
}> {
const errors: ValidationError[] = [];
const catalogPath = join(DATA_DIR, "catalog.json");
const catalog = await readJsonFile<CatalogFile>(catalogPath);
if (!catalog || !Array.isArray(catalog.curriculums)) {
console.error("Cannot validate curriculums: data/catalog.json must contain a curriculums array.");
process.exit(1);
return {
curriculumFileCount: 0,
errors: [
{
file: "catalog.json",
message: "Cannot validate curriculums: data/catalog.json must contain a curriculums array.",
},
],
};
}
const globalCurriculums = new Set(
@ -263,15 +273,21 @@ async function main() {
}
}
if (errors.length > 0) {
console.error(`\n✖ Curriculum validation failed with ${errors.length} error(s):`);
for (const error of errors) {
return { curriculumFileCount: curriculumFiles.length, errors };
}
async function main() {
const result = await validateCurriculum();
if (result.errors.length > 0) {
console.error(`\n✖ Curriculum validation failed with ${result.errors.length} error(s):`);
for (const error of result.errors) {
console.error(`- [${error.file}] ${error.message}`);
}
process.exit(1);
}
console.log(`✓ Curriculum validation passed for ${curriculumFiles.length} curriculum file(s).`);
console.log(`✓ Curriculum validation passed for ${result.curriculumFileCount} curriculum file(s).`);
}
main();
if (import.meta.main) {
void main();
}