feat(experimental): partial rebuilds (#716)

This commit is contained in:
kabirgh 2024-02-09 15:07:32 +00:00 committed by GitHub
parent a87704cd05
commit fe353d946b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 604 additions and 11 deletions

View file

@ -15,7 +15,7 @@
"docs": "npx quartz build --serve -d docs", "docs": "npx quartz build --serve -d docs",
"check": "tsc --noEmit && npx prettier . --check", "check": "tsc --noEmit && npx prettier . --check",
"format": "npx prettier . --write", "format": "npx prettier . --write",
"test": "tsx ./quartz/util/path.test.ts", "test": "tsx ./quartz/util/path.test.ts && tsx ./quartz/depgraph.test.ts",
"profile": "0x -D prof ./quartz/bootstrap-cli.mjs build --concurrency=1" "profile": "0x -D prof ./quartz/bootstrap-cli.mjs build --concurrency=1"
}, },
"engines": { "engines": {

View file

@ -17,6 +17,10 @@ import { glob, toPosixPath } from "./util/glob"
import { trace } from "./util/trace" import { trace } from "./util/trace"
import { options } from "./util/sourcemap" import { options } from "./util/sourcemap"
import { Mutex } from "async-mutex" import { Mutex } from "async-mutex"
import DepGraph from "./depgraph"
import { getStaticResourcesFromPlugins } from "./plugins"
type Dependencies = Record<string, DepGraph<FilePath> | null>
type BuildData = { type BuildData = {
ctx: BuildCtx ctx: BuildCtx
@ -29,8 +33,11 @@ type BuildData = {
toRebuild: Set<FilePath> toRebuild: Set<FilePath>
toRemove: Set<FilePath> toRemove: Set<FilePath>
lastBuildMs: number lastBuildMs: number
dependencies: Dependencies
} }
type FileEvent = "add" | "change" | "delete"
async function buildQuartz(argv: Argv, mut: Mutex, clientRefresh: () => void) { async function buildQuartz(argv: Argv, mut: Mutex, clientRefresh: () => void) {
const ctx: BuildCtx = { const ctx: BuildCtx = {
argv, argv,
@ -68,12 +75,24 @@ async function buildQuartz(argv: Argv, mut: Mutex, clientRefresh: () => void) {
const parsedFiles = await parseMarkdown(ctx, filePaths) const parsedFiles = await parseMarkdown(ctx, filePaths)
const filteredContent = filterContent(ctx, parsedFiles) const filteredContent = filterContent(ctx, parsedFiles)
const dependencies: Record<string, DepGraph<FilePath> | null> = {}
// Only build dependency graphs if we're doing a fast rebuild
if (argv.fastRebuild) {
const staticResources = getStaticResourcesFromPlugins(ctx)
for (const emitter of cfg.plugins.emitters) {
dependencies[emitter.name] =
(await emitter.getDependencyGraph?.(ctx, filteredContent, staticResources)) ?? null
}
}
await emitContent(ctx, filteredContent) await emitContent(ctx, filteredContent)
console.log(chalk.green(`Done processing ${fps.length} files in ${perf.timeSince()}`)) console.log(chalk.green(`Done processing ${fps.length} files in ${perf.timeSince()}`))
release() release()
if (argv.serve) { if (argv.serve) {
return startServing(ctx, mut, parsedFiles, clientRefresh) return startServing(ctx, mut, parsedFiles, clientRefresh, dependencies)
} }
} }
@ -83,9 +102,11 @@ async function startServing(
mut: Mutex, mut: Mutex,
initialContent: ProcessedContent[], initialContent: ProcessedContent[],
clientRefresh: () => void, clientRefresh: () => void,
dependencies: Dependencies, // emitter name: dep graph
) { ) {
const { argv } = ctx const { argv } = ctx
// cache file parse results
const contentMap = new Map<FilePath, ProcessedContent>() const contentMap = new Map<FilePath, ProcessedContent>()
for (const content of initialContent) { for (const content of initialContent) {
const [_tree, vfile] = content const [_tree, vfile] = content
@ -95,6 +116,7 @@ async function startServing(
const buildData: BuildData = { const buildData: BuildData = {
ctx, ctx,
mut, mut,
dependencies,
contentMap, contentMap,
ignored: await isGitIgnored(), ignored: await isGitIgnored(),
initialSlugs: ctx.allSlugs, initialSlugs: ctx.allSlugs,
@ -110,19 +132,181 @@ async function startServing(
ignoreInitial: true, ignoreInitial: true,
}) })
const buildFromEntry = argv.fastRebuild ? partialRebuildFromEntrypoint : rebuildFromEntrypoint
watcher watcher
.on("add", (fp) => rebuildFromEntrypoint(fp, "add", clientRefresh, buildData)) .on("add", (fp) => buildFromEntry(fp, "add", clientRefresh, buildData))
.on("change", (fp) => rebuildFromEntrypoint(fp, "change", clientRefresh, buildData)) .on("change", (fp) => buildFromEntry(fp, "change", clientRefresh, buildData))
.on("unlink", (fp) => rebuildFromEntrypoint(fp, "delete", clientRefresh, buildData)) .on("unlink", (fp) => buildFromEntry(fp, "delete", clientRefresh, buildData))
return async () => { return async () => {
await watcher.close() await watcher.close()
} }
} }
async function partialRebuildFromEntrypoint(
filepath: string,
action: FileEvent,
clientRefresh: () => void,
buildData: BuildData, // note: this function mutates buildData
) {
const { ctx, ignored, dependencies, contentMap, mut, toRemove } = buildData
const { argv, cfg } = ctx
// don't do anything for gitignored files
if (ignored(filepath)) {
return
}
const buildStart = new Date().getTime()
buildData.lastBuildMs = buildStart
const release = await mut.acquire()
if (buildData.lastBuildMs > buildStart) {
release()
return
}
const perf = new PerfTimer()
console.log(chalk.yellow("Detected change, rebuilding..."))
// UPDATE DEP GRAPH
const fp = joinSegments(argv.directory, toPosixPath(filepath)) as FilePath
const staticResources = getStaticResourcesFromPlugins(ctx)
let processedFiles: ProcessedContent[] = []
switch (action) {
case "add":
// add to cache when new file is added
processedFiles = await parseMarkdown(ctx, [fp])
processedFiles.forEach(([tree, vfile]) => contentMap.set(vfile.data.filePath!, [tree, vfile]))
// update the dep graph by asking all emitters whether they depend on this file
for (const emitter of cfg.plugins.emitters) {
const emitterGraph =
(await emitter.getDependencyGraph?.(ctx, processedFiles, staticResources)) ?? null
// emmiter may not define a dependency graph. nothing to update if so
if (emitterGraph) {
dependencies[emitter.name]?.updateIncomingEdgesForNode(emitterGraph, fp)
}
}
break
case "change":
// invalidate cache when file is changed
processedFiles = await parseMarkdown(ctx, [fp])
processedFiles.forEach(([tree, vfile]) => contentMap.set(vfile.data.filePath!, [tree, vfile]))
// only content files can have added/removed dependencies because of transclusions
if (path.extname(fp) === ".md") {
for (const emitter of cfg.plugins.emitters) {
// get new dependencies from all emitters for this file
const emitterGraph =
(await emitter.getDependencyGraph?.(ctx, processedFiles, staticResources)) ?? null
// emmiter may not define a dependency graph. nothing to update if so
if (emitterGraph) {
// merge the new dependencies into the dep graph
dependencies[emitter.name]?.updateIncomingEdgesForNode(emitterGraph, fp)
}
}
}
break
case "delete":
toRemove.add(fp)
break
}
if (argv.verbose) {
console.log(`Updated dependency graphs in ${perf.timeSince()}`)
}
// EMIT
perf.addEvent("rebuild")
let emittedFiles = 0
const destinationsToDelete = new Set<FilePath>()
for (const emitter of cfg.plugins.emitters) {
const depGraph = dependencies[emitter.name]
// emitter hasn't defined a dependency graph. call it with all processed files
if (depGraph === null) {
if (argv.verbose) {
console.log(
`Emitter ${emitter.name} doesn't define a dependency graph. Calling it with all files...`,
)
}
const files = [...contentMap.values()].filter(
([_node, vfile]) => !toRemove.has(vfile.data.filePath!),
)
const emittedFps = await emitter.emit(ctx, files, staticResources)
if (ctx.argv.verbose) {
for (const file of emittedFps) {
console.log(`[emit:${emitter.name}] ${file}`)
}
}
emittedFiles += emittedFps.length
continue
}
// only call the emitter if it uses this file
if (depGraph.hasNode(fp)) {
// re-emit using all files that are needed for the downstream of this file
// eg. for ContentIndex, the dep graph could be:
// a.md --> contentIndex.json
// b.md ------^
//
// if a.md changes, we need to re-emit contentIndex.json,
// and supply [a.md, b.md] to the emitter
const upstreams = [...depGraph.getLeafNodeAncestors(fp)] as FilePath[]
if (action === "delete" && upstreams.length === 1) {
// if there's only one upstream, the destination is solely dependent on this file
destinationsToDelete.add(upstreams[0])
}
const upstreamContent = upstreams
// filter out non-markdown files
.filter((file) => contentMap.has(file))
// if file was deleted, don't give it to the emitter
.filter((file) => !toRemove.has(file))
.map((file) => contentMap.get(file)!)
const emittedFps = await emitter.emit(ctx, upstreamContent, staticResources)
if (ctx.argv.verbose) {
for (const file of emittedFps) {
console.log(`[emit:${emitter.name}] ${file}`)
}
}
emittedFiles += emittedFps.length
}
}
console.log(`Emitted ${emittedFiles} files to \`${argv.output}\` in ${perf.timeSince("rebuild")}`)
// CLEANUP
// delete files that are solely dependent on this file
await rimraf([...destinationsToDelete])
for (const file of toRemove) {
// remove from cache
contentMap.delete(file)
// remove the node from dependency graphs
Object.values(dependencies).forEach((depGraph) => depGraph?.removeNode(file))
}
toRemove.clear()
release()
clientRefresh()
}
async function rebuildFromEntrypoint( async function rebuildFromEntrypoint(
fp: string, fp: string,
action: "add" | "change" | "delete", action: FileEvent,
clientRefresh: () => void, clientRefresh: () => void,
buildData: BuildData, // note: this function mutates buildData buildData: BuildData, // note: this function mutates buildData
) { ) {

View file

@ -71,6 +71,11 @@ export const BuildArgv = {
default: false, default: false,
describe: "run a local server to live-preview your Quartz", describe: "run a local server to live-preview your Quartz",
}, },
fastRebuild: {
boolean: true,
default: false,
describe: "[experimental] rebuild only the changed files",
},
baseDir: { baseDir: {
string: true, string: true,
default: "", default: "",

96
quartz/depgraph.test.ts Normal file
View file

@ -0,0 +1,96 @@
import test, { describe } from "node:test"
import DepGraph from "./depgraph"
import assert from "node:assert"
describe("DepGraph", () => {
test("getLeafNodes", () => {
const graph = new DepGraph<string>()
graph.addEdge("A", "B")
graph.addEdge("B", "C")
graph.addEdge("D", "C")
assert.deepStrictEqual(graph.getLeafNodes("A"), new Set(["C"]))
assert.deepStrictEqual(graph.getLeafNodes("B"), new Set(["C"]))
assert.deepStrictEqual(graph.getLeafNodes("C"), new Set(["C"]))
assert.deepStrictEqual(graph.getLeafNodes("D"), new Set(["C"]))
})
describe("getLeafNodeAncestors", () => {
test("gets correct ancestors in a graph without cycles", () => {
const graph = new DepGraph<string>()
graph.addEdge("A", "B")
graph.addEdge("B", "C")
graph.addEdge("D", "B")
assert.deepStrictEqual(graph.getLeafNodeAncestors("A"), new Set(["A", "B", "D"]))
assert.deepStrictEqual(graph.getLeafNodeAncestors("B"), new Set(["A", "B", "D"]))
assert.deepStrictEqual(graph.getLeafNodeAncestors("C"), new Set(["A", "B", "D"]))
assert.deepStrictEqual(graph.getLeafNodeAncestors("D"), new Set(["A", "B", "D"]))
})
test("gets correct ancestors in a graph with cycles", () => {
const graph = new DepGraph<string>()
graph.addEdge("A", "B")
graph.addEdge("B", "C")
graph.addEdge("C", "A")
graph.addEdge("C", "D")
assert.deepStrictEqual(graph.getLeafNodeAncestors("A"), new Set(["A", "B", "C"]))
assert.deepStrictEqual(graph.getLeafNodeAncestors("B"), new Set(["A", "B", "C"]))
assert.deepStrictEqual(graph.getLeafNodeAncestors("C"), new Set(["A", "B", "C"]))
assert.deepStrictEqual(graph.getLeafNodeAncestors("D"), new Set(["A", "B", "C"]))
})
})
describe("updateIncomingEdgesForNode", () => {
test("merges when node exists", () => {
// A.md -> B.md -> B.html
const graph = new DepGraph<string>()
graph.addEdge("A.md", "B.md")
graph.addEdge("B.md", "B.html")
// B.md is edited so it removes the A.md transclusion
// and adds C.md transclusion
// C.md -> B.md
const other = new DepGraph<string>()
other.addEdge("C.md", "B.md")
other.addEdge("B.md", "B.html")
// A.md -> B.md removed, C.md -> B.md added
// C.md -> B.md -> B.html
graph.updateIncomingEdgesForNode(other, "B.md")
const expected = {
nodes: ["A.md", "B.md", "B.html", "C.md"],
edges: [
["B.md", "B.html"],
["C.md", "B.md"],
],
}
assert.deepStrictEqual(graph.export(), expected)
})
test("adds node if it does not exist", () => {
// A.md -> B.md
const graph = new DepGraph<string>()
graph.addEdge("A.md", "B.md")
// Add a new file C.md that transcludes B.md
// B.md -> C.md
const other = new DepGraph<string>()
other.addEdge("B.md", "C.md")
// B.md -> C.md added
// A.md -> B.md -> C.md
graph.updateIncomingEdgesForNode(other, "C.md")
const expected = {
nodes: ["A.md", "B.md", "C.md"],
edges: [
["A.md", "B.md"],
["B.md", "C.md"],
],
}
assert.deepStrictEqual(graph.export(), expected)
})
})
})

187
quartz/depgraph.ts Normal file
View file

@ -0,0 +1,187 @@
export default class DepGraph<T> {
// node: incoming and outgoing edges
_graph = new Map<T, { incoming: Set<T>; outgoing: Set<T> }>()
constructor() {
this._graph = new Map()
}
export(): Object {
return {
nodes: this.nodes,
edges: this.edges,
}
}
toString(): string {
return JSON.stringify(this.export(), null, 2)
}
// BASIC GRAPH OPERATIONS
get nodes(): T[] {
return Array.from(this._graph.keys())
}
get edges(): [T, T][] {
let edges: [T, T][] = []
this.forEachEdge((edge) => edges.push(edge))
return edges
}
hasNode(node: T): boolean {
return this._graph.has(node)
}
addNode(node: T): void {
if (!this._graph.has(node)) {
this._graph.set(node, { incoming: new Set(), outgoing: new Set() })
}
}
removeNode(node: T): void {
if (this._graph.has(node)) {
this._graph.delete(node)
}
}
hasEdge(from: T, to: T): boolean {
return Boolean(this._graph.get(from)?.outgoing.has(to))
}
addEdge(from: T, to: T): void {
this.addNode(from)
this.addNode(to)
this._graph.get(from)!.outgoing.add(to)
this._graph.get(to)!.incoming.add(from)
}
removeEdge(from: T, to: T): void {
if (this._graph.has(from) && this._graph.has(to)) {
this._graph.get(from)!.outgoing.delete(to)
this._graph.get(to)!.incoming.delete(from)
}
}
// returns -1 if node does not exist
outDegree(node: T): number {
return this.hasNode(node) ? this._graph.get(node)!.outgoing.size : -1
}
// returns -1 if node does not exist
inDegree(node: T): number {
return this.hasNode(node) ? this._graph.get(node)!.incoming.size : -1
}
forEachOutNeighbor(node: T, callback: (neighbor: T) => void): void {
this._graph.get(node)?.outgoing.forEach(callback)
}
forEachInNeighbor(node: T, callback: (neighbor: T) => void): void {
this._graph.get(node)?.incoming.forEach(callback)
}
forEachEdge(callback: (edge: [T, T]) => void): void {
for (const [source, { outgoing }] of this._graph.entries()) {
for (const target of outgoing) {
callback([source, target])
}
}
}
// DEPENDENCY ALGORITHMS
// For the node provided:
// If node does not exist, add it
// If an incoming edge was added in other, it is added in this graph
// If an incoming edge was deleted in other, it is deleted in this graph
updateIncomingEdgesForNode(other: DepGraph<T>, node: T): void {
this.addNode(node)
// Add edge if it is present in other
other.forEachInNeighbor(node, (neighbor) => {
this.addEdge(neighbor, node)
})
// For node provided, remove incoming edge if it is absent in other
this.forEachEdge(([source, target]) => {
if (target === node && !other.hasEdge(source, target)) {
this.removeEdge(source, target)
}
})
}
// Get all leaf nodes (i.e. destination paths) reachable from the node provided
// Eg. if the graph is A -> B -> C
// D ---^
// and the node is B, this function returns [C]
getLeafNodes(node: T): Set<T> {
let stack: T[] = [node]
let visited = new Set<T>()
let leafNodes = new Set<T>()
// DFS
while (stack.length > 0) {
let node = stack.pop()!
// If the node is already visited, skip it
if (visited.has(node)) {
continue
}
visited.add(node)
// Check if the node is a leaf node (i.e. destination path)
if (this.outDegree(node) === 0) {
leafNodes.add(node)
}
// Add all unvisited neighbors to the stack
this.forEachOutNeighbor(node, (neighbor) => {
if (!visited.has(neighbor)) {
stack.push(neighbor)
}
})
}
return leafNodes
}
// Get all ancestors of the leaf nodes reachable from the node provided
// Eg. if the graph is A -> B -> C
// D ---^
// and the node is B, this function returns [A, B, D]
getLeafNodeAncestors(node: T): Set<T> {
const leafNodes = this.getLeafNodes(node)
let visited = new Set<T>()
let upstreamNodes = new Set<T>()
// Backwards DFS for each leaf node
leafNodes.forEach((leafNode) => {
let stack: T[] = [leafNode]
while (stack.length > 0) {
let node = stack.pop()!
if (visited.has(node)) {
continue
}
visited.add(node)
// Add node if it's not a leaf node (i.e. destination path)
// Assumes destination file cannot depend on another destination file
if (this.outDegree(node) !== 0) {
upstreamNodes.add(node)
}
// Add all unvisited parents to the stack
this.forEachInNeighbor(node, (parentNode) => {
if (!visited.has(parentNode)) {
stack.push(parentNode)
}
})
}
})
return upstreamNodes
}
}

View file

@ -9,6 +9,7 @@ import { NotFound } from "../../components"
import { defaultProcessedContent } from "../vfile" import { defaultProcessedContent } from "../vfile"
import { write } from "./helpers" import { write } from "./helpers"
import { i18n } from "../../i18n" import { i18n } from "../../i18n"
import DepGraph from "../../depgraph"
export const NotFoundPage: QuartzEmitterPlugin = () => { export const NotFoundPage: QuartzEmitterPlugin = () => {
const opts: FullPageLayout = { const opts: FullPageLayout = {
@ -27,6 +28,9 @@ export const NotFoundPage: QuartzEmitterPlugin = () => {
getQuartzComponents() { getQuartzComponents() {
return [Head, Body, pageBody, Footer] return [Head, Body, pageBody, Footer]
}, },
async getDependencyGraph(_ctx, _content, _resources) {
return new DepGraph<FilePath>()
},
async emit(ctx, _content, resources): Promise<FilePath[]> { async emit(ctx, _content, resources): Promise<FilePath[]> {
const cfg = ctx.cfg.configuration const cfg = ctx.cfg.configuration
const slug = "404" as FullSlug const slug = "404" as FullSlug

View file

@ -2,12 +2,17 @@ import { FilePath, FullSlug, joinSegments, resolveRelative, simplifySlug } from
import { QuartzEmitterPlugin } from "../types" import { QuartzEmitterPlugin } from "../types"
import path from "path" import path from "path"
import { write } from "./helpers" import { write } from "./helpers"
import DepGraph from "../../depgraph"
export const AliasRedirects: QuartzEmitterPlugin = () => ({ export const AliasRedirects: QuartzEmitterPlugin = () => ({
name: "AliasRedirects", name: "AliasRedirects",
getQuartzComponents() { getQuartzComponents() {
return [] return []
}, },
async getDependencyGraph(_ctx, _content, _resources) {
// TODO implement
return new DepGraph<FilePath>()
},
async emit(ctx, content, _resources): Promise<FilePath[]> { async emit(ctx, content, _resources): Promise<FilePath[]> {
const { argv } = ctx const { argv } = ctx
const fps: FilePath[] = [] const fps: FilePath[] = []

View file

@ -3,6 +3,7 @@ import { QuartzEmitterPlugin } from "../types"
import path from "path" import path from "path"
import fs from "fs" import fs from "fs"
import { glob } from "../../util/glob" import { glob } from "../../util/glob"
import DepGraph from "../../depgraph"
export const Assets: QuartzEmitterPlugin = () => { export const Assets: QuartzEmitterPlugin = () => {
return { return {
@ -10,6 +11,24 @@ export const Assets: QuartzEmitterPlugin = () => {
getQuartzComponents() { getQuartzComponents() {
return [] return []
}, },
async getDependencyGraph(ctx, _content, _resources) {
const { argv, cfg } = ctx
const graph = new DepGraph<FilePath>()
const fps = await glob("**", argv.directory, ["**/*.md", ...cfg.configuration.ignorePatterns])
for (const fp of fps) {
const ext = path.extname(fp)
const src = joinSegments(argv.directory, fp) as FilePath
const name = (slugifyFilePath(fp as FilePath, true) + ext) as FilePath
const dest = joinSegments(argv.output, name) as FilePath
graph.addEdge(src, dest)
}
return graph
},
async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> { async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> {
// glob all non MD/MDX/HTML files in content folder and copy it over // glob all non MD/MDX/HTML files in content folder and copy it over
const assetsPath = argv.output const assetsPath = argv.output

View file

@ -2,6 +2,7 @@ import { FilePath, joinSegments } from "../../util/path"
import { QuartzEmitterPlugin } from "../types" import { QuartzEmitterPlugin } from "../types"
import fs from "fs" import fs from "fs"
import chalk from "chalk" import chalk from "chalk"
import DepGraph from "../../depgraph"
export function extractDomainFromBaseUrl(baseUrl: string) { export function extractDomainFromBaseUrl(baseUrl: string) {
const url = new URL(`https://${baseUrl}`) const url = new URL(`https://${baseUrl}`)
@ -13,6 +14,9 @@ export const CNAME: QuartzEmitterPlugin = () => ({
getQuartzComponents() { getQuartzComponents() {
return [] return []
}, },
async getDependencyGraph(_ctx, _content, _resources) {
return new DepGraph<FilePath>()
},
async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> { async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> {
if (!cfg.configuration.baseUrl) { if (!cfg.configuration.baseUrl) {
console.warn(chalk.yellow("CNAME emitter requires `baseUrl` to be set in your configuration")) console.warn(chalk.yellow("CNAME emitter requires `baseUrl` to be set in your configuration"))

View file

@ -14,6 +14,7 @@ import { googleFontHref, joinStyles } from "../../util/theme"
import { Features, transform } from "lightningcss" import { Features, transform } from "lightningcss"
import { transform as transpile } from "esbuild" import { transform as transpile } from "esbuild"
import { write } from "./helpers" import { write } from "./helpers"
import DepGraph from "../../depgraph"
type ComponentResources = { type ComponentResources = {
css: string[] css: string[]
@ -149,9 +150,10 @@ function addGlobalPageResources(
loadTime: "afterDOMReady", loadTime: "afterDOMReady",
contentType: "inline", contentType: "inline",
script: ` script: `
const socket = new WebSocket('${wsUrl}') const socket = new WebSocket('${wsUrl}')
socket.addEventListener('message', () => document.location.reload()) // reload(true) ensures resources like images and scripts are fetched again in firefox
`, socket.addEventListener('message', () => document.location.reload(true))
`,
}) })
} }
} }
@ -171,6 +173,24 @@ export const ComponentResources: QuartzEmitterPlugin<Options> = (opts?: Partial<
getQuartzComponents() { getQuartzComponents() {
return [] return []
}, },
async getDependencyGraph(ctx, content, _resources) {
// This emitter adds static resources to the `resources` parameter. One
// important resource this emitter adds is the code to start a websocket
// connection and listen to rebuild messages, which triggers a page reload.
// The resources parameter with the reload logic is later used by the
// ContentPage emitter while creating the final html page. In order for
// the reload logic to be included, and so for partial rebuilds to work,
// we need to run this emitter for all markdown files.
const graph = new DepGraph<FilePath>()
for (const [_tree, file] of content) {
const sourcePath = file.data.filePath!
const slug = file.data.slug!
graph.addEdge(sourcePath, joinSegments(ctx.argv.output, slug + ".html") as FilePath)
}
return graph
},
async emit(ctx, _content, resources): Promise<FilePath[]> { async emit(ctx, _content, resources): Promise<FilePath[]> {
const promises: Promise<FilePath>[] = [] const promises: Promise<FilePath>[] = []
const cfg = ctx.cfg.configuration const cfg = ctx.cfg.configuration

View file

@ -7,6 +7,7 @@ import { QuartzEmitterPlugin } from "../types"
import { toHtml } from "hast-util-to-html" import { toHtml } from "hast-util-to-html"
import { write } from "./helpers" import { write } from "./helpers"
import { i18n } from "../../i18n" import { i18n } from "../../i18n"
import DepGraph from "../../depgraph"
export type ContentIndex = Map<FullSlug, ContentDetails> export type ContentIndex = Map<FullSlug, ContentDetails>
export type ContentDetails = { export type ContentDetails = {
@ -92,6 +93,26 @@ export const ContentIndex: QuartzEmitterPlugin<Partial<Options>> = (opts) => {
opts = { ...defaultOptions, ...opts } opts = { ...defaultOptions, ...opts }
return { return {
name: "ContentIndex", name: "ContentIndex",
async getDependencyGraph(ctx, content, _resources) {
const graph = new DepGraph<FilePath>()
for (const [_tree, file] of content) {
const sourcePath = file.data.filePath!
graph.addEdge(
sourcePath,
joinSegments(ctx.argv.output, "static/contentIndex.json") as FilePath,
)
if (opts?.enableSiteMap) {
graph.addEdge(sourcePath, joinSegments(ctx.argv.output, "sitemap.xml") as FilePath)
}
if (opts?.enableRSS) {
graph.addEdge(sourcePath, joinSegments(ctx.argv.output, "index.xml") as FilePath)
}
}
return graph
},
async emit(ctx, content, _resources) { async emit(ctx, content, _resources) {
const cfg = ctx.cfg.configuration const cfg = ctx.cfg.configuration
const emitted: FilePath[] = [] const emitted: FilePath[] = []

View file

@ -4,11 +4,12 @@ import HeaderConstructor from "../../components/Header"
import BodyConstructor from "../../components/Body" import BodyConstructor from "../../components/Body"
import { pageResources, renderPage } from "../../components/renderPage" import { pageResources, renderPage } from "../../components/renderPage"
import { FullPageLayout } from "../../cfg" import { FullPageLayout } from "../../cfg"
import { FilePath, pathToRoot } from "../../util/path" import { FilePath, joinSegments, pathToRoot } from "../../util/path"
import { defaultContentPageLayout, sharedPageComponents } from "../../../quartz.layout" import { defaultContentPageLayout, sharedPageComponents } from "../../../quartz.layout"
import { Content } from "../../components" import { Content } from "../../components"
import chalk from "chalk" import chalk from "chalk"
import { write } from "./helpers" import { write } from "./helpers"
import DepGraph from "../../depgraph"
export const ContentPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpts) => { export const ContentPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpts) => {
const opts: FullPageLayout = { const opts: FullPageLayout = {
@ -27,6 +28,18 @@ export const ContentPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOp
getQuartzComponents() { getQuartzComponents() {
return [Head, Header, Body, ...header, ...beforeBody, pageBody, ...left, ...right, Footer] return [Head, Header, Body, ...header, ...beforeBody, pageBody, ...left, ...right, Footer]
}, },
async getDependencyGraph(ctx, content, _resources) {
// TODO handle transclusions
const graph = new DepGraph<FilePath>()
for (const [_tree, file] of content) {
const sourcePath = file.data.filePath!
const slug = file.data.slug!
graph.addEdge(sourcePath, joinSegments(ctx.argv.output, slug + ".html") as FilePath)
}
return graph
},
async emit(ctx, content, resources): Promise<FilePath[]> { async emit(ctx, content, resources): Promise<FilePath[]> {
const cfg = ctx.cfg.configuration const cfg = ctx.cfg.configuration
const fps: FilePath[] = [] const fps: FilePath[] = []
@ -60,7 +73,7 @@ export const ContentPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOp
fps.push(fp) fps.push(fp)
} }
if (!containsIndex) { if (!containsIndex && !ctx.argv.fastRebuild) {
console.log( console.log(
chalk.yellow( chalk.yellow(
`\nWarning: you seem to be missing an \`index.md\` home page file at the root of your \`${ctx.argv.directory}\` folder. This may cause errors when deploying.`, `\nWarning: you seem to be missing an \`index.md\` home page file at the root of your \`${ctx.argv.directory}\` folder. This may cause errors when deploying.`,

View file

@ -19,6 +19,7 @@ import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.lay
import { FolderContent } from "../../components" import { FolderContent } from "../../components"
import { write } from "./helpers" import { write } from "./helpers"
import { i18n } from "../../i18n" import { i18n } from "../../i18n"
import DepGraph from "../../depgraph"
export const FolderPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpts) => { export const FolderPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpts) => {
const opts: FullPageLayout = { const opts: FullPageLayout = {
@ -37,6 +38,13 @@ export const FolderPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpt
getQuartzComponents() { getQuartzComponents() {
return [Head, Header, Body, ...header, ...beforeBody, pageBody, ...left, ...right, Footer] return [Head, Header, Body, ...header, ...beforeBody, pageBody, ...left, ...right, Footer]
}, },
async getDependencyGraph(ctx, content, _resources) {
// Example graph:
// nested/file.md --> nested/file.html
// \-------> nested/index.html
// TODO implement
return new DepGraph<FilePath>()
},
async emit(ctx, content, resources): Promise<FilePath[]> { async emit(ctx, content, resources): Promise<FilePath[]> {
const fps: FilePath[] = [] const fps: FilePath[] = []
const allFiles = content.map((c) => c[1].data) const allFiles = content.map((c) => c[1].data)

View file

@ -2,12 +2,27 @@ import { FilePath, QUARTZ, joinSegments } from "../../util/path"
import { QuartzEmitterPlugin } from "../types" import { QuartzEmitterPlugin } from "../types"
import fs from "fs" import fs from "fs"
import { glob } from "../../util/glob" import { glob } from "../../util/glob"
import DepGraph from "../../depgraph"
export const Static: QuartzEmitterPlugin = () => ({ export const Static: QuartzEmitterPlugin = () => ({
name: "Static", name: "Static",
getQuartzComponents() { getQuartzComponents() {
return [] return []
}, },
async getDependencyGraph({ argv, cfg }, _content, _resources) {
const graph = new DepGraph<FilePath>()
const staticPath = joinSegments(QUARTZ, "static")
const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns)
for (const fp of fps) {
graph.addEdge(
joinSegments("static", fp) as FilePath,
joinSegments(argv.output, "static", fp) as FilePath,
)
}
return graph
},
async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> { async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> {
const staticPath = joinSegments(QUARTZ, "static") const staticPath = joinSegments(QUARTZ, "static")
const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns) const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns)

View file

@ -16,6 +16,7 @@ import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.lay
import { TagContent } from "../../components" import { TagContent } from "../../components"
import { write } from "./helpers" import { write } from "./helpers"
import { i18n } from "../../i18n" import { i18n } from "../../i18n"
import DepGraph from "../../depgraph"
export const TagPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpts) => { export const TagPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpts) => {
const opts: FullPageLayout = { const opts: FullPageLayout = {
@ -34,6 +35,10 @@ export const TagPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOpts)
getQuartzComponents() { getQuartzComponents() {
return [Head, Header, Body, ...header, ...beforeBody, pageBody, ...left, ...right, Footer] return [Head, Header, Body, ...header, ...beforeBody, pageBody, ...left, ...right, Footer]
}, },
async getDependencyGraph(ctx, _content, _resources) {
// TODO implement
return new DepGraph<FilePath>()
},
async emit(ctx, content, resources): Promise<FilePath[]> { async emit(ctx, content, resources): Promise<FilePath[]> {
const fps: FilePath[] = [] const fps: FilePath[] = []
const allFiles = content.map((c) => c[1].data) const allFiles = content.map((c) => c[1].data)

View file

@ -4,6 +4,7 @@ import { ProcessedContent } from "./vfile"
import { QuartzComponent } from "../components/types" import { QuartzComponent } from "../components/types"
import { FilePath } from "../util/path" import { FilePath } from "../util/path"
import { BuildCtx } from "../util/ctx" import { BuildCtx } from "../util/ctx"
import DepGraph from "../depgraph"
export interface PluginTypes { export interface PluginTypes {
transformers: QuartzTransformerPluginInstance[] transformers: QuartzTransformerPluginInstance[]
@ -38,4 +39,9 @@ export type QuartzEmitterPluginInstance = {
name: string name: string
emit(ctx: BuildCtx, content: ProcessedContent[], resources: StaticResources): Promise<FilePath[]> emit(ctx: BuildCtx, content: ProcessedContent[], resources: StaticResources): Promise<FilePath[]>
getQuartzComponents(ctx: BuildCtx): QuartzComponent[] getQuartzComponents(ctx: BuildCtx): QuartzComponent[]
getDependencyGraph?(
ctx: BuildCtx,
content: ProcessedContent[],
resources: StaticResources,
): Promise<DepGraph<FilePath>>
} }

View file

@ -6,6 +6,7 @@ export interface Argv {
verbose: boolean verbose: boolean
output: string output: string
serve: boolean serve: boolean
fastRebuild: boolean
port: number port: number
wsPort: number wsPort: number
remoteDevHost?: string remoteDevHost?: string