#!/usr/bin/awk -f # coprolit.awk version 2 by Ben Collver # # Static page generator for gopher and fossil SCM. # Usage: awk -f coprolit.awk # Requirements: fossil webdump function exists(filename, result, retval) { result = getline < filename close(filename) if (result == -1) { retval = 0 } else { retval = 1 } return retval } function fossil_configuration(arr, cmd, fsout, k, v) { fsout = gettemp() cmd = sprintf("fossil configuration export project %s -R %s", fsout, _repo) system(cmd) while ((getline 0) { if (/^#/ || /^config/) { # ignore comments & config lines } else { sub(/^[0-9][0-9]* */, "") if (match($0, /'[^']*'/)) { k = substr($0, RSTART + 1, RLENGTH - 2) $0 = substr($0, RSTART + RLENGTH + 1) if (match($0, /'[^']*'/)) { v = substr($0, RSTART + 1, RLENGTH - 2) arr[k] = v } } } } close(fsout) unlink(fsout) return } function fossil_remote( cmd, retval) { # get remote repository URL retval = "unknown" cmd = sprintf("fossil remote -R %s 2>&1", _repo) while ((cmd | getline) > 0) { # remove user name from the remote repository URL sub(/\/[^/]*@/, "/") retval = $0 } close(cmd) return retval } function generate_branches( branches, cmd, commits, fsout, i, m, out, refs, sel) { mkdir(_work "/branches") out = _work "/branches/gophermap" unlink(out) fsout = gettemp() cmd = sprintf("fossil branch ls -R %s >%s 2>&1", _repo, fsout) system(cmd) refs["count"] = 1 refs[1] = "fossil branch ls" info(out, sprintf("# %s / Branches", _conf["project-name"])) menu(out, "Branches") m = 0 while ((getline < fsout) > 0) { m++ branches[m] = $1 sel = _root "/timeline/" $1 item(out, 1, $1, sel, _server, _port) } close(fsout) unlink(fsout) reference(out, refs) close(out) for (i = 1; i <= m; i++) { generate_timeline(branches[i]) } return } function generate_commit(commit, branch, cmd, comment, file, fsout, has_downloads, hash, is_checkin, m, out, parents, refs, slug, type) { out = sprintf("%s/info/%s/gophermap", _work, commit) if (exists(out)) { # This has already been generated, don't do again return } mkdir(_work "/info/" commit) mkdir(_work "/patch") fsout = gettemp() cmd = sprintf("fossil timeline %s -n 1 --full -R %s >%s 2>&1", commit, _repo, fsout) system(cmd) m = 1 refs[m] = sprintf("fossil timeline %s -n 1 --full", commit) info(out, sprintf("# %s / Check-in [%s]", _conf["project-name"], commit)) menu(out, "Commit") while ((getline < fsout) > 0) { if ($1 == "Commit:") { hash = $2 } else if ($1 == "Comment:") { comment = $0 sub(/^Comment: */, "", comment) } else if ($1 == "Branch") { branch = $2 } } close(fsout) unlink(fsout) cmd = sprintf("fossil whatis %s -R %s >%s 2>&1", commit, _repo, fsout) system(cmd) m++ refs[m] = sprintf("fossil whatis %s", commit) type = "unknown" while ((getline < fsout) > 0) { if (/^type:/) { sub(/^type: */, "") type = $0 } } close(fsout) unlink(fsout) if (type ~ /^Check-in/) { is_checkin = 1 } else { is_checkin = 0 } has_downloads = 0 parents = 0 if (is_checkin) { if (_download_count < _download_max) { has_downloads = 1 _download_count++ # export tarball m++ refs[m] = generate_tarball(commit) # export zip m++ refs[m] = generate_zip(commit) } cmd = sprintf("fossil timeline parents %s -n 0 --oneline -R %s >%s", commit, _repo, fsout) system(cmd) while ((getline 0) { if (/^... end of timeline/) { sub(/^\(/, "", $5) sub(/\)$/, "", $5) parents = $5 } } close(fsout) unlink(fsout) if (parents > 1) { # export patch file = sprintf("%s/patch/%s.txt", _work, commit) printf "%s\n\n", type >file if (length(comment) < 66) { println(file, comment) } else { wrap("print", file, comment, 65) } printf "\n" >>file close(file) cmd = sprintf("fossil diff -v --checkin %s --numstat -R %s >>%s", commit, _repo, file) system(cmd) printf "\n" >>file close(file) cmd = sprintf("fossil diff -v --checkin %s -R %s >>%s", commit, _repo, file) system(cmd) m++ refs[m] = sprintf("fossil diff -v --checkin %s --numstat", commit) m++ refs[m] = sprintf("fossil diff -v --checkin %s", commit) } } info(out, "# Overview") info(out, "") if (length(comment) < 66) { info(out, sprintf("Comment: %s", comment)) } else { info(out, "Comment:") wrap("info", out, comment, 65) } if (is_checkin) { info(out, "") info(out, "Downloads:") if (has_downloads) { slug = sprintf("%s/tarball/%s/%s-%s.tar.gz", _root, commit, _conf["short-project-name"], commit) item(out, "9", "Tarball", slug, _server, _port) slug = sprintf("%s/zip/%s/%s-%s.zip", _root, commit, _conf["short-project-name"], commit) item(out, "9", "Zip archive", slug, _server, _port) } else { info(out, sprintf("Tarball: fossil tarball %s %s-%s.tar.gz --name %s", commit, _conf["short-project-name"], commit, _conf["short-project-name"])) info(out, sprintf("Zip: fossil zip %s %s-%s.zip --name %s", commit, _conf["short-project-name"], commit, _conf["short-project-name"])) } } info(out, "") info(out, "SHA3-256:") info(out, " " hash) info(out, sprintf("Type: %s", type)) if (is_checkin && parents > 1) { info(out, "") info(out, "# Changes") info(out, "") file = sprintf("%s/patch/%s.txt", _root, commit) item(out, 0, "Patch", file, _server, _port) } refs["count"] = m reference(out, refs) close(out) return } function generate_files( cmd, file) { # export tip tarball & zip generate_tarball("tip") generate_zip("tip") # extract tip tarball file = sprintf("%s/tarball/tip/%s-tip.tar.gz", _work, _conf["short-project-name"]) cmd = sprintf("tar zxf %s -C %s --transform 's,%s,files,'", file, _work, _conf["short-project-name"]) system(cmd) return } function generate_home( cmd, fsout, out) { if (_verbose) { print "Generating home (" _conf["project-name"] ") ..." } out = _work "/gophermap" unlink(out) fsout = gettemp() cmd = sprintf("fossil wiki export -h \"%s\" -R %s >%s 2>&1", _conf["project-name"], _repo, fsout) system(cmd) info(out, "# Home - " _conf["project-name"]) menu(out, "Home") cmd = sprintf("webdump -ilr -w 60 <%s", fsout) while ((cmd | getline) > 0) { info(out, $0) } close(fsout) unlink(fsout) close(out) return } function generate_tarball(commit, cmd, file, name, retval) { if (_verbose) { print "Generating tarball (" commit ") ..." } mkdir(_work "/tarball/" commit) name = _conf["short-project-name"] file = sprintf("%s/tarball/%s/%s-%s.tar.gz", _work, commit, name, commit) cmd = sprintf("fossil tarball %s %s --name %s -R %s", commit, file, name, _repo) system(cmd) retval = sprintf("fossil tarball %s %s-%s.tar.gz --name %s", commit, name, commit, name) return retval } function generate_tags( tags, cmd, commits, fsout, i, m, out, query, refs, sel) { mkdir(_work "/tags") out = _work "/tags/gophermap" unlink(out) fsout = gettemp() query = "SELECT " \ " substr(tag.tagname, 5) AS tagname, " \ " datetime(MAX(tagxref.mtime)) AS mtime " \ "FROM repository.tag " \ "JOIN repository.tagxref ON tagxref.tagid=tag.tagid " \ "WHERE " \ " tag.tagname LIKE 'sym-%%' AND " \ " tagxref.tagtype = 1 " \ "GROUP BY tag.tagid, tag.tagname " \ "ORDER BY MAX(tagxref.mtime) DESC" # cmd = sprintf("fossil tag ls -R %s >%s 2>&1", _repo, fsout) cmd = sprintf("fossil sql --readonly -R %s >%s 2>&1", _repo, fsout) printf ".mode tabs\n%s\n", query | cmd close(cmd) refs["count"] = 3 refs[1] = "fossil tag ls" refs[1] = "fossil sql --readonly" refs[3] = query info(out, sprintf("# %s / Tags", _conf["project-name"])) menu(out, "Tags") m = 0 while ((getline < fsout) > 0) { m++ tags[m] = $1 sel = _root "/tags/" $1 item(out, 1, $1 " -- " $2, sel, _server, _port) } close(fsout) unlink(fsout) reference(out, refs) close(out) for (i = 1; i <= m; i++) { generate_tag_timeline(tags[i]) } return } function generate_tag_timeline(tag, author, authors, cmd, comment_abbrev, commit, commits, date, dates, fsout, i, label, line, lines, m, out, refs, sel, time, total_commits) { if (_verbose) { print "Generating timeline for tag (" tag ") ..." } out = sprintf("%s/tags/%s/gophermap", _work, tag) _download_count = 0 _download_max = _download mkdir(_work "/tags/" tag) fsout = gettemp() cmd = sprintf("fossil tag find -n 0 -t ci %s -R %s >%s 2>&1", tag, _repo, fsout) system(cmd) refs["count"] = 1 refs[1] = sprintf("fossil tag find -n 0 -t ci %s", tag) total_commits = 0 line = "" while ((getline < fsout) > 0) { if ($1 == "===") { date = $2 } else if (/^[0-9]/) { # check whether line buffer is present from prior iterations if (length(line) > 0) { if (match(line, /user: [^ ]*/)) { author = substr(line, RSTART + 7, RLENGTH - 6) } else { author = "unknown" } sub(/ \(user:.*\)/, "", line) if (m < _timeline) { m++ authors[m] = author commits[m] = commit lines[m] = line dates[m] = date " " time } } time = $1 commit = $2 sub(/^\[/, "", commit) sub(/\]$/, "", commit) line = $0 sub(/^[^ ]* [^ ]* /, "", line) } else if (/^ /) { sub(/^ */, "") line = line $0 } else if (/^... end of timeline /) { sub(/^\(/, "", $5) sub(/\)$/, "", $5) total_commits = $5 } } if (length(line) > 0 && m < _timeline) { if (match(line, /user: [^ ]*/)) { author = substr(line, RSTART + 6, RLENGTH - 6) } else { author = "unknown" } sub(/ \(user:.*\)/, "", line) m++ authors[m] = author commits[m] = commit lines[m] = line dates[m] = date " " time } close(fsout) unlink(fsout) info(out, sprintf("# %s / Timeline / Tag %s", _conf["project-name"], tag)) menu(out, "Timeline") for (i = 1; i <= m; i++) { if (length(comment) == 0) { comment_abbrev = commit } else if (length(comment) < 42) { comment_abbrev = comment } else { comment_abbrev = substr(comment, 1, 37) "..." } label = sprintf("%-17s %-41s %s", substr(dates[i], 1, 16), comment_abbrev, authors[i]) sel = _root "/info/" commits[i] item(out, 1, label, sel, _server, _port) } if (m < total_commits) { info(out, " " total_commits - m \ " more commits remaining, fetch the repository") } reference(out, refs) close(out) for (i = 1; i <= m; i++) { generate_commit(commits[i]) } return } function generate_ticket(ticket, ctime, foundin, mtime, priority, resolution, severity, status, title, uuid, type, cmd, out) { if (_verbose) { print "Generating ticket (" ticket ") ..." } out = sprintf("%s/tickets/%s.txt", _work, ticket) unlink(out) printf "# %s\n\n", _conf["project-name"] >out printf "## View Ticket\n\n" >>out printf "Ticket Hash: %s\n", uuid >>out if (length(title) < 66) { printf "Title: %s\n", title >>out } else { printf "Title:\n" >>out wrap("print", out, title, 65) } printf "Status: %s\n", status >>out printf "Type: %s\n", type >>out printf "Severity: %s\n", severity >>out printf "Priority: %s\n", priority >>out printf "Resolution: %s\n", resolution >>out printf "Modified: %s\n", mtime >>out printf "Created: %s\n", ctime >>out printf "Found In: %s\n", foundin >>out cmd = sprintf("fossil ticket history %s -R %s 2>&1", ticket, _repo) while ((cmd | getline) > 0) { if (/^Ticket Change/) { printf "\n" >>out } print $0 >>out } close(cmd) printf "\nReference:\n\n" >>out printf "fossil ticket history %s\n", ticket >>out close(out) return } function generate_tickets( ctime, cmd, count, field, fnr, fsout, i, label, mtime, oldfs, out, query, refs, sel, tags, ticket) { mkdir(_work "/tickets") out = _work "/tickets/gophermap" unlink(out) fsout = gettemp() query = "SELECT " \ " tkt_id, " \ " tkt_uuid, " \ " datetime(tkt_mtime) AS mtime, " \ " datetime(tkt_ctime) AS ctime, " \ " type, " \ " status, " \ " subsystem, " \ " priority, " \ " severity, " \ " foundin, " \ " resolution, " \ " title, " \ " comment " \ "FROM repository.ticket" # cmd = sprintf("fossil ticket show 0 -R %s >%s 2>&1", _repo, fsout) cmd = sprintf("fossil sql --readonly -R %s >%s 2>&1", _repo, fsout) printf ".headers on\n.mode tabs\n%s\n", query | cmd close(cmd) refs["count"] = 3 refs[1] = "fossil ticket show 0" refs[2] = "fossil sql --readonly" refs[3] = query info(out, sprintf("# %s / Tickets", _conf["project-name"])) menu(out, "Tickets") oldfs = FS FS = "\t" while ((getline < fsout) > 0) { fnr++ if (fnr == 1) { count = NF for (i = 1; i <= count; i++) { field[$i] = i } } else { ticket = substr($field["tkt_uuid"], 1, 10) label = "Ticket: " ticket sel = sprintf("%s/tickets/%s.txt", _root, ticket) item(out, 0, label, sel, _server, _port) info(out, "Mtime: " $field["mtime"]) info(out, "Type: " $field["type"]) info(out, "Status: " $field["status"]) if (length($field["title"]) < 61) { info(out, "Title: " $field["title"]) } else { info(out, "Title:") wrap("info", out, $field["title"], 65) } info(out, "") generate_ticket(ticket, $field["ctime"], $field["foundin"], $field["mtime"], $field["priority"], $field["resolution"], $field["severity"], $field["status"], $field["title"], $field["tkt_uuid"], $field["type"]) } } FS = oldfs close(fsout) unlink(fsout) reference(out, refs) close(out) return } function generate_timeline(branch, author, branch_opt, cmd, comment, comment_abbrev, commit, commits, date, download_count, download_max, fsout, i, label, m, out, refs, sel, total_commits) { if (_verbose) { print "Generating timeline (" branch ") ..." } _download_count = 0 if (length(branch) > 0) { _download_max = -1 branch_opt = "-b " branch " " out = sprintf("%s/timeline/%s/gophermap", _work, branch) } else { _download_max = _download branch_opt = "" out = sprintf("%s/timeline/gophermap", _work) } unlink(out) if (length(branch) > 0) { mkdir(_work "/timeline/" branch) } else { mkdir(_work "/timeline") } fsout = gettemp() cmd = sprintf("fossil timeline -n 0 %s-t ci --oneline -R %s >%s 2>&1", branch_opt, _repo, fsout) system(cmd) total_commits = 0 while ((getline 0) { if (/^... end of timeline /) { sub(/^\(/, "", $5) sub(/\)$/, "", $5) total_commits = $5 } } close(fsout) unlink(fsout) refs["count"] = 1 refs[1] = sprintf("fossil timeline %s%s-t ci --medium", branch_opt, _timeline_opt) cmd = sprintf("fossil timeline %s%s-t ci --medium -R %s >%s 2>&1", branch_opt, _timeline_opt, _repo, fsout) system(cmd) if (length(branch) > 0) { info(out, sprintf("# %s / Timeline / Branch %s", _conf["project-name"], branch)) } else { info(out, sprintf("# %s / Timeline", _conf["project-name"])) } menu(out, "Timeline") author = "unknown" commit = "" date = "unknown" comment = "" while ((getline < fsout) > 0) { if ($1 == "Commit:") { commit = $2 m++ commits[m] = commit } else if ($1 == "Date:") { sub(/^Date: */, "") date = $0 } else if ($1 == "Author:") { sub(/^Author: */, "") author = $0 } else if ($1 == "Comment:") { sub(/^Comment: */, "") comment = $0 } else if (length($0) == 0) { if (length(comment) == 0) { comment_abbrev = commit } else if (length(comment) < 42) { comment_abbrev = comment } else { comment_abbrev = substr(comment, 1, 37) "..." } label = sprintf("%-17s %-41s %s", substr(date, 1, 16), comment_abbrev, author) sel = _root "/info/" commit item(out, 1, label, sel, _server, _port) } } close(fsout) unlink(fsout) if (m < total_commits) { info(out, " " total_commits - m \ " more commits remaining, fetch the repository") } reference(out, refs) close(out) for (i = 1; i <= m; i++) { generate_commit(commits[i]) } return } function generate_wiki( cmd, fsout, i, m, out, pages, refs, sel) { mkdir(_work "/wiki") out = _work "/wiki/gophermap" unlink(out) fsout = gettemp() cmd = sprintf("fossil wiki ls -R %s >%s 2>&1", _repo, fsout) system(cmd) refs["count"] = 1 refs[1] = "fossil wiki ls" info(out, sprintf("# %s / Wiki", _conf["project-name"])) menu(out, "Wiki") m = 0 while ((getline < fsout) > 0) { m++ pages[m] = $0 sel = _root "/wiki/" safe_filename($0) ".txt" item(out, 0, $0, sel, _server, _port) } close(fsout) unlink(fsout) reference(out, refs) close(out) for (i = 1; i <= m; i++) { generate_wiki_page(pages[i]) } return } function generate_wiki_page(page, cmd, fsout, out) { if (_verbose) { print "Generating wiki page (" page ") ..." } out = sprintf("%s/wiki/%s.txt", _work, safe_filename(page)) unlink(out) fsout = gettemp() cmd = sprintf("fossil wiki export -h \"%s\" -R %s >%s 2>&1", page, _repo, fsout) system(cmd) printf "# %s\n\n", page >out cmd = sprintf("webdump -ilr -w 60 <%s", fsout) while ((cmd | getline) > 0) { print >>out } printf "\nReference:\n\n" >>out printf "fossil wiki export -h \"%s\"\n", page >>out close(out) close(fsout) unlink(fsout) return } function generate_zip(commit, cmd, file, name, retval) { if (_verbose) { print "Generating zip (" commit ") ..." } mkdir(_work "/zip/" commit) name = _conf["short-project-name"] file = sprintf("%s/zip/%s/%s-%s.zip", _work, commit, name, commit) cmd = sprintf("fossil zip %s %s --name %s -R %s", commit, file, name, _repo) system(cmd) retval = sprintf("fossil zip %s %s-%s.zip --name %s", commit, name, commit, name) return retval } function gettemp( cmd, result, retval) { cmd = "mktemp" while ((cmd | getline) > 0) { retval = $0 } result = close(cmd) if (result != 0) { print "Error: mktemp failed exit status: " result exit } if (length(retval) == 0) { print "Error: mktemp failed, no tmpfile" exit } return retval } function info(out, str) { if (length(out) == 0) { printf "i%s\tErr\t%s\t%s\r\n", str, _server, _port } else { printf "i%s\tErr\t%s\t%s\r\n", str, _server, _port >>out } return } function item(out, type, label, sel, host, port, line) { line = item_str(type, label, sel, host, port) if (length(out) == 0) { printf "%s\r\n", line } else { printf "%s\r\n", line >>out } return } function item_str(type, label, sel, host, port) { retval = sprintf("%s%s\t%s\t%s\t%s", type, label, sel, host, port) return retval } function main( cmd, format) { if (ARGC < 3) { usage() exit 0 } _repo = ARGV[1] _root = ARGV[2] _download = 5 _port = 70 _server = "localhost" _timeline = 100 _verbose = 0 _work = "output" format = "gopher" for (i = 3; i < ARGC; i++) { if (ARGV[i] == "--dir") { _work = ARGV[i + 1] i++ } else if (ARGV[i] == "--download") { _download = ARGV[i + 1] i++ } else if (ARGV[i] == "--format") { format = ARGV[i + 1] i++ if (format != "geomyidae" && format != "gopher") { print "Error: Unknown format: " format exit 1 } } else if (ARGV[i] == "--port") { _port = ARGV[i + 1] i++ } else if (ARGV[i] == "--server") { _server = ARGV[i + 1] i++ } else if (ARGV[i] == "--timeline") { _timeline = ARGV[i + 1] i++ } else if (ARGV[i] == "--verbose" || ARGV[i] == "-v") { _verbose = 1 } else { print "Error: Unrecognized option: " ARGV[i] exit 1 } } if (_timeline > 0) { _timeline_opt = "-n " _timeline " " } else { _timeline_opt = "" } _download_count = 0 fossil_configuration(_conf) if (length(_conf["project-name"]) == 0) { print "Error: Could not find fossil project name" exit 1 } if (length(_conf["short-project-name"]) == 0) { _conf["short-project-name"] = tolower(safe_filename(substr( \ _conf["project-name"], 1, 8))) } _remote = fossil_remote() mkdir(_work) generate_home() generate_tickets() generate_wiki() quit_if_uptodate(_work) generate_timeline("") generate_files() generate_branches() generate_tags() if (format == "geomyidae") { cmd = sprintf("./geomyidae.sh \"%s\"", _work) system(cmd) } return } function menu(out, cur, i, label, opts, path) { opts[1] = "Timeline" opts[2] = "Files" opts[3] = "Branches" opts[4] = "Tags" opts[5] = "Tickets" opts[6] = "Wiki" path[1] = "timeline" path[2] = "files" path[3] = "branches" path[4] = "tags" path[5] = "tickets" path[6] = "wiki" item(out, "h", "fossil clone " _remote, "URL:" _remote, _server, _port) for (i = 1; i < 7; i++) { if (cur == opts[i]) { label = sprintf("(%s)", opts[i]) } else { label = sprintf(" %s ", opts[i]) } item(out, 1, label, _root "/" path[i], _server, _port) } info(out, "---") return } function mkdir(dir) { system("mkdir -p " dir) return } function println(out, str) { if (length(out) == 0) { print str } else { print str >>out } return } function quit_if_uptodate(workdir, cmd, path) { cmd = "fossil timeline -n 1 --oneline -R " _repo if ((cmd | getline) > 0) { path = workdir "/info/" $1 if (exists(path "/gophermap") || exists(path "/index.gph")) { if (_verbose) { print "Directory (" _work ") up-to-date, quitting early..." } exit 0 } } close(cmd) return } function reference(out, refs, i) { info(out, "") info(out, "# Reference") info(out, "") for (i = 1; i <= refs["count"]; i++) { if (length(refs[i]) > 65) { wrap("info", out, refs[i], 65) } else { info(out, refs[i]) } } return } function safe_filename(name) { gsub(/:/, "_", name) gsub(/\\/, "_", name) gsub(/\//, "_", name) gsub(/ /, "_", name) return name } function unlink(filename, cmd) { cmd = sprintf("rm %s 2>/dev/null", filename) system(cmd) return } function usage() { print "Usage: coprolit.awk REPO ROOT options" print "" print "REPO = file.fossil" print "ROOT = Gopher root selector" print "" print "Options:" print "--dir (default: output)" print "--download (default: 5)" print " limit number of tarballs per section, 0 = none" print "--format (geomyidae | gopher)" print "--port portnum" print "--server hostname" print "--timeline (default: 100)" print " limit number of items in timeline, 0 = unlimited" print "--verbose (or -v)" print "" return } # wrap() will break long lines into line continuations function wrap(which, out, str, len) { line = 1 buf = str while (length(buf) > len) { chunk = substr(buf, 1, len) if (match(chunk, / [^ ]*$/)) { before = substr(buf, 1, RSTART-1) after = substr(buf, RSTART+1) if (which == "info") { info(out, " " before) } else { println(out, " " before) } buf = after } else if (match(chunk, /-[^-]*$/)) { before = substr(buf, 1, RSTART) after = substr(buf, RSTART+1) if (which == "info") { info(out, " " before) } else { println(out, " " before) } buf = after } else { break } line++ } if (which == "info") { info(out, " " buf) } else { println(out, " " buf) } return } BEGIN { main() }