Optimising by adding threading, combining HTML output to one file. - staticgit - A git static site generator, the site you are viewing now!
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
---
(DIR) commit 89f13ce268930a8a931f4a82312653c96ac8d8ca
(DIR) parent 884ed087159dcf350421c692b8c530b7db260c28
(HTM) Author: Jay Scott <me@jay.scot>
Date: Sat, 13 Jul 2024 19:37:19 +0100
Optimising by adding threading, combining HTML output to one file.
Diffstat:
M Makefile | 2 +-
M main.go | 452 ++++++++++++++-----------------
2 files changed, 210 insertions(+), 244 deletions(-)
---
(DIR) diff --git a/Makefile b/Makefile
@@ -8,7 +8,7 @@ all: run
run:
@echo "Running $(APP_NAME)..."
- @go run . -g -p ./git -o tmp -i .ssh,dotfiles
+ @go run . -g -p ./git -o tmp -i .ssh,jay.scot,internal-docs
build:
@echo "Building $(APP_NAME) for local architecture..."
(DIR) diff --git a/main.go b/main.go
@@ -10,23 +10,19 @@ import (
"regexp"
"sort"
"strings"
+ "sync"
"github.com/go-git/go-git/v5"
- "github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
-type BranchInfo struct {
- Name string
- LastCommit string
- LastCommitDate string
-}
-
type CommitInfo struct {
Hash string
Author string
Date string
Message string
+ Added int
+ Removed int
}
type RepoInfo struct {
@@ -36,109 +32,124 @@ type RepoInfo struct {
}
const (
- baseTemplate = `
+ base = `
<!DOCTYPE html>
<html>
<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-<meta name="viewport" content="width=device-width, initial-scale=1" />
-<title>{{.Title}}</title>
-<link rel="icon" type="image/png" href="{{.IconPath}}favicon.png" />
-<link rel="stylesheet" type="text/css" href="{{.IconPath}}style.css" />
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
+ <title>{{.Title}}</title>
+ <link rel="icon" type="image/png" href="{{.IconPath}}favicon.png" />
+ <style>
+ body{color:#000;background-color:#FFF;font-family:monospace}
+ table{padding-left:20px}
+ h1{font-size:2em;margin:10}
+ h2{font-size:1.5em;margin:10}
+ table td{padding:0 .4em}
+ a{color:#000;text-decoration:none}
+ a:hover{color:#333;font-weight:700}
+ a:target{background-color:#CCC}
+ .desc{color:#555;margin-bottom:1em}
+ hr{border:0;border-top:1px solid #AAA;height:2px}
+ table tr:hover{background-color:#EEE}
+ </style>
</head>
<body>
-<table>
-<tr><td><img src="{{.IconPath}}logo.png" alt="" width="32" height="32" /></td>
-<td><span class="desc">{{.Title}}</span></td></tr><tr><td></td><td>
-</td></tr>
-</table>
-<hr/>
-<div id="content">
-{{template "content" .}}
-</div>
+ <table>
+ <tr>
+ <td>
+ <img src="{{.IconPath}}logo.png" alt="" width="32" height="32" />
+ </td>
+ <td>
+ <span class="desc">{{.Title}}</span>
+ </td>
+ </tr>
+ </table>
+
+ <hr/>
+
+ <div id="content">
+ {{template "content" .}}
+ </div>
</body>
</html>
`
-
- branchesContent = `
-{{define "content"}}
-<h1>{{.RepoName}} - Branches</h1>
-<table>
-<thead>
-<tr><td><b>Branch Name</b></td><td><b>Last Commit</b></td><td><b>Last Commit Date</b></td></tr>
-</thead>
-<tbody>
-{{range .Branches}}
-<tr><td>{{.Name}}</td><td>{{.LastCommit}}</td><td>{{.LastCommitDate}}</td></tr>
-{{end}}
-</tbody>
-</table>
-{{end}}
-`
-
- commitHistoryContent = `
+ details = `
{{define "content"}}
-<h1>{{.RepoName}} - Commit History</h1>
-<table>
-<thead>
-<tr><td><b>Hash</b></td><td><b>Author</b></td><td><b>Date</b></td><td><b>Message</b></td></tr>
-</thead>
-<tbody>
-{{range .Commits}}
-<tr><td>{{.Hash}}</td><td>{{.Author}}</td><td>{{.Date}}</td><td>{{.Message}}</td></tr>
-{{end}}
-</tbody>
-</table>
+ <pre>{{.ReadmeContent}}</pre>
+
+ <h2>Commit History</h2>
+ <hr>
+ <table>
+ <thead>
+ <tr>
+ <td><b>Date</b></td>
+ <td><b>Message</b></td>
+ <td><b>Author</b></td>
+ <td><b>Hash</b></td>
+ <td><b>Added</b></td>
+ <td><b>Deleted</b></td>
+ </tr>
+ </thead>
+ <tbody>
+ {{range .Commits}}
+ <tr>
+ <td>{{.Date}}</td>
+ <td>{{.Message}}</td>
+ <td>{{.Author}}</td>
+ <td>{{.Hash}}</td>
+ <td style="color: green;">+{{.Added}}</td>
+ <td style="color: red;">-{{.Removed}}</td>
+ </tr>
+ {{end}}
+ </tbody>
+ </table>
{{end}}
`
- indexContent = `
-{{define "content"}}
-<table id="index">
-<thead>
-<tr><td><b>Name</b></td><td><b>Description</b></td><td><b>Last commit</b></td><td><b>Links</b></td></tr>
-</thead>
-<tbody>
-{{range $group, $repos := .Repos}}
-<tr><td colspan="4"><b>{{if eq $group ""}} {{else}}<hr>{{end}}</b></td></tr>
- {{range $repos}}
- <tr>
- <td>{{.Name}}</td>
- <td>{{.Description}}</td>
- <td>| {{.LastCommit}} | </td>
- <td>
- <a href="{{.Name}}/README.html">readme</a> |
- <a href="{{.Name}}/commits.html">commits</a> |
- <a href="{{.Name}}/branches.html">branches</a>
- </td>
- </tr>
- {{end}}
-{{end}}
-</tbody>
-</table>
-{{end}}
-`
- readmeContent = `
+ index = `
{{define "content"}}
-<h1>{{.RepoName}}</h1>
-<pre>{{.ReadmeContent}}</pre>
+ <table id="index">
+ <thead>
+ <tr>
+ <td><b>Name</b></td>
+ <td><b>Description</b></td>
+ <td><b>Last commit</b></td>
+ </tr>
+ </thead>
+ <tbody>
+ {{range $group, $repos := .Repos}}
+ <tr>
+ <td colspan="4">{{if eq $group ""}}{{else}}<hr>{{end}}</td>
+ </tr>
+ {{range $repos}}
+ <tr>
+ <td><a href="{{.Name}}/index.html">{{.Name}}</a></td>
+ <td>{{.Description}}</td>
+ <td>{{.LastCommit}}</td>
+ </tr>
+ {{end}}
+ {{end}}
+ </tbody>
+ </table>
{{end}}
`
)
var (
- reposPath string
- groupFlag bool
- ignoreDirs map[string]bool
- outputRoot string
-
- branchesTmpl = template.Must(template.New("base").Parse(baseTemplate + branchesContent))
- commitHistoryTmpl = template.Must(template.New("base").Parse(baseTemplate + commitHistoryContent))
- indexTmpl = template.Must(template.New("base").Parse(baseTemplate + indexContent))
- readmeTmpl = template.Must(template.New("base").Parse(baseTemplate + readmeContent))
+ templates = map[string]*template.Template{
+ "index": template.Must(template.New("base").Parse(base + index)),
+ "details": template.Must(template.New("base").Parse(base + details)),
+ }
+
+ reposPath string
+ groupFlag bool
+ ignoreDirs map[string]bool
+ outputRoot string
+ commitLimit int
)
-func generateIndexHTML(repoInfos []RepoInfo) error {
+func generateIndex(repoInfos []RepoInfo) error {
groupedRepos := groupRepos(repoInfos)
indexOutputPath := filepath.Join(outputRoot, "index.html")
@@ -148,64 +159,95 @@ func generateIndexHTML(repoInfos []RepoInfo) error {
}
defer indexFile.Close()
- return indexTmpl.Execute(indexFile, struct {
+ return templates["index"].Execute(indexFile, struct {
Title string
IconPath string
Repos map[string][]RepoInfo
}{
- Title: "git clone git@git.jay.scot:<reponame>",
+ Title: "repos for days!",
IconPath: "./",
Repos: groupedRepos,
})
}
-func getBranchInfo(repo *git.Repository) ([]BranchInfo, error) {
- branches, err := repo.Branches()
+func generateRepo(repoName, repoPath, outputDir string) error {
+ repo, err := git.PlainOpen(repoPath)
if err != nil {
- return nil, fmt.Errorf("failed to get branches: %w", err)
+ return fmt.Errorf("failed to open git repository: %w", err)
}
- var branchInfos []BranchInfo
- err = branches.ForEach(func(branch *plumbing.Reference) error {
- commit, err := repo.CommitObject(branch.Hash())
- if err != nil {
- return fmt.Errorf("failed to get commit for branch %s: %w", branch.Name().Short(), err)
- }
-
- branchInfos = append(branchInfos, BranchInfo{
- Name: branch.Name().Short(),
- LastCommit: commit.Hash.String()[:7],
- LastCommitDate: commit.Author.When.Format("02 Jan 2006 15:04:05"),
- })
- return nil
- })
+ readme, err := getReadme(repoPath)
+ if err != nil {
+ return fmt.Errorf("failed to get README: %w", err)
+ }
+ commits, err := getCommits(repo)
if err != nil {
- return nil, fmt.Errorf("failed to iterate over branches: %w", err)
+ return fmt.Errorf("failed to get commit history: %w", err)
}
- return branchInfos, nil
-}
+ outputPath := filepath.Join(outputDir, "index.html")
-func getCommitHistory(repo *git.Repository) ([]CommitInfo, error) {
- ref, err := repo.Head()
+ f, err := os.Create(outputPath)
if err != nil {
- return nil, fmt.Errorf("failed to get HEAD reference: %w", err)
+ return fmt.Errorf("failed to create details HTML file: %w", err)
}
+ defer f.Close()
- commits, err := repo.Log(&git.LogOptions{From: ref.Hash()})
+ return templates["details"].Execute(f, struct {
+ Title string
+ IconPath string
+ RepoName string
+ ReadmeContent string
+ Commits []CommitInfo
+ }{
+ Title: "git clone git@git.jay.scot:" + repoName,
+ IconPath: "../",
+ RepoName: repoName,
+ ReadmeContent: readme,
+ Commits: commits,
+ })
+}
+
+func getCommits(repo *git.Repository) ([]CommitInfo, error) {
+ commitIter, err := repo.CommitObjects()
if err != nil {
- return nil, fmt.Errorf("failed to get commit log: %w", err)
+ return nil, fmt.Errorf("failed to get commit objects: %w", err)
}
var commitHistory []CommitInfo
- err = commits.ForEach(func(c *object.Commit) error {
+ count := 0
+ reachedLimit := false
+
+ err = commitIter.ForEach(func(c *object.Commit) error {
+ if reachedLimit {
+ return nil
+ }
+
+ stats, err := c.Stats()
+ if err != nil {
+ return fmt.Errorf("failed to get commit stats: %w", err)
+ }
+
+ added, removed := 0, 0
+ for _, stat := range stats {
+ added += stat.Addition
+ removed += stat.Deletion
+ }
+
commitHistory = append(commitHistory, CommitInfo{
Hash: c.Hash.String()[:7],
Author: c.Author.Name,
Date: c.Author.When.Format("02 Jan 2006 15:04:05"),
- Message: strings.Split(c.Message, "\n")[0], // Get only the first line of the commit message
+ Message: strings.Split(c.Message, "\n")[0],
+ Added: added,
+ Removed: removed,
})
+
+ count++
+ if count >= commitLimit {
+ reachedLimit = true
+ }
return nil
})
@@ -261,7 +303,7 @@ func getReadme(repoPath string) (string, error) {
return "No README found!", nil
}
-func getRepoInfo(repoPath string) (RepoInfo, error) {
+func getRepo(repoPath string) (RepoInfo, error) {
repo, err := git.PlainOpen(repoPath)
if err != nil {
return RepoInfo{}, fmt.Errorf("failed to open repository: %w", err)
@@ -313,7 +355,7 @@ func getGroup(description string) string {
return ""
}
-func parseIgnoreDirs(ignoreDirs string) map[string]bool {
+func parseIgnored(ignoreDirs string) map[string]bool {
ignoreMap := make(map[string]bool)
for _, dir := range strings.Split(ignoreDirs, ",") {
if trimmedDir := strings.TrimSpace(dir); trimmedDir != "" {
@@ -323,149 +365,73 @@ func parseIgnoreDirs(ignoreDirs string) map[string]bool {
return ignoreMap
}
-func processBranches(repoName, repoPath, outputDir string) error {
- repo, err := git.PlainOpen(repoPath)
- if err != nil {
- return fmt.Errorf("failed to open git repository: %w", err)
- }
-
- branches, err := getBranchInfo(repo)
- if err != nil {
- return fmt.Errorf("failed to get branch information: %w", err)
- }
-
- outputPath := filepath.Join(outputDir, "branches.html")
-
- f, err := os.Create(outputPath)
- if err != nil {
- return fmt.Errorf("failed to create branches HTML file: %w", err)
- }
- defer f.Close()
-
- return branchesTmpl.Execute(f, struct {
- Title string
- IconPath string
- RepoName string
- Branches []BranchInfo
- }{
- Title: repoName + " - Branches",
- IconPath: "../",
- RepoName: repoName,
- Branches: branches,
- })
-}
-
-func processCommitHistory(repoName, repoPath, outputDir string) error {
- repo, err := git.PlainOpen(repoPath)
- if err != nil {
- return fmt.Errorf("failed to open git repository: %w", err)
- }
-
- commits, err := getCommitHistory(repo)
- if err != nil {
- return fmt.Errorf("failed to get commit history: %w", err)
- }
-
- outputPath := filepath.Join(outputDir, "commits.html")
-
- f, err := os.Create(outputPath)
- if err != nil {
- return fmt.Errorf("failed to create commit history HTML file: %w", err)
- }
- defer f.Close()
-
- return commitHistoryTmpl.Execute(f, struct {
- Title string
- IconPath string
- RepoName string
- Commits []CommitInfo
- }{
- Title: repoName + " - History",
- IconPath: "../",
- RepoName: repoName,
- Commits: commits,
- })
-}
-
-func processReadme(repoName, repoPath, outputDir string) error {
- readme, err := getReadme(repoPath)
+func processRepos() error {
+ repos, err := os.ReadDir(reposPath)
if err != nil {
- return fmt.Errorf("failed to get README: %w", err)
+ return fmt.Errorf("failed to read repos directory: %w", err)
}
- outputPath := filepath.Join(outputDir, "README.html")
+ var wg sync.WaitGroup
+ repoInfosChan := make(chan RepoInfo, len(repos))
+ errorsChan := make(chan error, len(repos))
- f, err := os.Create(outputPath)
- if err != nil {
- return fmt.Errorf("failed to create README HTML file: %w", err)
+ for _, r := range repos {
+ if r.IsDir() && !ignoreDirs[r.Name()] {
+ wg.Add(1)
+ go func(r os.DirEntry) {
+ defer wg.Done()
+ repoPath := filepath.Join(reposPath, r.Name())
+ repoInfo, err := getRepo(repoPath)
+ if err != nil {
+ errorsChan <- fmt.Errorf("failed to get info for repo %s: %w", r.Name(), err)
+ return
+ }
+ repoInfosChan <- repoInfo
+
+ outputDir := filepath.Join(outputRoot, r.Name())
+ if err := os.MkdirAll(outputDir, 0755); err != nil {
+ errorsChan <- fmt.Errorf("failed to create output directory for repo %s: %w", r.Name(), err)
+ return
+ }
+
+ if err := generateRepo(r.Name(), repoPath, outputDir); err != nil {
+ errorsChan <- fmt.Errorf("failed to process repo %s: %w", r.Name(), err)
+ }
+ }(r)
+ }
}
- defer f.Close()
- return readmeTmpl.Execute(f, struct {
- Title string
- IconPath string
- RepoName string
- ReadmeContent string
- }{
- Title: repoName + " - Readme!",
- IconPath: "../",
- RepoName: repoName,
- ReadmeContent: readme,
- })
-}
+ go func() {
+ wg.Wait()
+ close(repoInfosChan)
+ close(errorsChan)
+ }()
-func processRepositories() error {
- repos, err := os.ReadDir(reposPath)
- if err != nil {
- return fmt.Errorf("failed to read repos directory: %w", err)
+ var repoInfos []RepoInfo
+ for repoInfo := range repoInfosChan {
+ repoInfos = append(repoInfos, repoInfo)
}
- var repoInfos []RepoInfo
- for _, r := range repos {
- if r.IsDir() && !ignoreDirs[r.Name()] {
- repoPath := filepath.Join(reposPath, r.Name())
- repoInfo, err := getRepoInfo(repoPath)
- if err != nil {
- fmt.Printf("Failed to get info for repo %s: %v\n", r.Name(), err)
- continue
- }
- repoInfos = append(repoInfos, repoInfo)
-
- outputDir := filepath.Join(outputRoot, r.Name())
- if err := os.MkdirAll(outputDir, 0755); err != nil {
- fmt.Printf("Failed to create output directory for repo %s: %v\n", r.Name(), err)
- continue
- }
-
- if err := processReadme(r.Name(), repoPath, outputDir); err != nil {
- fmt.Printf("Failed to process README for repo %s: %v\n", r.Name(), err)
- }
-
- if err := processCommitHistory(r.Name(), repoPath, outputDir); err != nil {
- fmt.Printf("Failed to process commit history for repo %s: %v\n", r.Name(), err)
- }
-
- if err := processBranches(r.Name(), repoPath, outputDir); err != nil {
- fmt.Printf("Failed to process branches for repo %s: %v\n", r.Name(), err)
- }
- }
+ for err := range errorsChan {
+ fmt.Println(err)
}
- return generateIndexHTML(repoInfos)
+ return generateIndex(repoInfos)
}
func main() {
flag.StringVar(&reposPath, "p", "", "Path to the git repositories (required)")
+ flag.StringVar(&outputRoot, "o", ".", "Root path where output directories will be created")
flag.BoolVar(&groupFlag, "g", false, "Group repositories based on description tags")
+ flag.IntVar(&commitLimit, "c", 100, "Limit for the number of commits to display (default 100)")
ignoreFlag := flag.String("i", "", "Directories to ignore (comma-separated)")
- flag.StringVar(&outputRoot, "o", ".", "Root path where output directories will be created")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s [options]\n\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Options:\n")
flag.PrintDefaults()
fmt.Fprintf(os.Stderr, "\nExample:\n")
- fmt.Fprintf(os.Stderr, " %s -p /path/to/repos -g -i dir1,dir2 -o /path/to/output\n", os.Args[0])
+ fmt.Fprintf(os.Stderr, " %s -p /path/to/repos -g -i dir1,dir2 -o /path/to/output -c50\n", os.Args[0])
}
flag.Parse()
@@ -475,9 +441,9 @@ func main() {
os.Exit(1)
}
- ignoreDirs = parseIgnoreDirs(*ignoreFlag)
+ ignoreDirs = parseIgnored(*ignoreFlag)
- if err := processRepositories(); err != nil {
+ if err := processRepos(); err != nil {
log.Fatalf("Error processing repositories: %v", err)
}
}