Fix module mount in sub folder - hugo - [fork] hugo port for 9front
(HTM) git clone git@git.drkhsh.at/hugo.git
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) Submodules
(DIR) README
(DIR) LICENSE
---
(DIR) commit 80dd6ddde27ce36f5432fb780e94d4974b5277c7
(DIR) parent 299731012441378bb9c057ceb0a3c277108aaf01
(HTM) Author: Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
Date: Fri, 31 Jan 2020 17:15:14 +0100
Fix module mount in sub folder
This addresses a specific issue, but is a also a major simplification of the filesystem file mounts.
Fixes #6730
Diffstat:
M hugofs/decorators.go | 3 +--
M hugofs/fileinfo.go | 34 ++++++++++++++++++++++++++-----
M hugofs/nosymlink_test.go | 1 +
M hugofs/rootmapping_fs.go | 566 +++++++++++++++----------------
M hugofs/rootmapping_fs_test.go | 146 ++++++++++++++++++++++++-------
M hugofs/walk.go | 10 +++++++++-
M hugofs/walk_test.go | 21 +++++++++++++++++++++
M hugolib/filesystems/basefs.go | 1 +
M hugolib/filesystems/basefs_test.go | 2 --
M hugolib/hugo_modules_test.go | 45 +++++++++++++++++++++++++++++++
10 files changed, 494 insertions(+), 335 deletions(-)
---
(DIR) diff --git a/hugofs/decorators.go b/hugofs/decorators.go
@@ -79,7 +79,7 @@ func DecorateBasePathFs(base *afero.BasePathFs) afero.Fs {
}
// NewBaseFileDecorator decorates the given Fs to provide the real filename
-// and an Opener func. If
+// and an Opener func.
func NewBaseFileDecorator(fs afero.Fs) afero.Fs {
ffs := &baseFileDecoratorFs{Fs: fs}
@@ -102,7 +102,6 @@ func NewBaseFileDecorator(fs afero.Fs) afero.Fs {
opener := func() (afero.File, error) {
return ffs.open(filename)
-
}
return decorateFileInfo(fi, ffs, opener, filename, "", meta), nil
(DIR) diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go
@@ -18,6 +18,7 @@ import (
"os"
"path/filepath"
"runtime"
+ "sort"
"strings"
"time"
@@ -271,13 +272,21 @@ func (fi *dirNameOnlyFileInfo) Sys() interface{} {
return nil
}
-func newDirNameOnlyFileInfo(name string, isOrdered bool, fileOpener func() (afero.File, error)) FileMetaInfo {
+func newDirNameOnlyFileInfo(name string, meta FileMeta, isOrdered bool, fileOpener func() (afero.File, error)) FileMetaInfo {
name = normalizeFilename(name)
_, base := filepath.Split(name)
- return NewFileMetaInfo(&dirNameOnlyFileInfo{name: base}, FileMeta{
- metaKeyFilename: name,
- metaKeyIsOrdered: isOrdered,
- metaKeyOpener: fileOpener})
+
+ m := copyFileMeta(meta)
+ if _, found := m[metaKeyFilename]; !found {
+ m.setIfNotZero(metaKeyFilename, name)
+ }
+ m[metaKeyOpener] = fileOpener
+ m[metaKeyIsOrdered] = isOrdered
+
+ return NewFileMetaInfo(
+ &dirNameOnlyFileInfo{name: base},
+ m,
+ )
}
func decorateFileInfo(
@@ -339,3 +348,18 @@ func fileInfosToNames(fis []os.FileInfo) []string {
}
return names
}
+
+func fromSlash(filenames []string) []string {
+ for i, name := range filenames {
+ filenames[i] = filepath.FromSlash(name)
+ }
+ return filenames
+}
+
+func sortFileInfos(fis []os.FileInfo) {
+ sort.Slice(fis, func(i, j int) bool {
+ fimi, fimj := fis[i].(FileMetaInfo), fis[j].(FileMetaInfo)
+ return fimi.Meta().Filename() < fimj.Meta().Filename()
+
+ })
+}
(DIR) diff --git a/hugofs/nosymlink_test.go b/hugofs/nosymlink_test.go
@@ -137,6 +137,7 @@ func TestNoSymlinkFs(t *testing.T) {
c.Assert(err, qt.IsNil)
// There is at least one unsported symlink inside workDir
_, err = f.Readdir(-1)
+ c.Assert(err, qt.IsNil)
f.Close()
c.Assert(logger.WarnCounter.Count(), qt.Equals, uint64(1))
(DIR) diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go
@@ -27,15 +27,18 @@ import (
"github.com/spf13/afero"
)
-var filepathSeparator = string(filepath.Separator)
+var (
+ filepathSeparator = string(filepath.Separator)
+)
// NewRootMappingFs creates a new RootMappingFs on top of the provided with
-// of root mappings with some optional metadata about the root.
+// root mappings with some optional metadata about the root.
// Note that From represents a virtual root that maps to the actual filename in To.
func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
rootMapToReal := radix.New()
+ var virtualRoots []RootMapping
- for i, rm := range rms {
+ for _, rm := range rms {
(&rm).clean()
fromBase := files.ResolveComponentFolder(rm.From)
@@ -56,11 +59,13 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
}
// Extract "blog" from "content/blog"
rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator)
- if rm.Meta != nil {
- rm.Meta[metaKeyBaseDir] = rm.ToBasedir
- rm.Meta[metaKeyMountRoot] = rm.path
+ if rm.Meta == nil {
+ rm.Meta = make(FileMeta)
}
+ rm.Meta[metaKeyBaseDir] = rm.ToBasedir
+ rm.Meta[metaKeyMountRoot] = rm.path
+
meta := copyFileMeta(rm.Meta)
if !fi.IsDir() {
@@ -70,7 +75,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
rm.fi = NewFileMetaInfo(fi, meta)
- key := rm.rootKey()
+ key := filepathSeparator + rm.From
var mappings []RootMapping
v, found := rootMapToReal.Get(key)
if found {
@@ -80,30 +85,38 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
mappings = append(mappings, rm)
rootMapToReal.Insert(key, mappings)
- rms[i] = rm
+ virtualRoots = append(virtualRoots, rm)
}
- rfs := &RootMappingFs{Fs: fs,
- virtualRoots: rms,
- rootMapToReal: rootMapToReal}
+ rootMapToReal.Insert(filepathSeparator, virtualRoots)
+
+ rfs := &RootMappingFs{
+ Fs: fs,
+ rootMapToReal: rootMapToReal,
+ }
return rfs, nil
}
-// NewRootMappingFsFromFromTo is a convenicence variant of NewRootMappingFs taking
-// From and To as string pairs.
-func NewRootMappingFsFromFromTo(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) {
+func newRootMappingFsFromFromTo(
+ baseDir string,
+ fs afero.Fs,
+ fromTo ...string,
+) (*RootMappingFs, error) {
+
rms := make([]RootMapping, len(fromTo)/2)
for i, j := 0, 0; j < len(fromTo); i, j = i+1, j+2 {
rms[i] = RootMapping{
- From: fromTo[j],
- To: fromTo[j+1],
+ From: fromTo[j],
+ To: fromTo[j+1],
+ ToBasedir: baseDir,
}
}
return NewRootMappingFs(fs, rms...)
}
+// RootMapping describes a virtual file or directory mount.
type RootMapping struct {
From string // The virtual mount.
To string // The source directory or file.
@@ -127,21 +140,16 @@ func (r RootMapping) filename(name string) string {
return filepath.Join(r.To, strings.TrimPrefix(name, r.From))
}
-func (r RootMapping) rootKey() string {
- return r.From
-}
-
// A RootMappingFs maps several roots into one. Note that the root of this filesystem
// is directories only, and they will be returned in Readdir and Readdirnames
// in the order given.
type RootMappingFs struct {
afero.Fs
rootMapToReal *radix.Tree
- virtualRoots []RootMapping
- filter func(r RootMapping) bool
}
func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {
+ base = filepathSeparator + fs.cleanName(base)
roots := fs.getRootsWithPrefix(base)
if roots == nil {
@@ -176,138 +184,46 @@ func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {
return fss, nil
}
-// LstatIfPossible returns the os.FileInfo structure describing a given file.
-func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
- fis, _, b, err := fs.doLstat(name, false)
- if err != nil {
- return nil, b, err
- }
- return fis[0], b, nil
-}
-
-func (fs *RootMappingFs) virtualDirOpener(name string, isRoot bool) func() (afero.File, error) {
- return func() (afero.File, error) { return &rootMappingFile{name: name, isRoot: isRoot, fs: fs}, nil }
-}
-
-func (fs *RootMappingFs) doLstat(name string, allowMultiple bool) ([]FileMetaInfo, []FileMetaInfo, bool, error) {
- if fs.isRoot(name) {
- return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, true))}, nil, false, nil
- }
-
- roots := fs.getRoots(name)
- rootsWithPrefix := fs.getRootsWithPrefix(name)
- hasRootMappingsBelow := len(rootsWithPrefix) != 0
-
- if len(roots) == 0 {
- if hasRootMappingsBelow {
- // No exact matches, but we have root mappings below name,
- // let's make it look like a directory.
- return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, false))}, nil, false, nil
- }
-
- return nil, nil, false, os.ErrNotExist
- }
-
- // We may have a mapping for both static and static/subdir.
- // These will not show in any Readdir so append them
- // manually.
- rootsInDir := fs.filterRootsBelow(rootsWithPrefix, name)
-
- var (
- fis []FileMetaInfo
- dirs []FileMetaInfo
- b bool
- root RootMapping
- err error
- )
-
- for _, root = range roots {
- var fi os.FileInfo
- fi, b, err = fs.statRoot(root, name)
- if err != nil {
- if os.IsNotExist(err) {
- continue
+// Filter creates a copy of this filesystem with only mappings matching a filter.
+func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs {
+ rootMapToReal := radix.New()
+ fs.rootMapToReal.Walk(func(b string, v interface{}) bool {
+ rms := v.([]RootMapping)
+ var nrms []RootMapping
+ for _, rm := range rms {
+ if f(rm) {
+ nrms = append(nrms, rm)
}
- return nil, nil, false, err
}
- fim := fi.(FileMetaInfo)
-
- fis = append(fis, fim)
- }
-
- for _, root = range rootsInDir {
-
- fi, _, err := fs.statRoot(root, "")
- if err != nil {
- if os.IsNotExist(err) {
- continue
- }
- return nil, nil, false, err
+ if len(nrms) != 0 {
+ rootMapToReal.Insert(b, nrms)
}
- fim := fi.(FileMetaInfo)
- dirs = append(dirs, fim)
- }
-
- if len(fis) == 0 && len(dirs) == 0 {
- return nil, nil, false, os.ErrNotExist
- }
-
- if allowMultiple || len(fis) == 1 {
- return fis, dirs, b, nil
- }
-
- if len(fis) == 0 {
- return nil, nil, false, os.ErrNotExist
- }
-
- // Open it in this composite filesystem.
- opener := func() (afero.File, error) {
- return fs.Open(name)
- }
+ return false
+ })
- return []FileMetaInfo{decorateFileInfo(fis[0], fs, opener, "", "", root.Meta)}, nil, b, nil
+ fs.rootMapToReal = rootMapToReal
+ return &fs
}
-// Open opens the namedrootMappingFile file for reading.
-func (fs *RootMappingFs) Open(name string) (afero.File, error) {
- if fs.isRoot(name) {
- return &rootMappingFile{name: name, fs: fs, isRoot: true}, nil
- }
-
- fis, dirs, _, err := fs.doLstat(name, true)
+// LstatIfPossible returns the os.FileInfo structure describing a given file.
+func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+ fis, err := fs.doLstat(name)
if err != nil {
- return nil, err
+ return nil, false, err
}
+ return fis[0], false, nil
+}
- if len(fis) == 1 {
- fi := fis[0]
- meta := fi.(FileMetaInfo).Meta()
- f, err := meta.Open()
- if err != nil {
- return nil, err
- }
-
- f = &rootMappingFile{File: f, fs: fs, name: name, meta: meta}
-
- if len(dirs) > 0 {
- return &readDirDirsAppender{File: f, dirs: dirs}, nil
- }
-
- return f, nil
- }
+// Open opens the named file for reading.
+func (fs *RootMappingFs) Open(name string) (afero.File, error) {
+ fis, err := fs.doLstat(name)
- f, err := fs.newUnionFile(fis...)
if err != nil {
return nil, err
}
- if len(dirs) > 0 {
- return &readDirDirsAppender{File: f, dirs: dirs}, nil
- }
-
- return f, nil
-
+ return fs.newUnionFile(fis...)
}
// Stat returns the os.FileInfo structure describing a given file. If there is
@@ -318,80 +234,51 @@ func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) {
}
-// Filter creates a copy of this filesystem with the applied filter.
-func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs {
- fs.filter = f
- return &fs
-}
-
-func (fs *RootMappingFs) isRoot(name string) bool {
- return name == "" || name == filepathSeparator
+func (fs *RootMappingFs) hasPrefix(prefix string) bool {
+ hasPrefix := false
+ fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool {
+ hasPrefix = true
+ return true
+ })
+ return hasPrefix
}
-func (fs *RootMappingFs) getRoots(name string) []RootMapping {
- name = filepath.Clean(name)
- _, v, found := fs.rootMapToReal.LongestPrefix(name)
+func (fs *RootMappingFs) getRoot(key string) []RootMapping {
+ v, found := fs.rootMapToReal.Get(key)
if !found {
return nil
}
- rm := v.([]RootMapping)
-
- return fs.applyFilterToRoots(rm)
+ return v.([]RootMapping)
}
-func (fs *RootMappingFs) applyFilterToRoots(rm []RootMapping) []RootMapping {
- if fs.filter == nil {
- return rm
+func (fs *RootMappingFs) getRoots(key string) (string, []RootMapping) {
+ s, v, found := fs.rootMapToReal.LongestPrefix(key)
+ if !found || (s == filepathSeparator && key != filepathSeparator) {
+ return "", nil
}
+ return s, v.([]RootMapping)
- var filtered []RootMapping
- for _, m := range rm {
- if fs.filter(m) {
- filtered = append(filtered, m)
- }
- }
+}
+
+func (fs *RootMappingFs) debug() {
+ fmt.Println("debug():")
+ fs.rootMapToReal.Walk(func(s string, v interface{}) bool {
+ fmt.Println("Key", s)
+ return false
+ })
- return filtered
}
func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping {
- if fs.isRoot(prefix) {
- return fs.virtualRoots
- }
- prefix = filepath.Clean(prefix)
var roots []RootMapping
-
fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool {
roots = append(roots, v.([]RootMapping)...)
return false
})
- return fs.applyFilterToRoots(roots)
-}
-
-// Filter out the mappings inside the name directory.
-func (fs *RootMappingFs) filterRootsBelow(roots []RootMapping, name string) []RootMapping {
- if len(roots) == 0 {
- return nil
- }
-
- sepCount := strings.Count(name, filepathSeparator)
- var filtered []RootMapping
- for _, x := range roots {
- if name == x.From {
- continue
- }
-
- if strings.Count(x.From, filepathSeparator)-sepCount != 1 {
- continue
- }
-
- filtered = append(filtered, x)
-
- }
- return filtered
+ return roots
}
func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {
@@ -400,6 +287,10 @@ func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {
if err != nil {
return nil, err
}
+ if len(fis) == 1 {
+ return f, nil
+ }
+
rf := &rootMappingFile{File: f, fs: fs, name: meta.Name(), meta: meta}
if len(fis) == 1 {
return rf, err
@@ -439,148 +330,241 @@ func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {
}
-func (fs *RootMappingFs) statRoot(root RootMapping, name string) (os.FileInfo, bool, error) {
- filename := root.filename(name)
+func (fs *RootMappingFs) cleanName(name string) string {
+ return strings.Trim(filepath.Clean(name), filepathSeparator)
+}
+
+func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error) {
+ prefix = filepathSeparator + fs.cleanName(prefix)
+
+ var fis []os.FileInfo
- var b bool
- var fi os.FileInfo
- var err error
+ seen := make(map[string]bool) // Prevent duplicate directories
+ level := strings.Count(prefix, filepathSeparator)
- if ls, ok := fs.Fs.(afero.Lstater); ok {
- fi, b, err = ls.LstatIfPossible(filename)
+ // First add any real files/directories.
+ rms := fs.getRoot(prefix)
+ for _, rm := range rms {
+ f, err := rm.fi.Meta().Open()
if err != nil {
- return nil, b, err
+ return nil, err
}
-
- } else {
- fi, err = fs.Fs.Stat(filename)
+ direntries, err := f.Readdir(-1)
if err != nil {
- return nil, b, err
+ f.Close()
+ return nil, err
}
- }
- // Opens the real directory/file.
- opener := func() (afero.File, error) {
- return fs.Fs.Open(filename)
- }
+ for _, fi := range direntries {
+ meta := fi.(FileMetaInfo).Meta()
+ mergeFileMeta(rm.Meta, meta)
+ if fi.IsDir() {
+ name := fi.Name()
+ if seen[name] {
+ continue
+ }
+ seen[name] = true
+ opener := func() (afero.File, error) {
+ return fs.Open(filepath.Join(rm.From, name))
+ }
+ fi = newDirNameOnlyFileInfo(name, meta, false, opener)
+ }
- if fi.IsDir() {
- if name == "" {
- name = root.From
+ fis = append(fis, fi)
}
- _, name = filepath.Split(name)
- fi = newDirNameOnlyFileInfo(name, false, opener)
- }
-
- return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil
-}
-
-type rootMappingFile struct {
- afero.File
- fs *RootMappingFs
- name string
- meta FileMeta
- isRoot bool
-}
-
-type readDirDirsAppender struct {
- afero.File
- dirs []FileMetaInfo
-}
-
-func (f *readDirDirsAppender) Readdir(count int) ([]os.FileInfo, error) {
- fis, err := f.File.Readdir(count)
- if err != nil {
- return nil, err
+ f.Close()
}
- for _, dir := range f.dirs {
- fis = append(fis, dir)
- }
- return fis, nil
-
-}
-
-func (f *readDirDirsAppender) Readdirnames(count int) ([]string, error) {
- fis, err := f.Readdir(count)
- if err != nil {
- return nil, err
- }
- return fileInfosToNames(fis), nil
-}
+ // Next add any file mounts inside the given directory.
+ prefixInside := prefix + filepathSeparator
+ fs.rootMapToReal.WalkPrefix(prefixInside, func(s string, v interface{}) bool {
-func (f *rootMappingFile) Close() error {
- if f.File == nil {
- return nil
- }
- return f.File.Close()
-}
+ if (strings.Count(s, filepathSeparator) - level) != 1 {
+ // This directory is not part of the current, but we
+ // need to include the first name part to make it
+ // navigable.
+ path := strings.TrimPrefix(s, prefixInside)
+ parts := strings.Split(path, filepathSeparator)
+ name := parts[0]
-func (f *rootMappingFile) Name() string {
- return f.name
-}
+ if seen[name] {
+ return false
+ }
+ seen[name] = true
+ opener := func() (afero.File, error) {
+ return fs.Open(path)
+ }
-func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {
- if f.File == nil {
- filesn := make([]os.FileInfo, 0)
- roots := f.fs.getRootsWithPrefix(f.name)
- seen := make(map[string]bool) // Do not return duplicate directories
+ fi := newDirNameOnlyFileInfo(name, nil, false, opener)
+ fis = append(fis, fi)
- j := 0
- for _, rm := range roots {
- if count != -1 && j >= count {
- break
- }
+ return false
+ }
+ rms := v.([]RootMapping)
+ for _, rm := range rms {
if !rm.fi.IsDir() {
// A single file mount
- filesn = append(filesn, rm.fi)
+ fis = append(fis, rm.fi)
continue
}
-
- from := rm.From
- name := from
- if !f.isRoot {
- _, name = filepath.Split(from)
- }
-
+ name := filepath.Base(rm.From)
if seen[name] {
continue
}
seen[name] = true
opener := func() (afero.File, error) {
- return f.fs.Open(from)
+ return fs.Open(rm.From)
}
- j++
+ fi := newDirNameOnlyFileInfo(name, rm.Meta, false, opener)
- fi := newDirNameOnlyFileInfo(name, false, opener)
+ fis = append(fis, fi)
- if rm.Meta != nil {
- mergeFileMeta(rm.Meta, fi.Meta())
+ }
+
+ return false
+ })
+
+ return fis, nil
+}
+
+func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {
+ name = fs.cleanName(name)
+ key := filepathSeparator + name
+
+ roots := fs.getRoot(key)
+
+ if roots == nil {
+ if fs.hasPrefix(key) {
+ // We have directories mounted below this.
+ // Make it look like a directory.
+ return []FileMetaInfo{newDirNameOnlyFileInfo(name, nil, true, fs.virtualDirOpener(name))}, nil
+ }
+
+ // Find any real files or directories with this key.
+ _, roots := fs.getRoots(key)
+ if roots == nil {
+ return nil, os.ErrNotExist
+ }
+
+ var err error
+ var fis []FileMetaInfo
+
+ for _, rm := range roots {
+ var fi FileMetaInfo
+ fi, _, err = fs.statRoot(rm, name)
+ if err == nil {
+ fis = append(fis, fi)
}
+ }
+
+ if fis != nil {
+ return fis, nil
+ }
- filesn = append(filesn, fi)
+ if err == nil {
+ err = os.ErrNotExist
}
- return filesn, nil
+
+ return nil, err
}
- if f.File == nil {
- panic(fmt.Sprintf("no File for %q", f.name))
+ fileCount := 0
+ for _, root := range roots {
+ if !root.fi.IsDir() {
+ fileCount++
+ }
+ if fileCount > 1 {
+ break
+ }
}
- fis, err := f.File.Readdir(count)
+ if fileCount == 0 {
+ // Dir only.
+ return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, true, fs.virtualDirOpener(name))}, nil
+ }
+
+ if fileCount > 1 {
+ // Not supported by this filesystem.
+ return nil, errors.Errorf("found multiple files with name %q, use .Readdir or the source filesystem directly", name)
+
+ }
+
+ return []FileMetaInfo{roots[0].fi}, nil
+
+}
+
+func (fs *RootMappingFs) statRoot(root RootMapping, name string) (FileMetaInfo, bool, error) {
+ filename := root.filename(name)
+
+ fi, b, err := lstatIfPossible(fs.Fs, filename)
if err != nil {
- return nil, err
+ return nil, b, err
}
- for i, fi := range fis {
- fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
+ var opener func() (afero.File, error)
+ if fi.IsDir() {
+ // Make sure metadata gets applied in Readdir.
+ opener = fs.realDirOpener(filename, root.Meta)
+ } else {
+ // Opens the real file directly.
+ opener = func() (afero.File, error) {
+ return fs.Fs.Open(filename)
+ }
}
- return fis, nil
+ return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil
+
+}
+
+func (fs *RootMappingFs) virtualDirOpener(name string) func() (afero.File, error) {
+ return func() (afero.File, error) { return &rootMappingFile{name: name, fs: fs}, nil }
+}
+
+func (fs *RootMappingFs) realDirOpener(name string, meta FileMeta) func() (afero.File, error) {
+ return func() (afero.File, error) {
+ f, err := fs.Fs.Open(name)
+ if err != nil {
+ return nil, err
+ }
+ return &rootMappingFile{name: name, meta: meta, fs: fs, File: f}, nil
+ }
+}
+
+type rootMappingFile struct {
+ afero.File
+ fs *RootMappingFs
+ name string
+ meta FileMeta
+}
+
+func (f *rootMappingFile) Close() error {
+ if f.File == nil {
+ return nil
+ }
+ return f.File.Close()
+}
+
+func (f *rootMappingFile) Name() string {
+ return f.name
+}
+
+func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {
+ if f.File != nil {
+ fis, err := f.File.Readdir(count)
+ if err != nil {
+ return nil, err
+ }
+
+ for i, fi := range fis {
+ fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
+ }
+ return fis, nil
+ }
+ return f.fs.collectDirEntries(f.name)
}
func (f *rootMappingFile) Readdirnames(count int) ([]string, error) {
(DIR) diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go
@@ -14,9 +14,10 @@
package hugofs
import (
+ "fmt"
"io/ioutil"
- "os"
"path/filepath"
+ "sort"
"testing"
"github.com/spf13/viper"
@@ -34,8 +35,12 @@ func TestLanguageRootMapping(t *testing.T) {
fs := NewBaseFileDecorator(afero.NewMemMapFs())
c.Assert(afero.WriteFile(fs, filepath.Join("content/sv/svdir", "main.txt"), []byte("main sv"), 0755), qt.IsNil)
+
c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "sv-f.txt"), []byte("some sv blog content"), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", "en-f.txt"), []byte("some en blog content in a"), 0755), qt.IsNil)
+ c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent/d1", "sv-d1-f.txt"), []byte("some sv blog content"), 0755), qt.IsNil)
+ c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent/d1", "en-d1-f.txt"), []byte("some en blog content in a"), 0755), qt.IsNil)
+
c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myotherenblogcontent", "en-f2.txt"), []byte("some en content"), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvdocs", "sv-docs.txt"), []byte("some sv docs content"), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join("themes/b/myenblogcontent", "en-b-f.txt"), []byte("some en content"), 0755), qt.IsNil)
@@ -72,19 +77,30 @@ func TestLanguageRootMapping(t *testing.T) {
collected, err := collectFilenames(rfs, "content", "content")
c.Assert(err, qt.IsNil)
- c.Assert(collected, qt.DeepEquals, []string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"})
-
- bfs := afero.NewBasePathFs(rfs, "content")
- collected, err = collectFilenames(bfs, "", "")
- c.Assert(err, qt.IsNil)
- c.Assert(collected, qt.DeepEquals, []string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"})
+ c.Assert(collected, qt.DeepEquals,
+ []string{"blog/d1/en-d1-f.txt", "blog/d1/sv-d1-f.txt", "blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}, qt.Commentf("%#v", collected))
dirs, err := rfs.Dirs(filepath.FromSlash("content/blog"))
c.Assert(err, qt.IsNil)
-
c.Assert(len(dirs), qt.Equals, 4)
+ for _, dir := range dirs {
+ f, err := dir.Meta().Open()
+ c.Assert(err, qt.IsNil)
+ f.Close()
+ }
+
+ blog, err := rfs.Open(filepath.FromSlash("content/blog"))
+ c.Assert(err, qt.IsNil)
+ fis, err := blog.Readdir(-1)
+ for _, fi := range fis {
+ f, err := fi.(FileMetaInfo).Meta().Open()
+ c.Assert(err, qt.IsNil)
+ f.Close()
+ }
+ blog.Close()
getDirnames := func(name string, rfs *RootMappingFs) []string {
+ c.Helper()
filename := filepath.FromSlash(name)
f, err := rfs.Open(filename)
c.Assert(err, qt.IsNil)
@@ -109,16 +125,16 @@ func TestLanguageRootMapping(t *testing.T) {
return rm.Meta.Lang() == "en"
})
- c.Assert(getDirnames("content/blog", rfsEn), qt.DeepEquals, []string{"en-f.txt", "en-f2.txt"})
+ c.Assert(getDirnames("content/blog", rfsEn), qt.DeepEquals, []string{"d1", "en-f.txt", "en-f2.txt"})
rfsSv := rfs.Filter(func(rm RootMapping) bool {
return rm.Meta.Lang() == "sv"
})
- c.Assert(getDirnames("content/blog", rfsSv), qt.DeepEquals, []string{"sv-f.txt", "svdir"})
+ c.Assert(getDirnames("content/blog", rfsSv), qt.DeepEquals, []string{"d1", "sv-f.txt", "svdir"})
// Make sure we have not messed with the original
- c.Assert(getDirnames("content/blog", rfs), qt.DeepEquals, []string{"sv-f.txt", "en-f.txt", "svdir", "en-f2.txt"})
+ c.Assert(getDirnames("content/blog", rfs), qt.DeepEquals, []string{"d1", "sv-f.txt", "en-f.txt", "svdir", "en-f2.txt"})
c.Assert(getDirnames("content", rfsSv), qt.DeepEquals, []string{"blog", "docs"})
c.Assert(getDirnames("content", rfs), qt.DeepEquals, []string{"blog", "docs"})
@@ -135,7 +151,7 @@ func TestRootMappingFsDirnames(t *testing.T) {
c.Assert(fs.Mkdir("f3t", 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join("f2t", testfile), []byte("some content"), 0755), qt.IsNil)
- rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", "f1t", "static/cf2", "f2t", "static/af3", "f3t")
+ rfs, err := newRootMappingFsFromFromTo("", fs, "static/bf1", "f1t", "static/cf2", "f2t", "static/af3", "f3t")
c.Assert(err, qt.IsNil)
fif, err := rfs.Stat(filepath.Join("static/cf2", testfile))
@@ -144,12 +160,12 @@ func TestRootMappingFsDirnames(t *testing.T) {
fifm := fif.(FileMetaInfo).Meta()
c.Assert(fifm.Filename(), qt.Equals, filepath.FromSlash("f2t/myfile.txt"))
- root, err := rfs.Open(filepathSeparator)
+ root, err := rfs.Open("static")
c.Assert(err, qt.IsNil)
dirnames, err := root.Readdirnames(-1)
c.Assert(err, qt.IsNil)
- c.Assert(dirnames, qt.DeepEquals, []string{"bf1", "cf2", "af3"})
+ c.Assert(dirnames, qt.DeepEquals, []string{"af3", "bf1", "cf2"})
}
@@ -165,7 +181,7 @@ func TestRootMappingFsFilename(t *testing.T) {
c.Assert(fs.MkdirAll(filepath.Join(workDir, "f1t/foo"), 0777), qt.IsNil)
c.Assert(afero.WriteFile(fs, testfilename, []byte("content"), 0666), qt.IsNil)
- rfs, err := NewRootMappingFsFromFromTo(fs, "static/f1", filepath.Join(workDir, "f1t"), "static/f2", filepath.Join(workDir, "f2t"))
+ rfs, err := newRootMappingFsFromFromTo(workDir, fs, "static/f1", filepath.Join(workDir, "f1t"), "static/f2", filepath.Join(workDir, "f2t"))
c.Assert(err, qt.IsNil)
fi, err := rfs.Stat(filepath.FromSlash("static/f1/foo/file.txt"))
@@ -256,12 +272,9 @@ func TestRootMappingFsMount(t *testing.T) {
c.Assert(err, qt.IsNil)
c.Assert(string(b), qt.Equals, "some no content")
- // Check file mappings
- single, err := rfs.Stat(filepath.FromSlash("content/singles/p1.md"))
- c.Assert(err, qt.IsNil)
- c.Assert(single.IsDir(), qt.Equals, false)
- singlem := single.(FileMetaInfo).Meta()
- c.Assert(singlem.Lang(), qt.Equals, "no") // First match
+ // Ambigous
+ _, err = rfs.Stat(filepath.FromSlash("content/singles/p1.md"))
+ c.Assert(err, qt.Not(qt.IsNil))
singlesDir, err := rfs.Open(filepath.FromSlash("content/singles"))
c.Assert(err, qt.IsNil)
@@ -308,19 +321,20 @@ func TestRootMappingFsMountOverlap(t *testing.T) {
rfs, err := NewRootMappingFs(fs, rm...)
c.Assert(err, qt.IsNil)
- getDirnames := func(name string) []string {
+ checkDirnames := func(name string, expect []string) {
+ c.Helper()
name = filepath.FromSlash(name)
f, err := rfs.Open(name)
c.Assert(err, qt.IsNil)
defer f.Close()
names, err := f.Readdirnames(-1)
c.Assert(err, qt.IsNil)
- return names
+ c.Assert(names, qt.DeepEquals, expect, qt.Commentf(fmt.Sprintf("%#v", names)))
}
- c.Assert(getDirnames("static"), qt.DeepEquals, []string{"a.txt", "b", "e"})
- c.Assert(getDirnames("static/b"), qt.DeepEquals, []string{"b.txt", "c"})
- c.Assert(getDirnames("static/b/c"), qt.DeepEquals, []string{"c.txt"})
+ checkDirnames("static", []string{"a.txt", "b", "e"})
+ checkDirnames("static/b", []string{"b.txt", "c"})
+ checkDirnames("static/b/c", []string{"c.txt"})
fi, err := rfs.Stat(filepath.FromSlash("static/b/b.txt"))
c.Assert(err, qt.IsNil)
@@ -330,32 +344,96 @@ func TestRootMappingFsMountOverlap(t *testing.T) {
func TestRootMappingFsOs(t *testing.T) {
c := qt.New(t)
- fs := afero.NewOsFs()
+ fs := NewBaseFileDecorator(afero.NewOsFs())
- d, err := ioutil.TempDir("", "hugo-root-mapping")
+ d, clean, err := htesting.CreateTempDir(fs, "hugo-root-mapping-os")
c.Assert(err, qt.IsNil)
- defer func() {
- os.RemoveAll(d)
- }()
+ defer clean()
testfile := "myfile.txt"
c.Assert(fs.Mkdir(filepath.Join(d, "f1t"), 0755), qt.IsNil)
c.Assert(fs.Mkdir(filepath.Join(d, "f2t"), 0755), qt.IsNil)
c.Assert(fs.Mkdir(filepath.Join(d, "f3t"), 0755), qt.IsNil)
+
+ // Deep structure
+ deepDir := filepath.Join(d, "d1", "d2", "d3", "d4", "d5")
+ c.Assert(fs.MkdirAll(deepDir, 0755), qt.IsNil)
+ for i := 1; i <= 3; i++ {
+ c.Assert(fs.MkdirAll(filepath.Join(d, "d1", "d2", "d3", "d4", fmt.Sprintf("d4-%d", i)), 0755), qt.IsNil)
+ c.Assert(afero.WriteFile(fs, filepath.Join(d, "d1", "d2", "d3", fmt.Sprintf("f-%d.txt", i)), []byte("some content"), 0755), qt.IsNil)
+ }
+
c.Assert(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755), qt.IsNil)
- rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", filepath.Join(d, "f1t"), "static/cf2", filepath.Join(d, "f2t"), "static/af3", filepath.Join(d, "f3t"))
+ rfs, err := newRootMappingFsFromFromTo(
+ d,
+ fs,
+ "static/bf1", filepath.Join(d, "f1t"),
+ "static/cf2", filepath.Join(d, "f2t"),
+ "static/af3", filepath.Join(d, "f3t"),
+ "static/a/b/c", filepath.Join(d, "d1", "d2", "d3"),
+ "layouts", filepath.Join(d, "d1"),
+ )
+
c.Assert(err, qt.IsNil)
fif, err := rfs.Stat(filepath.Join("static/cf2", testfile))
c.Assert(err, qt.IsNil)
c.Assert(fif.Name(), qt.Equals, "myfile.txt")
- root, err := rfs.Open(filepathSeparator)
+ root, err := rfs.Open("static")
c.Assert(err, qt.IsNil)
dirnames, err := root.Readdirnames(-1)
c.Assert(err, qt.IsNil)
- c.Assert(dirnames, qt.DeepEquals, []string{"bf1", "cf2", "af3"})
+ c.Assert(dirnames, qt.DeepEquals, []string{"a", "af3", "bf1", "cf2"}, qt.Commentf(fmt.Sprintf("%#v", dirnames)))
+ getDirnames := func(dirname string) []string {
+ dirname = filepath.FromSlash(dirname)
+ f, err := rfs.Open(dirname)
+ c.Assert(err, qt.IsNil)
+ defer f.Close()
+ dirnames, err := f.Readdirnames(-1)
+ c.Assert(err, qt.IsNil)
+ sort.Strings(dirnames)
+ return dirnames
+ }
+
+ c.Assert(getDirnames("static/a/b"), qt.DeepEquals, []string{"c"})
+ c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt"})
+ c.Assert(getDirnames("static/a/b/c/d4"), qt.DeepEquals, []string{"d4-1", "d4-2", "d4-3", "d5"})
+
+ all, err := collectFilenames(rfs, "static", "static")
+ c.Assert(err, qt.IsNil)
+
+ c.Assert(all, qt.DeepEquals, []string{"a/b/c/f-1.txt", "a/b/c/f-2.txt", "a/b/c/f-3.txt", "cf2/myfile.txt"})
+
+ fis, err := collectFileinfos(rfs, "static", "static")
+ c.Assert(err, qt.IsNil)
+
+ c.Assert(fis[9].Meta().PathFile(), qt.Equals, filepath.FromSlash("d1/d2/d3/f-1.txt"))
+
+ dirc := fis[3].Meta()
+
+ f, err := dirc.Open()
+ c.Assert(err, qt.IsNil)
+ defer f.Close()
+ fileInfos, err := f.Readdir(-1)
+ c.Assert(err, qt.IsNil)
+ sortFileInfos(fileInfos)
+ i := 0
+ for _, fi := range fileInfos {
+ if fi.IsDir() {
+ continue
+ }
+ i++
+ meta := fi.(FileMetaInfo).Meta()
+ c.Assert(meta.Filename(), qt.Equals, filepath.Join(d, fmt.Sprintf("/d1/d2/d3/f-%d.txt", i)))
+ c.Assert(meta.PathFile(), qt.Equals, filepath.FromSlash(fmt.Sprintf("d1/d2/d3/f-%d.txt", i)))
+ }
+
+ _, err = rfs.Stat(filepath.FromSlash("layouts/d2/d3/f-1.txt"))
+ c.Assert(err, qt.IsNil)
+ _, err = rfs.Stat(filepath.FromSlash("layouts/d2/d3"))
+ c.Assert(err, qt.IsNil)
}
(DIR) diff --git a/hugofs/walk.go b/hugofs/walk.go
@@ -124,7 +124,6 @@ func (w *Walkway) Walk() error {
if w.checkErr(w.root, err) {
return nil
}
-
return w.walkFn(w.root, nil, errors.Wrapf(err, "walk: %q", w.root))
}
fi = info.(FileMetaInfo)
@@ -154,6 +153,15 @@ func (w *Walkway) checkErr(filename string, err error) bool {
logUnsupportedSymlink(filename, w.logger)
return true
}
+
+ if os.IsNotExist(err) {
+ // The file may be removed in process.
+ // This may be a ERROR situation, but it is not possible
+ // to determine as a general case.
+ w.logger.WARN.Printf("File %q not found, skipping.", filename)
+ return true
+ }
+
return false
}
(DIR) diff --git a/hugofs/walk_test.go b/hugofs/walk_test.go
@@ -176,6 +176,27 @@ func collectFilenames(fs afero.Fs, base, root string) ([]string, error) {
}
+func collectFileinfos(fs afero.Fs, base, root string) ([]FileMetaInfo, error) {
+ var fis []FileMetaInfo
+
+ walkFn := func(path string, info FileMetaInfo, err error) error {
+ if err != nil {
+ return err
+ }
+
+ fis = append(fis, info)
+
+ return nil
+ }
+
+ w := NewWalkway(WalkwayConfig{Fs: fs, BasePath: base, Root: root, WalkFn: walkFn})
+
+ err := w.Walk()
+
+ return fis, err
+
+}
+
func BenchmarkWalk(b *testing.B) {
c := qt.New(b)
fs := NewBaseFileDecorator(afero.NewMemMapFs())
(DIR) diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go
@@ -258,6 +258,7 @@ func (s SourceFilesystems) MakeStaticPathRelative(filename string) string {
// MakePathRelative creates a relative path from the given filename.
// It will return an empty string if the filename is not a member of this filesystem.
func (d *SourceFilesystem) MakePathRelative(filename string) string {
+
for _, dir := range d.Dirs {
meta := dir.(hugofs.FileMetaInfo).Meta()
currentPath := meta.Filename()
(DIR) diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go
@@ -173,9 +173,7 @@ theme = ["atheme"]
filename = filepath.FromSlash(filename)
f, err := fs.Open(filename)
c.Assert(err, qt.IsNil)
- name := f.Name()
f.Close()
- c.Assert(name, qt.Equals, filename)
}
}
}
(DIR) diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go
@@ -38,6 +38,47 @@ import (
"github.com/spf13/viper"
)
+// https://github.com/gohugoio/hugo/issues/6730
+func TestHugoModulesTargetInSubFolder(t *testing.T) {
+ config := `
+baseURL="https://example.org"
+workingDir = %q
+
+[module]
+[[module.imports]]
+path="github.com/gohugoio/hugoTestModule2"
+ [[module.imports.mounts]]
+ source = "templates/hooks"
+ target = "layouts/_default/_markup"
+
+`
+
+ b := newTestSitesBuilder(t)
+ workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-target-in-subfolder-test")
+ b.Assert(err, qt.IsNil)
+ defer clean()
+ b.Fs = hugofs.NewDefault(viper.New())
+ b.WithWorkingDir(workingDir).WithConfigFile("toml", fmt.Sprintf(config, workingDir))
+ b.WithTemplates("_default/single.html", `{{ .Content }}`)
+ b.WithContent("p1.md", `---
+title: "Page"
+---
+
+[A link](https://bep.is)
+
+`)
+ b.WithSourceFile("go.mod", `
+module github.com/gohugoio/tests/testHugoModules
+
+
+`)
+
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/p1/index.html", `<p>Page|https://bep.is|Title: |Text: A link|END</p>`)
+
+}
+
// TODO(bep) this fails when testmodBuilder is also building ...
func TestHugoModules(t *testing.T) {
if !isCI() {
@@ -588,6 +629,9 @@ workingDir = %q
{{ $mypage := .Site.GetPage "/blog/mypage.md" }}
{{ with $mypage }}MYPAGE: {{ .Title }}|Path: {{ path.Join .File.Path }}|FilePath: {{ path.Join .File.FileInfo.Meta.PathFile }}|{{ end }}
+{{ $mybundle := .Site.GetPage "/blog/mybundle" }}
+{{ with $mybundle }}MYBUNDLE: {{ .Title }}|Path: {{ path.Join .File.Path }}|FilePath: {{ path.Join .File.FileInfo.Meta.PathFile }}|{{ end }}
+
`, "_default/_markup/render-link.html", `
{{ $link := .Destination }}
@@ -640,6 +684,7 @@ README: Readme Title
/README.md|Path: _index.md|FilePath: README.md
Readme Content.
MYPAGE: My Page|Path: blog/mypage.md|FilePath: mycontent/mypage.md|
+MYBUNDLE: My Bundle|Path: blog/mybundle/index.md|FilePath: mycontent/mybundle/index.md|
`)
b.AssertFileContent("public/blog/mypage/index.html", `
<a href="https://example.com/blog/mybundle/">Relative Link From Page</a>