content_map.go - hugo - [fork] hugo port for 9front
(HTM) git clone https://git.drkhsh.at/hugo.git
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) Submodules
(DIR) README
(DIR) LICENSE
---
content_map.go (11972B)
---
1 // Copyright 2019 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 package hugolib
15
16 import (
17 "context"
18 "fmt"
19 "path"
20 "path/filepath"
21 "strings"
22 "unicode"
23
24 "github.com/bep/logg"
25 "github.com/gohugoio/hugo/common/hugio"
26 "github.com/gohugoio/hugo/common/paths"
27 "github.com/gohugoio/hugo/hugofs/files"
28 "github.com/gohugoio/hugo/hugolib/pagesfromdata"
29 "github.com/gohugoio/hugo/identity"
30 "github.com/gohugoio/hugo/source"
31
32 "github.com/gohugoio/hugo/resources/page"
33 "github.com/gohugoio/hugo/resources/page/pagemeta"
34 "github.com/gohugoio/hugo/resources/resource"
35
36 "github.com/gohugoio/hugo/hugofs"
37 )
38
39 // Used to mark ambiguous keys in reverse index lookups.
40 var ambiguousContentNode = &pageState{}
41
42 var trimCutsetDotSlashSpace = func(r rune) bool {
43 return r == '.' || r == '/' || unicode.IsSpace(r)
44 }
45
46 type contentMapConfig struct {
47 lang string
48 taxonomyConfig taxonomiesConfigValues
49 taxonomyDisabled bool
50 taxonomyTermDisabled bool
51 pageDisabled bool
52 isRebuild bool
53 }
54
55 var _ contentNodeI = (*resourceSource)(nil)
56
57 type resourceSource struct {
58 langIndex int
59 path *paths.Path
60 opener hugio.OpenReadSeekCloser
61 fi hugofs.FileMetaInfo
62 rc *pagemeta.ResourceConfig
63
64 r resource.Resource
65 }
66
67 func (r resourceSource) clone() *resourceSource {
68 r.r = nil
69 return &r
70 }
71
72 func (r *resourceSource) LangIndex() int {
73 return r.langIndex
74 }
75
76 func (r *resourceSource) MarkStale() {
77 resource.MarkStale(r.r)
78 }
79
80 func (r *resourceSource) resetBuildState() {
81 if rr, ok := r.r.(buildStateReseter); ok {
82 rr.resetBuildState()
83 }
84 }
85
86 func (r *resourceSource) isPage() bool {
87 _, ok := r.r.(page.Page)
88 return ok
89 }
90
91 func (r *resourceSource) GetIdentity() identity.Identity {
92 if r.r != nil {
93 return r.r.(identity.IdentityProvider).GetIdentity()
94 }
95 return r.path
96 }
97
98 func (r *resourceSource) ForEeachIdentity(f func(identity.Identity) bool) bool {
99 return f(r.GetIdentity())
100 }
101
102 func (r *resourceSource) Path() string {
103 return r.path.Path()
104 }
105
106 func (r *resourceSource) isContentNodeBranch() bool {
107 return false
108 }
109
110 var _ contentNodeI = (*resourceSources)(nil)
111
112 type resourceSources []*resourceSource
113
114 func (n resourceSources) MarkStale() {
115 for _, r := range n {
116 if r != nil {
117 r.MarkStale()
118 }
119 }
120 }
121
122 func (n resourceSources) Path() string {
123 panic("not supported")
124 }
125
126 func (n resourceSources) isContentNodeBranch() bool {
127 return false
128 }
129
130 func (n resourceSources) resetBuildState() {
131 for _, r := range n {
132 if r != nil {
133 r.resetBuildState()
134 }
135 }
136 }
137
138 func (n resourceSources) GetIdentity() identity.Identity {
139 for _, r := range n {
140 if r != nil {
141 return r.GetIdentity()
142 }
143 }
144 return nil
145 }
146
147 func (n resourceSources) ForEeachIdentity(f func(identity.Identity) bool) bool {
148 for _, r := range n {
149 if r != nil {
150 if f(r.GetIdentity()) {
151 return true
152 }
153 }
154 }
155 return false
156 }
157
158 func (cfg contentMapConfig) getTaxonomyConfig(s string) (v viewName) {
159 for _, n := range cfg.taxonomyConfig.views {
160 if strings.HasPrefix(s, n.pluralTreeKey) {
161 return n
162 }
163 }
164 return
165 }
166
167 func (m *pageMap) insertPageWithLock(s string, p *pageState) (contentNodeI, contentNodeI, bool) {
168 u, n, replaced := m.treePages.InsertIntoValuesDimensionWithLock(s, p)
169
170 if replaced && !m.s.h.isRebuild() && m.s.conf.PrintPathWarnings {
171 var messageDetail string
172 if p1, ok := n.(*pageState); ok && p1.File() != nil {
173 messageDetail = fmt.Sprintf(" file: %q", p1.File().Filename())
174 }
175 if p2, ok := u.(*pageState); ok && p2.File() != nil {
176 messageDetail += fmt.Sprintf(" file: %q", p2.File().Filename())
177 }
178
179 m.s.Log.Warnf("Duplicate content path: %q%s", s, messageDetail)
180 }
181
182 return u, n, replaced
183 }
184
185 func (m *pageMap) insertResourceWithLock(s string, r contentNodeI) (contentNodeI, contentNodeI, bool) {
186 u, n, replaced := m.treeResources.InsertIntoValuesDimensionWithLock(s, r)
187 if replaced {
188 m.handleDuplicateResourcePath(s, r, n)
189 }
190 return u, n, replaced
191 }
192
193 func (m *pageMap) insertResource(s string, r contentNodeI) (contentNodeI, contentNodeI, bool) {
194 u, n, replaced := m.treeResources.InsertIntoValuesDimension(s, r)
195 if replaced {
196 m.handleDuplicateResourcePath(s, r, n)
197 }
198 return u, n, replaced
199 }
200
201 func (m *pageMap) handleDuplicateResourcePath(s string, updated, existing contentNodeI) {
202 if m.s.h.isRebuild() || !m.s.conf.PrintPathWarnings {
203 return
204 }
205 var messageDetail string
206 if r1, ok := existing.(*resourceSource); ok && r1.fi != nil {
207 messageDetail = fmt.Sprintf(" file: %q", r1.fi.Meta().Filename)
208 }
209 if r2, ok := updated.(*resourceSource); ok && r2.fi != nil {
210 messageDetail += fmt.Sprintf(" file: %q", r2.fi.Meta().Filename)
211 }
212
213 m.s.Log.Warnf("Duplicate resource path: %q%s", s, messageDetail)
214 }
215
216 func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCount uint64, resourceCount uint64, addErr error) {
217 if fi.IsDir() {
218 return
219 }
220
221 insertResource := func(fim hugofs.FileMetaInfo) error {
222 resourceCount++
223 pi := fi.Meta().PathInfo
224 key := pi.Base()
225 tree := m.treeResources
226
227 commit := tree.Lock(true)
228 defer commit()
229
230 r := func() (hugio.ReadSeekCloser, error) {
231 return fim.Meta().Open()
232 }
233
234 var rs *resourceSource
235 if pi.IsContent() {
236 // Create the page now as we need it at assembly time.
237 // The other resources are created if needed.
238 pageResource, pi, err := m.s.h.newPage(
239 &pageMeta{
240 f: source.NewFileInfo(fim),
241 pathInfo: pi,
242 bundled: true,
243 },
244 )
245 if err != nil {
246 return err
247 }
248 if pageResource == nil {
249 // Disabled page.
250 return nil
251 }
252 key = pi.Base()
253
254 rs = &resourceSource{r: pageResource, langIndex: pageResource.s.languagei}
255 } else {
256 rs = &resourceSource{path: pi, opener: r, fi: fim, langIndex: fim.Meta().LangIndex}
257 }
258
259 _, _, _ = m.insertResource(key, rs)
260
261 return nil
262 }
263
264 meta := fi.Meta()
265 pi := meta.PathInfo
266
267 switch pi.Type() {
268 case paths.TypeFile, paths.TypeContentResource:
269 m.s.Log.Trace(logg.StringFunc(
270 func() string {
271 return fmt.Sprintf("insert resource: %q", fi.Meta().Filename)
272 },
273 ))
274 if err := insertResource(fi); err != nil {
275 addErr = err
276 return
277 }
278 case paths.TypeContentData:
279 pc, rc, err := m.addPagesFromGoTmplFi(fi, buildConfig)
280 pageCount += pc
281 resourceCount += rc
282 if err != nil {
283 addErr = err
284 return
285 }
286
287 default:
288 m.s.Log.Trace(logg.StringFunc(
289 func() string {
290 return fmt.Sprintf("insert bundle: %q", fi.Meta().Filename)
291 },
292 ))
293
294 pageCount++
295
296 // A content file.
297 p, pi, err := m.s.h.newPage(
298 &pageMeta{
299 f: source.NewFileInfo(fi),
300 pathInfo: pi,
301 bundled: false,
302 },
303 )
304 if err != nil {
305 addErr = err
306 return
307 }
308 if p == nil {
309 // Disabled page.
310 return
311 }
312
313 m.insertPageWithLock(pi.Base(), p)
314
315 }
316 return
317 }
318
319 func (m *pageMap) addPagesFromGoTmplFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCount uint64, resourceCount uint64, addErr error) {
320 meta := fi.Meta()
321 pi := meta.PathInfo
322
323 m.s.Log.Trace(logg.StringFunc(
324 func() string {
325 return fmt.Sprintf("insert pages from data file: %q", fi.Meta().Filename)
326 },
327 ))
328
329 if !files.IsGoTmplExt(pi.Ext()) {
330 addErr = fmt.Errorf("unsupported data file extension %q", pi.Ext())
331 return
332 }
333
334 s := m.s.h.resolveSite(fi.Meta().Lang)
335 f := source.NewFileInfo(fi)
336 h := s.h
337
338 contentAdapter := s.pageMap.treePagesFromTemplateAdapters.Get(pi.Base())
339 var rebuild bool
340 if contentAdapter != nil {
341 // Rebuild
342 contentAdapter = contentAdapter.CloneForGoTmpl(fi)
343 rebuild = true
344 } else {
345 contentAdapter = pagesfromdata.NewPagesFromTemplate(
346 pagesfromdata.PagesFromTemplateOptions{
347 GoTmplFi: fi,
348 Site: s,
349 DepsFromSite: func(s page.Site) pagesfromdata.PagesFromTemplateDeps {
350 ss := s.(*Site)
351 return pagesfromdata.PagesFromTemplateDeps{
352 TemplateStore: ss.GetTemplateStore(),
353 }
354 },
355 DependencyManager: s.Conf.NewIdentityManager("pagesfromdata"),
356 Watching: s.Conf.Watching(),
357 HandlePage: func(pt *pagesfromdata.PagesFromTemplate, pc *pagemeta.PageConfig) error {
358 s := pt.Site.(*Site)
359 if err := pc.CompileForPagesFromDataPre(pt.GoTmplFi.Meta().PathInfo.Base(), m.s.Log, s.conf.MediaTypes.Config); err != nil {
360 return err
361 }
362
363 ps, pi, err := h.newPage(
364 &pageMeta{
365 f: f,
366 s: s,
367 pageMetaParams: &pageMetaParams{
368 pageConfig: pc,
369 },
370 },
371 )
372 if err != nil {
373 return err
374 }
375
376 if ps == nil {
377 // Disabled page.
378 return nil
379 }
380
381 u, n, replaced := s.pageMap.insertPageWithLock(pi.Base(), ps)
382
383 if h.isRebuild() {
384 if replaced {
385 pt.AddChange(n.GetIdentity())
386 } else {
387 pt.AddChange(u.GetIdentity())
388 // New content not in use anywhere.
389 // To make sure that these gets listed in any site.RegularPages ranges or similar
390 // we could invalidate everything, but first try to collect a sample set
391 // from the surrounding pages.
392 var surroundingIDs []identity.Identity
393 ids := h.pageTrees.collectIdentitiesSurrounding(pi.Base(), 10)
394 if len(ids) > 0 {
395 surroundingIDs = append(surroundingIDs, ids...)
396 } else {
397 // No surrounding pages found, so invalidate everything.
398 surroundingIDs = []identity.Identity{identity.GenghisKhan}
399 }
400 for _, id := range surroundingIDs {
401 pt.AddChange(id)
402 }
403 }
404 }
405
406 return nil
407 },
408 HandleResource: func(pt *pagesfromdata.PagesFromTemplate, rc *pagemeta.ResourceConfig) error {
409 s := pt.Site.(*Site)
410 if err := rc.Compile(
411 pt.GoTmplFi.Meta().PathInfo.Base(),
412 s.Conf.PathParser(),
413 s.conf.MediaTypes.Config,
414 ); err != nil {
415 return err
416 }
417
418 rs := &resourceSource{path: rc.PathInfo, rc: rc, opener: nil, fi: pt.GoTmplFi, langIndex: s.languagei}
419
420 _, n, replaced := s.pageMap.insertResourceWithLock(rc.PathInfo.Base(), rs)
421
422 if h.isRebuild() && replaced {
423 pt.AddChange(n.GetIdentity())
424 }
425 return nil
426 },
427 },
428 )
429
430 s.pageMap.treePagesFromTemplateAdapters.Insert(pi.Base(), contentAdapter)
431
432 }
433
434 handleBuildInfo := func(s *Site, bi pagesfromdata.BuildInfo) {
435 resourceCount += bi.NumResourcesAdded
436 pageCount += bi.NumPagesAdded
437 s.handleContentAdapterChanges(bi, buildConfig)
438 }
439
440 bi, err := contentAdapter.Execute(context.Background())
441 if err != nil {
442 addErr = err
443 return
444 }
445 handleBuildInfo(s, bi)
446
447 if !rebuild && bi.EnableAllLanguages {
448 // Clone and insert the adapter for the other sites.
449 for _, ss := range s.h.Sites {
450 if s == ss {
451 continue
452 }
453
454 clone := contentAdapter.CloneForSite(ss)
455
456 // Make sure it gets executed for the first time.
457 bi, err := clone.Execute(context.Background())
458 if err != nil {
459 addErr = err
460 return
461 }
462 handleBuildInfo(ss, bi)
463
464 // Insert into the correct language tree so it get rebuilt on changes.
465 ss.pageMap.treePagesFromTemplateAdapters.Insert(pi.Base(), clone)
466
467 }
468 }
469
470 return
471 }
472
473 // The home page is represented with the zero string.
474 // All other keys starts with a leading slash. No trailing slash.
475 // Slashes are Unix-style.
476 func cleanTreeKey(elem ...string) string {
477 var s string
478 if len(elem) > 0 {
479 s = elem[0]
480 if len(elem) > 1 {
481 s = path.Join(elem...)
482 }
483 }
484 s = strings.TrimFunc(s, trimCutsetDotSlashSpace)
485 s = filepath.ToSlash(strings.ToLower(paths.Sanitize(s)))
486 if s == "" || s == "/" {
487 return ""
488 }
489 if s[0] != '/' {
490 s = "/" + s
491 }
492 return s
493 }