deploy_test.go - hugo - [fork] hugo port for 9front
(HTM) git clone https://git.drkhsh.at/hugo.git
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) Submodules
(DIR) README
(DIR) LICENSE
---
deploy_test.go (32481B)
---
1 // Copyright 2019 The Hugo Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 //go:build withdeploy
15
16 package deploy
17
18 import (
19 "bytes"
20 "compress/gzip"
21 "context"
22 "crypto/md5"
23 "fmt"
24 "io"
25 "os"
26 "path"
27 "path/filepath"
28 "regexp"
29 "sort"
30 "testing"
31
32 "github.com/gohugoio/hugo/common/loggers"
33 "github.com/gohugoio/hugo/deploy/deployconfig"
34 "github.com/gohugoio/hugo/hugofs"
35 "github.com/gohugoio/hugo/media"
36 "github.com/google/go-cmp/cmp"
37 "github.com/google/go-cmp/cmp/cmpopts"
38 "github.com/spf13/afero"
39 "gocloud.dev/blob"
40 "gocloud.dev/blob/fileblob"
41 "gocloud.dev/blob/memblob"
42 )
43
44 func TestFindDiffs(t *testing.T) {
45 hash1 := []byte("hash 1")
46 hash2 := []byte("hash 2")
47 makeLocal := func(path string, size int64, hash []byte) *localFile {
48 return &localFile{NativePath: path, SlashPath: filepath.ToSlash(path), UploadSize: size, md5: hash}
49 }
50 makeRemote := func(path string, size int64, hash []byte) *blob.ListObject {
51 return &blob.ListObject{Key: path, Size: size, MD5: hash}
52 }
53
54 tests := []struct {
55 Description string
56 Local []*localFile
57 Remote []*blob.ListObject
58 Force bool
59 WantUpdates []*fileToUpload
60 WantDeletes []string
61 }{
62 {
63 Description: "empty -> no diffs",
64 },
65 {
66 Description: "local == remote -> no diffs",
67 Local: []*localFile{
68 makeLocal("aaa", 1, hash1),
69 makeLocal("bbb", 2, hash1),
70 makeLocal("ccc", 3, hash2),
71 },
72 Remote: []*blob.ListObject{
73 makeRemote("aaa", 1, hash1),
74 makeRemote("bbb", 2, hash1),
75 makeRemote("ccc", 3, hash2),
76 },
77 },
78 {
79 Description: "local w/ separators == remote -> no diffs",
80 Local: []*localFile{
81 makeLocal(filepath.Join("aaa", "aaa"), 1, hash1),
82 makeLocal(filepath.Join("bbb", "bbb"), 2, hash1),
83 makeLocal(filepath.Join("ccc", "ccc"), 3, hash2),
84 },
85 Remote: []*blob.ListObject{
86 makeRemote("aaa/aaa", 1, hash1),
87 makeRemote("bbb/bbb", 2, hash1),
88 makeRemote("ccc/ccc", 3, hash2),
89 },
90 },
91 {
92 Description: "local == remote with force flag true -> diffs",
93 Local: []*localFile{
94 makeLocal("aaa", 1, hash1),
95 makeLocal("bbb", 2, hash1),
96 makeLocal("ccc", 3, hash2),
97 },
98 Remote: []*blob.ListObject{
99 makeRemote("aaa", 1, hash1),
100 makeRemote("bbb", 2, hash1),
101 makeRemote("ccc", 3, hash2),
102 },
103 Force: true,
104 WantUpdates: []*fileToUpload{
105 {makeLocal("aaa", 1, nil), reasonForce},
106 {makeLocal("bbb", 2, nil), reasonForce},
107 {makeLocal("ccc", 3, nil), reasonForce},
108 },
109 },
110 {
111 Description: "local == remote with route.Force true -> diffs",
112 Local: []*localFile{
113 {NativePath: "aaa", SlashPath: "aaa", UploadSize: 1, matcher: &deployconfig.Matcher{Force: true}, md5: hash1},
114 makeLocal("bbb", 2, hash1),
115 },
116 Remote: []*blob.ListObject{
117 makeRemote("aaa", 1, hash1),
118 makeRemote("bbb", 2, hash1),
119 },
120 WantUpdates: []*fileToUpload{
121 {makeLocal("aaa", 1, nil), reasonForce},
122 },
123 },
124 {
125 Description: "extra local file -> upload",
126 Local: []*localFile{
127 makeLocal("aaa", 1, hash1),
128 makeLocal("bbb", 2, hash2),
129 },
130 Remote: []*blob.ListObject{
131 makeRemote("aaa", 1, hash1),
132 },
133 WantUpdates: []*fileToUpload{
134 {makeLocal("bbb", 2, nil), reasonNotFound},
135 },
136 },
137 {
138 Description: "extra remote file -> delete",
139 Local: []*localFile{
140 makeLocal("aaa", 1, hash1),
141 },
142 Remote: []*blob.ListObject{
143 makeRemote("aaa", 1, hash1),
144 makeRemote("bbb", 2, hash2),
145 },
146 WantDeletes: []string{"bbb"},
147 },
148 {
149 Description: "diffs in size or md5 -> upload",
150 Local: []*localFile{
151 makeLocal("aaa", 1, hash1),
152 makeLocal("bbb", 2, hash1),
153 makeLocal("ccc", 1, hash2),
154 },
155 Remote: []*blob.ListObject{
156 makeRemote("aaa", 1, nil),
157 makeRemote("bbb", 1, hash1),
158 makeRemote("ccc", 1, hash1),
159 },
160 WantUpdates: []*fileToUpload{
161 {makeLocal("aaa", 1, nil), reasonMD5Missing},
162 {makeLocal("bbb", 2, nil), reasonSize},
163 {makeLocal("ccc", 1, nil), reasonMD5Differs},
164 },
165 },
166 {
167 Description: "mix of updates and deletes",
168 Local: []*localFile{
169 makeLocal("same", 1, hash1),
170 makeLocal("updated", 2, hash1),
171 makeLocal("updated2", 1, hash2),
172 makeLocal("new", 1, hash1),
173 makeLocal("new2", 2, hash2),
174 },
175 Remote: []*blob.ListObject{
176 makeRemote("same", 1, hash1),
177 makeRemote("updated", 1, hash1),
178 makeRemote("updated2", 1, hash1),
179 makeRemote("stale", 1, hash1),
180 makeRemote("stale2", 1, hash1),
181 },
182 WantUpdates: []*fileToUpload{
183 {makeLocal("new", 1, nil), reasonNotFound},
184 {makeLocal("new2", 2, nil), reasonNotFound},
185 {makeLocal("updated", 2, nil), reasonSize},
186 {makeLocal("updated2", 1, nil), reasonMD5Differs},
187 },
188 WantDeletes: []string{"stale", "stale2"},
189 },
190 }
191
192 for _, tc := range tests {
193 t.Run(tc.Description, func(t *testing.T) {
194 local := map[string]*localFile{}
195 for _, l := range tc.Local {
196 local[l.SlashPath] = l
197 }
198 remote := map[string]*blob.ListObject{}
199 for _, r := range tc.Remote {
200 remote[r.Key] = r
201 }
202 d := newDeployer()
203 gotUpdates, gotDeletes := d.findDiffs(local, remote, tc.Force)
204 gotUpdates = applyOrdering(nil, gotUpdates)[0]
205 sort.Slice(gotDeletes, func(i, j int) bool { return gotDeletes[i] < gotDeletes[j] })
206 if diff := cmp.Diff(gotUpdates, tc.WantUpdates, cmpopts.IgnoreUnexported(localFile{})); diff != "" {
207 t.Errorf("updates differ:\n%s", diff)
208 }
209 if diff := cmp.Diff(gotDeletes, tc.WantDeletes); diff != "" {
210 t.Errorf("deletes differ:\n%s", diff)
211 }
212 })
213 }
214 }
215
216 func TestWalkLocal(t *testing.T) {
217 tests := map[string]struct {
218 Given []string
219 Expect []string
220 MapPath func(string) string
221 }{
222 "Empty": {
223 Given: []string{},
224 Expect: []string{},
225 },
226 "Normal": {
227 Given: []string{"file.txt", "normal_dir/file.txt"},
228 Expect: []string{"file.txt", "normal_dir/file.txt"},
229 },
230 "Hidden": {
231 Given: []string{"file.txt", ".hidden_dir/file.txt", "normal_dir/file.txt"},
232 Expect: []string{"file.txt", "normal_dir/file.txt"},
233 },
234 "Well Known": {
235 Given: []string{"file.txt", ".hidden_dir/file.txt", ".well-known/file.txt"},
236 Expect: []string{"file.txt", ".well-known/file.txt"},
237 },
238 "StripIndexHTML": {
239 Given: []string{"index.html", "file.txt", "dir/index.html", "dir/file.txt"},
240 Expect: []string{"index.html", "file.txt", "dir/", "dir/file.txt"},
241 MapPath: stripIndexHTML,
242 },
243 }
244
245 for desc, tc := range tests {
246 t.Run(desc, func(t *testing.T) {
247 fs := afero.NewMemMapFs()
248 for _, name := range tc.Given {
249 dir, _ := path.Split(name)
250 if dir != "" {
251 if err := fs.MkdirAll(dir, 0o755); err != nil {
252 t.Fatal(err)
253 }
254 }
255 if fd, err := fs.Create(name); err != nil {
256 t.Fatal(err)
257 } else {
258 fd.Close()
259 }
260 }
261 d := newDeployer()
262 if got, err := d.walkLocal(fs, nil, nil, nil, media.DefaultTypes, tc.MapPath); err != nil {
263 t.Fatal(err)
264 } else {
265 expect := map[string]any{}
266 for _, path := range tc.Expect {
267 if _, ok := got[path]; !ok {
268 t.Errorf("expected %q in results, but was not found", path)
269 }
270 expect[path] = nil
271 }
272 for path := range got {
273 if _, ok := expect[path]; !ok {
274 t.Errorf("got %q in results unexpectedly", path)
275 }
276 }
277 }
278 })
279 }
280 }
281
282 func TestStripIndexHTML(t *testing.T) {
283 tests := map[string]struct {
284 Input string
285 Output string
286 }{
287 "Unmapped": {Input: "normal_file.txt", Output: "normal_file.txt"},
288 "Stripped": {Input: "directory/index.html", Output: "directory/"},
289 "NoSlash": {Input: "prefix_index.html", Output: "prefix_index.html"},
290 "Root": {Input: "index.html", Output: "index.html"},
291 }
292 for desc, tc := range tests {
293 t.Run(desc, func(t *testing.T) {
294 got := stripIndexHTML(tc.Input)
295 if got != tc.Output {
296 t.Errorf("got %q, expect %q", got, tc.Output)
297 }
298 })
299 }
300 }
301
302 func TestStripIndexHTMLMatcher(t *testing.T) {
303 // StripIndexHTML should not affect matchers.
304 fs := afero.NewMemMapFs()
305 if err := fs.Mkdir("dir", 0o755); err != nil {
306 t.Fatal(err)
307 }
308 for _, name := range []string{"index.html", "dir/index.html", "file.txt"} {
309 if fd, err := fs.Create(name); err != nil {
310 t.Fatal(err)
311 } else {
312 fd.Close()
313 }
314 }
315 d := newDeployer()
316 const pattern = `\.html$`
317 matcher := &deployconfig.Matcher{Pattern: pattern, Gzip: true, Re: regexp.MustCompile(pattern)}
318 if got, err := d.walkLocal(fs, []*deployconfig.Matcher{matcher}, nil, nil, media.DefaultTypes, stripIndexHTML); err != nil {
319 t.Fatal(err)
320 } else {
321 for _, name := range []string{"index.html", "dir/"} {
322 lf := got[name]
323 if lf == nil {
324 t.Errorf("missing file %q", name)
325 } else if lf.matcher == nil {
326 t.Errorf("file %q has nil matcher, expect %q", name, pattern)
327 }
328 }
329 const name = "file.txt"
330 lf := got[name]
331 if lf == nil {
332 t.Errorf("missing file %q", name)
333 } else if lf.matcher != nil {
334 t.Errorf("file %q has matcher %q, expect nil", name, lf.matcher.Pattern)
335 }
336 }
337 }
338
339 func TestLocalFile(t *testing.T) {
340 const (
341 content = "hello world!"
342 )
343 contentBytes := []byte(content)
344 contentLen := int64(len(contentBytes))
345 contentMD5 := md5.Sum(contentBytes)
346 var buf bytes.Buffer
347 gz := gzip.NewWriter(&buf)
348 if _, err := gz.Write(contentBytes); err != nil {
349 t.Fatal(err)
350 }
351 gz.Close()
352 gzBytes := buf.Bytes()
353 gzLen := int64(len(gzBytes))
354 gzMD5 := md5.Sum(gzBytes)
355
356 tests := []struct {
357 Description string
358 Path string
359 Matcher *deployconfig.Matcher
360 MediaTypesConfig map[string]any
361 WantContent []byte
362 WantSize int64
363 WantMD5 []byte
364 WantContentType string // empty string is always OK, since content type detection is OS-specific
365 WantCacheControl string
366 WantContentEncoding string
367 }{
368 {
369 Description: "file with no suffix",
370 Path: "foo",
371 WantContent: contentBytes,
372 WantSize: contentLen,
373 WantMD5: contentMD5[:],
374 },
375 {
376 Description: "file with .txt suffix",
377 Path: "foo.txt",
378 WantContent: contentBytes,
379 WantSize: contentLen,
380 WantMD5: contentMD5[:],
381 },
382 {
383 Description: "CacheControl from matcher",
384 Path: "foo.txt",
385 Matcher: &deployconfig.Matcher{CacheControl: "max-age=630720000"},
386 WantContent: contentBytes,
387 WantSize: contentLen,
388 WantMD5: contentMD5[:],
389 WantCacheControl: "max-age=630720000",
390 },
391 {
392 Description: "ContentEncoding from matcher",
393 Path: "foo.txt",
394 Matcher: &deployconfig.Matcher{ContentEncoding: "foobar"},
395 WantContent: contentBytes,
396 WantSize: contentLen,
397 WantMD5: contentMD5[:],
398 WantContentEncoding: "foobar",
399 },
400 {
401 Description: "ContentType from matcher",
402 Path: "foo.txt",
403 Matcher: &deployconfig.Matcher{ContentType: "foo/bar"},
404 WantContent: contentBytes,
405 WantSize: contentLen,
406 WantMD5: contentMD5[:],
407 WantContentType: "foo/bar",
408 },
409 {
410 Description: "gzipped content",
411 Path: "foo.txt",
412 Matcher: &deployconfig.Matcher{Gzip: true},
413 WantContent: gzBytes,
414 WantSize: gzLen,
415 WantMD5: gzMD5[:],
416 WantContentEncoding: "gzip",
417 },
418 {
419 Description: "Custom MediaType",
420 Path: "foo.hugo",
421 MediaTypesConfig: map[string]any{
422 "hugo/custom": map[string]any{
423 "suffixes": []string{"hugo"},
424 },
425 },
426 WantContent: contentBytes,
427 WantSize: contentLen,
428 WantMD5: contentMD5[:],
429 WantContentType: "hugo/custom",
430 },
431 }
432
433 for _, tc := range tests {
434 t.Run(tc.Description, func(t *testing.T) {
435 fs := new(afero.MemMapFs)
436 if err := afero.WriteFile(fs, tc.Path, []byte(content), os.ModePerm); err != nil {
437 t.Fatal(err)
438 }
439 mediaTypes := media.DefaultTypes
440 if len(tc.MediaTypesConfig) > 0 {
441 mt, err := media.DecodeTypes(tc.MediaTypesConfig)
442 if err != nil {
443 t.Fatal(err)
444 }
445 mediaTypes = mt.Config
446 }
447 lf, err := newLocalFile(fs, tc.Path, filepath.ToSlash(tc.Path), tc.Matcher, mediaTypes)
448 if err != nil {
449 t.Fatal(err)
450 }
451 if got := lf.UploadSize; got != tc.WantSize {
452 t.Errorf("got size %d want %d", got, tc.WantSize)
453 }
454 if got := lf.MD5(); !bytes.Equal(got, tc.WantMD5) {
455 t.Errorf("got MD5 %x want %x", got, tc.WantMD5)
456 }
457 if got := lf.CacheControl(); got != tc.WantCacheControl {
458 t.Errorf("got CacheControl %q want %q", got, tc.WantCacheControl)
459 }
460 if got := lf.ContentEncoding(); got != tc.WantContentEncoding {
461 t.Errorf("got ContentEncoding %q want %q", got, tc.WantContentEncoding)
462 }
463 if tc.WantContentType != "" {
464 if got := lf.ContentType(); got != tc.WantContentType {
465 t.Errorf("got ContentType %q want %q", got, tc.WantContentType)
466 }
467 }
468 // Verify the reader last to ensure the previous operations don't
469 // interfere with it.
470 r, err := lf.Reader()
471 if err != nil {
472 t.Fatal(err)
473 }
474 gotContent, err := io.ReadAll(r)
475 if err != nil {
476 t.Fatal(err)
477 }
478 if !bytes.Equal(gotContent, tc.WantContent) {
479 t.Errorf("got content %q want %q", string(gotContent), string(tc.WantContent))
480 }
481 r.Close()
482 // Verify we can read again.
483 r, err = lf.Reader()
484 if err != nil {
485 t.Fatal(err)
486 }
487 gotContent, err = io.ReadAll(r)
488 if err != nil {
489 t.Fatal(err)
490 }
491 r.Close()
492 if !bytes.Equal(gotContent, tc.WantContent) {
493 t.Errorf("got content %q want %q", string(gotContent), string(tc.WantContent))
494 }
495 })
496 }
497 }
498
499 func TestOrdering(t *testing.T) {
500 tests := []struct {
501 Description string
502 Uploads []string
503 Ordering []*regexp.Regexp
504 Want [][]string
505 }{
506 {
507 Description: "empty",
508 Want: [][]string{nil},
509 },
510 {
511 Description: "no ordering",
512 Uploads: []string{"c", "b", "a", "d"},
513 Want: [][]string{{"a", "b", "c", "d"}},
514 },
515 {
516 Description: "one ordering",
517 Uploads: []string{"db", "c", "b", "a", "da"},
518 Ordering: []*regexp.Regexp{regexp.MustCompile("^d")},
519 Want: [][]string{{"da", "db"}, {"a", "b", "c"}},
520 },
521 {
522 Description: "two orderings",
523 Uploads: []string{"db", "c", "b", "a", "da"},
524 Ordering: []*regexp.Regexp{
525 regexp.MustCompile("^d"),
526 regexp.MustCompile("^b"),
527 },
528 Want: [][]string{{"da", "db"}, {"b"}, {"a", "c"}},
529 },
530 }
531
532 for _, tc := range tests {
533 t.Run(tc.Description, func(t *testing.T) {
534 uploads := make([]*fileToUpload, len(tc.Uploads))
535 for i, u := range tc.Uploads {
536 uploads[i] = &fileToUpload{Local: &localFile{SlashPath: u}}
537 }
538 gotUploads := applyOrdering(tc.Ordering, uploads)
539 var got [][]string
540 for _, subslice := range gotUploads {
541 var gotsubslice []string
542 for _, u := range subslice {
543 gotsubslice = append(gotsubslice, u.Local.SlashPath)
544 }
545 got = append(got, gotsubslice)
546 }
547 if diff := cmp.Diff(got, tc.Want); diff != "" {
548 t.Error(diff)
549 }
550 })
551 }
552 }
553
554 type fileData struct {
555 Name string // name of the file
556 Contents string // contents of the file
557 }
558
559 // initLocalFs initializes fs with some test files.
560 func initLocalFs(ctx context.Context, fs afero.Fs) ([]*fileData, error) {
561 // The initial local filesystem.
562 local := []*fileData{
563 {"aaa", "aaa"},
564 {"bbb", "bbb"},
565 {"subdir/aaa", "subdir-aaa"},
566 {"subdir/nested/aaa", "subdir-nested-aaa"},
567 {"subdir2/bbb", "subdir2-bbb"},
568 }
569 if err := writeFiles(fs, local); err != nil {
570 return nil, err
571 }
572 return local, nil
573 }
574
575 // fsTest represents an (afero.FS, Go CDK blob.Bucket) against which end-to-end
576 // tests can be run.
577 type fsTest struct {
578 name string
579 fs afero.Fs
580 bucket *blob.Bucket
581 }
582
583 // initFsTests initializes a pair of tests for end-to-end test:
584 // 1. An in-memory afero.Fs paired with an in-memory Go CDK bucket.
585 // 2. A filesystem-based afero.Fs paired with an filesystem-based Go CDK bucket.
586 // It returns the pair of tests and a cleanup function.
587 func initFsTests(t *testing.T) []*fsTest {
588 t.Helper()
589
590 tmpfsdir := t.TempDir()
591 tmpbucketdir := t.TempDir()
592
593 memfs := afero.NewMemMapFs()
594 membucket := memblob.OpenBucket(nil)
595 t.Cleanup(func() { membucket.Close() })
596
597 filefs := hugofs.NewBasePathFs(afero.NewOsFs(), tmpfsdir)
598 filebucket, err := fileblob.OpenBucket(tmpbucketdir, nil)
599 if err != nil {
600 t.Fatal(err)
601 }
602 t.Cleanup(func() { filebucket.Close() })
603
604 tests := []*fsTest{
605 {"mem", memfs, membucket},
606 {"file", filefs, filebucket},
607 }
608 return tests
609 }
610
611 // TestEndToEndSync verifies that basic adds, updates, and deletes are working
612 // correctly.
613 func TestEndToEndSync(t *testing.T) {
614 ctx := context.Background()
615 tests := initFsTests(t)
616 for _, test := range tests {
617 t.Run(test.name, func(t *testing.T) {
618 local, err := initLocalFs(ctx, test.fs)
619 if err != nil {
620 t.Fatal(err)
621 }
622 deployer := &Deployer{
623 localFs: test.fs,
624 bucket: test.bucket,
625 mediaTypes: media.DefaultTypes,
626 cfg: deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1},
627 }
628
629 // Initial deployment should sync remote with local.
630 if err := deployer.Deploy(ctx); err != nil {
631 t.Errorf("initial deploy: failed: %v", err)
632 }
633 wantSummary := deploySummary{NumLocal: 5, NumRemote: 0, NumUploads: 5, NumDeletes: 0}
634 if !cmp.Equal(deployer.summary, wantSummary) {
635 t.Errorf("initial deploy: got %v, want %v", deployer.summary, wantSummary)
636 }
637 if diff, err := verifyRemote(ctx, deployer.bucket, local); err != nil {
638 t.Errorf("initial deploy: failed to verify remote: %v", err)
639 } else if diff != "" {
640 t.Errorf("initial deploy: remote snapshot doesn't match expected:\n%v", diff)
641 }
642
643 // A repeat deployment shouldn't change anything.
644 if err := deployer.Deploy(ctx); err != nil {
645 t.Errorf("no-op deploy: %v", err)
646 }
647 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 0, NumDeletes: 0}
648 if !cmp.Equal(deployer.summary, wantSummary) {
649 t.Errorf("no-op deploy: got %v, want %v", deployer.summary, wantSummary)
650 }
651
652 // Make some changes to the local filesystem:
653 // 1. Modify file [0].
654 // 2. Delete file [1].
655 // 3. Add a new file (sorted last).
656 updatefd := local[0]
657 updatefd.Contents = "new contents"
658 deletefd := local[1]
659 local = append(local[:1], local[2:]...) // removing deleted [1]
660 newfd := &fileData{"zzz", "zzz"}
661 local = append(local, newfd)
662 if err := writeFiles(test.fs, []*fileData{updatefd, newfd}); err != nil {
663 t.Fatal(err)
664 }
665 if err := test.fs.Remove(deletefd.Name); err != nil {
666 t.Fatal(err)
667 }
668
669 // A deployment should apply those 3 changes.
670 if err := deployer.Deploy(ctx); err != nil {
671 t.Errorf("deploy after changes: failed: %v", err)
672 }
673 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 2, NumDeletes: 1}
674 if !cmp.Equal(deployer.summary, wantSummary) {
675 t.Errorf("deploy after changes: got %v, want %v", deployer.summary, wantSummary)
676 }
677 if diff, err := verifyRemote(ctx, deployer.bucket, local); err != nil {
678 t.Errorf("deploy after changes: failed to verify remote: %v", err)
679 } else if diff != "" {
680 t.Errorf("deploy after changes: remote snapshot doesn't match expected:\n%v", diff)
681 }
682
683 // Again, a repeat deployment shouldn't change anything.
684 if err := deployer.Deploy(ctx); err != nil {
685 t.Errorf("no-op deploy: %v", err)
686 }
687 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 0, NumDeletes: 0}
688 if !cmp.Equal(deployer.summary, wantSummary) {
689 t.Errorf("no-op deploy: got %v, want %v", deployer.summary, wantSummary)
690 }
691 })
692 }
693 }
694
695 // TestMaxDeletes verifies that the "maxDeletes" flag is working correctly.
696 func TestMaxDeletes(t *testing.T) {
697 ctx := context.Background()
698 tests := initFsTests(t)
699 for _, test := range tests {
700 t.Run(test.name, func(t *testing.T) {
701 local, err := initLocalFs(ctx, test.fs)
702 if err != nil {
703 t.Fatal(err)
704 }
705 deployer := &Deployer{
706 localFs: test.fs,
707 bucket: test.bucket,
708 mediaTypes: media.DefaultTypes,
709 cfg: deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1},
710 }
711
712 // Sync remote with local.
713 if err := deployer.Deploy(ctx); err != nil {
714 t.Errorf("initial deploy: failed: %v", err)
715 }
716 wantSummary := deploySummary{NumLocal: 5, NumRemote: 0, NumUploads: 5, NumDeletes: 0}
717 if !cmp.Equal(deployer.summary, wantSummary) {
718 t.Errorf("initial deploy: got %v, want %v", deployer.summary, wantSummary)
719 }
720
721 // Delete two files, [1] and [2].
722 if err := test.fs.Remove(local[1].Name); err != nil {
723 t.Fatal(err)
724 }
725 if err := test.fs.Remove(local[2].Name); err != nil {
726 t.Fatal(err)
727 }
728
729 // A deployment with maxDeletes=0 shouldn't change anything.
730 deployer.cfg.MaxDeletes = 0
731 if err := deployer.Deploy(ctx); err != nil {
732 t.Errorf("deploy failed: %v", err)
733 }
734 wantSummary = deploySummary{NumLocal: 3, NumRemote: 5, NumUploads: 0, NumDeletes: 0}
735 if !cmp.Equal(deployer.summary, wantSummary) {
736 t.Errorf("deploy: got %v, want %v", deployer.summary, wantSummary)
737 }
738
739 // A deployment with maxDeletes=1 shouldn't change anything either.
740 deployer.cfg.MaxDeletes = 1
741 if err := deployer.Deploy(ctx); err != nil {
742 t.Errorf("deploy failed: %v", err)
743 }
744 wantSummary = deploySummary{NumLocal: 3, NumRemote: 5, NumUploads: 0, NumDeletes: 0}
745 if !cmp.Equal(deployer.summary, wantSummary) {
746 t.Errorf("deploy: got %v, want %v", deployer.summary, wantSummary)
747 }
748
749 // A deployment with maxDeletes=2 should make the changes.
750 deployer.cfg.MaxDeletes = 2
751 if err := deployer.Deploy(ctx); err != nil {
752 t.Errorf("deploy failed: %v", err)
753 }
754 wantSummary = deploySummary{NumLocal: 3, NumRemote: 5, NumUploads: 0, NumDeletes: 2}
755 if !cmp.Equal(deployer.summary, wantSummary) {
756 t.Errorf("deploy: got %v, want %v", deployer.summary, wantSummary)
757 }
758
759 // Delete two more files, [0] and [3].
760 if err := test.fs.Remove(local[0].Name); err != nil {
761 t.Fatal(err)
762 }
763 if err := test.fs.Remove(local[3].Name); err != nil {
764 t.Fatal(err)
765 }
766
767 // A deployment with maxDeletes=-1 should make the changes.
768 deployer.cfg.MaxDeletes = -1
769 if err := deployer.Deploy(ctx); err != nil {
770 t.Errorf("deploy failed: %v", err)
771 }
772 wantSummary = deploySummary{NumLocal: 1, NumRemote: 3, NumUploads: 0, NumDeletes: 2}
773 if !cmp.Equal(deployer.summary, wantSummary) {
774 t.Errorf("deploy: got %v, want %v", deployer.summary, wantSummary)
775 }
776 })
777 }
778 }
779
780 // TestIncludeExclude verifies that the include/exclude options for targets work.
781 func TestIncludeExclude(t *testing.T) {
782 ctx := context.Background()
783 tests := []struct {
784 Include string
785 Exclude string
786 Want deploySummary
787 }{
788 {
789 Want: deploySummary{NumLocal: 5, NumUploads: 5},
790 },
791 {
792 Include: "**aaa",
793 Want: deploySummary{NumLocal: 3, NumUploads: 3},
794 },
795 {
796 Include: "**bbb",
797 Want: deploySummary{NumLocal: 2, NumUploads: 2},
798 },
799 {
800 Include: "aaa",
801 Want: deploySummary{NumLocal: 1, NumUploads: 1},
802 },
803 {
804 Exclude: "**aaa",
805 Want: deploySummary{NumLocal: 2, NumUploads: 2},
806 },
807 {
808 Exclude: "**bbb",
809 Want: deploySummary{NumLocal: 3, NumUploads: 3},
810 },
811 {
812 Exclude: "aaa",
813 Want: deploySummary{NumLocal: 4, NumUploads: 4},
814 },
815 {
816 Include: "**aaa",
817 Exclude: "**nested**",
818 Want: deploySummary{NumLocal: 2, NumUploads: 2},
819 },
820 }
821 for _, test := range tests {
822 t.Run(fmt.Sprintf("include %q exclude %q", test.Include, test.Exclude), func(t *testing.T) {
823 fsTests := initFsTests(t)
824 fsTest := fsTests[1] // just do file-based test
825
826 _, err := initLocalFs(ctx, fsTest.fs)
827 if err != nil {
828 t.Fatal(err)
829 }
830 tgt := &deployconfig.Target{
831 Include: test.Include,
832 Exclude: test.Exclude,
833 }
834 if err := tgt.ParseIncludeExclude(); err != nil {
835 t.Error(err)
836 }
837 deployer := &Deployer{
838 localFs: fsTest.fs,
839 cfg: deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1}, bucket: fsTest.bucket,
840 target: tgt,
841 mediaTypes: media.DefaultTypes,
842 }
843
844 // Sync remote with local.
845 if err := deployer.Deploy(ctx); err != nil {
846 t.Errorf("deploy: failed: %v", err)
847 }
848 if !cmp.Equal(deployer.summary, test.Want) {
849 t.Errorf("deploy: got %v, want %v", deployer.summary, test.Want)
850 }
851 })
852 }
853 }
854
855 // TestIncludeExcludeRemoteDelete verifies deleted local files that don't match include/exclude patterns
856 // are not deleted on the remote.
857 func TestIncludeExcludeRemoteDelete(t *testing.T) {
858 ctx := context.Background()
859
860 tests := []struct {
861 Include string
862 Exclude string
863 Want deploySummary
864 }{
865 {
866 Want: deploySummary{NumLocal: 3, NumRemote: 5, NumUploads: 0, NumDeletes: 2},
867 },
868 {
869 Include: "**aaa",
870 Want: deploySummary{NumLocal: 2, NumRemote: 3, NumUploads: 0, NumDeletes: 1},
871 },
872 {
873 Include: "subdir/**",
874 Want: deploySummary{NumLocal: 1, NumRemote: 2, NumUploads: 0, NumDeletes: 1},
875 },
876 {
877 Exclude: "**bbb",
878 Want: deploySummary{NumLocal: 2, NumRemote: 3, NumUploads: 0, NumDeletes: 1},
879 },
880 {
881 Exclude: "bbb",
882 Want: deploySummary{NumLocal: 3, NumRemote: 4, NumUploads: 0, NumDeletes: 1},
883 },
884 }
885 for _, test := range tests {
886 t.Run(fmt.Sprintf("include %q exclude %q", test.Include, test.Exclude), func(t *testing.T) {
887 fsTests := initFsTests(t)
888 fsTest := fsTests[1] // just do file-based test
889
890 local, err := initLocalFs(ctx, fsTest.fs)
891 if err != nil {
892 t.Fatal(err)
893 }
894 deployer := &Deployer{
895 localFs: fsTest.fs,
896 cfg: deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1}, bucket: fsTest.bucket,
897 mediaTypes: media.DefaultTypes,
898 }
899
900 // Initial sync to get the files on the remote
901 if err := deployer.Deploy(ctx); err != nil {
902 t.Errorf("deploy: failed: %v", err)
903 }
904
905 // Delete two files, [1] and [2].
906 if err := fsTest.fs.Remove(local[1].Name); err != nil {
907 t.Fatal(err)
908 }
909 if err := fsTest.fs.Remove(local[2].Name); err != nil {
910 t.Fatal(err)
911 }
912
913 // Second sync
914 tgt := &deployconfig.Target{
915 Include: test.Include,
916 Exclude: test.Exclude,
917 }
918 if err := tgt.ParseIncludeExclude(); err != nil {
919 t.Error(err)
920 }
921 deployer.target = tgt
922 if err := deployer.Deploy(ctx); err != nil {
923 t.Errorf("deploy: failed: %v", err)
924 }
925
926 if !cmp.Equal(deployer.summary, test.Want) {
927 t.Errorf("deploy: got %v, want %v", deployer.summary, test.Want)
928 }
929 })
930 }
931 }
932
933 // TestCompression verifies that gzip compression works correctly.
934 // In particular, MD5 hashes must be of the compressed content.
935 func TestCompression(t *testing.T) {
936 ctx := context.Background()
937
938 tests := initFsTests(t)
939 for _, test := range tests {
940 t.Run(test.name, func(t *testing.T) {
941 local, err := initLocalFs(ctx, test.fs)
942 if err != nil {
943 t.Fatal(err)
944 }
945 deployer := &Deployer{
946 localFs: test.fs,
947 bucket: test.bucket,
948 cfg: deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: ".*", Gzip: true, Re: regexp.MustCompile(".*")}}},
949 mediaTypes: media.DefaultTypes,
950 }
951
952 // Initial deployment should sync remote with local.
953 if err := deployer.Deploy(ctx); err != nil {
954 t.Errorf("initial deploy: failed: %v", err)
955 }
956 wantSummary := deploySummary{NumLocal: 5, NumRemote: 0, NumUploads: 5, NumDeletes: 0}
957 if !cmp.Equal(deployer.summary, wantSummary) {
958 t.Errorf("initial deploy: got %v, want %v", deployer.summary, wantSummary)
959 }
960
961 // A repeat deployment shouldn't change anything.
962 if err := deployer.Deploy(ctx); err != nil {
963 t.Errorf("no-op deploy: %v", err)
964 }
965 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 0, NumDeletes: 0}
966 if !cmp.Equal(deployer.summary, wantSummary) {
967 t.Errorf("no-op deploy: got %v, want %v", deployer.summary, wantSummary)
968 }
969
970 // Make an update to the local filesystem, on [1].
971 updatefd := local[1]
972 updatefd.Contents = "new contents"
973 if err := writeFiles(test.fs, []*fileData{updatefd}); err != nil {
974 t.Fatal(err)
975 }
976
977 // A deployment should apply the changes.
978 if err := deployer.Deploy(ctx); err != nil {
979 t.Errorf("deploy after changes: failed: %v", err)
980 }
981 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 1, NumDeletes: 0}
982 if !cmp.Equal(deployer.summary, wantSummary) {
983 t.Errorf("deploy after changes: got %v, want %v", deployer.summary, wantSummary)
984 }
985 })
986 }
987 }
988
989 // TestMatching verifies that matchers match correctly, and that the Force
990 // attribute for matcher works.
991 func TestMatching(t *testing.T) {
992 ctx := context.Background()
993 tests := initFsTests(t)
994 for _, test := range tests {
995 t.Run(test.name, func(t *testing.T) {
996 _, err := initLocalFs(ctx, test.fs)
997 if err != nil {
998 t.Fatal(err)
999 }
1000 deployer := &Deployer{
1001 localFs: test.fs,
1002 bucket: test.bucket,
1003 cfg: deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: "^subdir/aaa$", Force: true, Re: regexp.MustCompile("^subdir/aaa$")}}},
1004 mediaTypes: media.DefaultTypes,
1005 }
1006
1007 // Initial deployment to sync remote with local.
1008 if err := deployer.Deploy(ctx); err != nil {
1009 t.Errorf("initial deploy: failed: %v", err)
1010 }
1011 wantSummary := deploySummary{NumLocal: 5, NumRemote: 0, NumUploads: 5, NumDeletes: 0}
1012 if !cmp.Equal(deployer.summary, wantSummary) {
1013 t.Errorf("initial deploy: got %v, want %v", deployer.summary, wantSummary)
1014 }
1015
1016 // A repeat deployment should upload a single file, the one that matched the Force matcher.
1017 // Note that matching happens based on the ToSlash form, so this matches
1018 // even on Windows.
1019 if err := deployer.Deploy(ctx); err != nil {
1020 t.Errorf("no-op deploy with single force matcher: %v", err)
1021 }
1022 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 1, NumDeletes: 0}
1023 if !cmp.Equal(deployer.summary, wantSummary) {
1024 t.Errorf("no-op deploy with single force matcher: got %v, want %v", deployer.summary, wantSummary)
1025 }
1026
1027 // Repeat with a matcher that should now match 3 files.
1028 deployer.cfg.Matchers = []*deployconfig.Matcher{{Pattern: "aaa", Force: true, Re: regexp.MustCompile("aaa")}}
1029 if err := deployer.Deploy(ctx); err != nil {
1030 t.Errorf("no-op deploy with triple force matcher: %v", err)
1031 }
1032 wantSummary = deploySummary{NumLocal: 5, NumRemote: 5, NumUploads: 3, NumDeletes: 0}
1033 if !cmp.Equal(deployer.summary, wantSummary) {
1034 t.Errorf("no-op deploy with triple force matcher: got %v, want %v", deployer.summary, wantSummary)
1035 }
1036 })
1037 }
1038 }
1039
1040 // writeFiles writes the files in fds to fd.
1041 func writeFiles(fs afero.Fs, fds []*fileData) error {
1042 for _, fd := range fds {
1043 dir := path.Dir(fd.Name)
1044 if dir != "." {
1045 err := fs.MkdirAll(dir, os.ModePerm)
1046 if err != nil {
1047 return err
1048 }
1049 }
1050 f, err := fs.Create(fd.Name)
1051 if err != nil {
1052 return err
1053 }
1054 defer f.Close()
1055 _, err = f.WriteString(fd.Contents)
1056 if err != nil {
1057 return err
1058 }
1059 }
1060 return nil
1061 }
1062
1063 // verifyRemote that the current contents of bucket matches local.
1064 // It returns an empty string if the contents matched, and a non-empty string
1065 // capturing the diff if they didn't.
1066 func verifyRemote(ctx context.Context, bucket *blob.Bucket, local []*fileData) (string, error) {
1067 var cur []*fileData
1068 iter := bucket.List(nil)
1069 for {
1070 obj, err := iter.Next(ctx)
1071 if err == io.EOF {
1072 break
1073 }
1074 if err != nil {
1075 return "", err
1076 }
1077 contents, err := bucket.ReadAll(ctx, obj.Key)
1078 if err != nil {
1079 return "", err
1080 }
1081 cur = append(cur, &fileData{obj.Key, string(contents)})
1082 }
1083 if cmp.Equal(cur, local) {
1084 return "", nil
1085 }
1086 diff := "got: \n"
1087 for _, f := range cur {
1088 diff += fmt.Sprintf(" %s: %s\n", f.Name, f.Contents)
1089 }
1090 diff += "want: \n"
1091 for _, f := range local {
1092 diff += fmt.Sprintf(" %s: %s\n", f.Name, f.Contents)
1093 }
1094 return diff, nil
1095 }
1096
1097 func newDeployer() *Deployer {
1098 return &Deployer{
1099 logger: loggers.NewDefault(),
1100 cfg: deployconfig.DeployConfig{Workers: 2},
1101 }
1102 }