fs.go - afero - [fork] go afero port for 9front
(HTM) git clone https://git.drkhsh.at/afero.git
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
(DIR) LICENSE
---
fs.go (9728B)
---
1 // Copyright © 2021 Vasily Ovchinnikov <vasily@remerge.io>.
2 //
3 // The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp
4 // licensed under Apache License 2.0.
5 //
6 // Licensed under the Apache License, Version 2.0 (the "License");
7 // you may not use this file except in compliance with the License.
8 // You may obtain a copy of the License at
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16
17 package gcsfs
18
19 import (
20 "context"
21 "errors"
22 "os"
23 "path/filepath"
24 "strings"
25 "syscall"
26 "time"
27
28 "github.com/spf13/afero/gcsfs/internal/stiface"
29 )
30
31 const (
32 defaultFileMode = 0o755
33 gsPrefix = "gs://"
34 )
35
36 // Fs is a Fs implementation that uses functions provided by google cloud storage
37 type Fs struct {
38 ctx context.Context
39 client stiface.Client
40 separator string
41
42 buckets map[string]stiface.BucketHandle
43 rawGcsObjects map[string]*GcsFile
44
45 autoRemoveEmptyFolders bool // trigger for creating "virtual folders" (not required by GCSs)
46 }
47
48 func NewGcsFs(ctx context.Context, client stiface.Client) *Fs {
49 return NewGcsFsWithSeparator(ctx, client, "/")
50 }
51
52 func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *Fs {
53 return &Fs{
54 ctx: ctx,
55 client: client,
56 separator: folderSep,
57 rawGcsObjects: make(map[string]*GcsFile),
58
59 autoRemoveEmptyFolders: true,
60 }
61 }
62
63 // normSeparators will normalize all "\\" and "/" to the provided separator
64 func (fs *Fs) normSeparators(s string) string {
65 return strings.Replace(strings.Replace(s, "\\", fs.separator, -1), "/", fs.separator, -1)
66 }
67
68 func (fs *Fs) ensureTrailingSeparator(s string) string {
69 if len(s) > 0 && !strings.HasSuffix(s, fs.separator) {
70 return s + fs.separator
71 }
72 return s
73 }
74
75 func (fs *Fs) ensureNoLeadingSeparator(s string) string {
76 if len(s) > 0 && strings.HasPrefix(s, fs.separator) {
77 s = s[len(fs.separator):]
78 }
79
80 return s
81 }
82
83 func ensureNoPrefix(s string) string {
84 if len(s) > 0 && strings.HasPrefix(s, gsPrefix) {
85 return s[len(gsPrefix):]
86 }
87 return s
88 }
89
90 func validateName(s string) error {
91 if len(s) == 0 {
92 return ErrNoBucketInName
93 }
94 return nil
95 }
96
97 // Splits provided name into bucket name and path
98 func (fs *Fs) splitName(name string) (bucketName string, path string) {
99 splitName := strings.Split(name, fs.separator)
100
101 return splitName[0], strings.Join(splitName[1:], fs.separator)
102 }
103
104 func (fs *Fs) getBucket(name string) (stiface.BucketHandle, error) {
105 bucket := fs.buckets[name]
106 if bucket == nil {
107 bucket = fs.client.Bucket(name)
108 _, err := bucket.Attrs(fs.ctx)
109 if err != nil {
110 return nil, err
111 }
112 }
113 return bucket, nil
114 }
115
116 func (fs *Fs) getObj(name string) (stiface.ObjectHandle, error) {
117 bucketName, path := fs.splitName(name)
118
119 bucket, err := fs.getBucket(bucketName)
120 if err != nil {
121 return nil, err
122 }
123
124 return bucket.Object(path), nil
125 }
126
127 func (fs *Fs) Name() string { return "GcsFs" }
128
129 func (fs *Fs) Create(name string) (*GcsFile, error) {
130 name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
131 if err := validateName(name); err != nil {
132 return nil, err
133 }
134
135 if !fs.autoRemoveEmptyFolders {
136 baseDir := filepath.Base(name)
137 if stat, err := fs.Stat(baseDir); err != nil || !stat.IsDir() {
138 err = fs.MkdirAll(baseDir, 0)
139 if err != nil {
140 return nil, err
141 }
142 }
143 }
144
145 obj, err := fs.getObj(name)
146 if err != nil {
147 return nil, err
148 }
149 w := obj.NewWriter(fs.ctx)
150 err = w.Close()
151 if err != nil {
152 return nil, err
153 }
154 file := NewGcsFile(fs.ctx, fs, obj, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0, name)
155
156 fs.rawGcsObjects[name] = file
157 return file, nil
158 }
159
160 func (fs *Fs) Mkdir(name string, _ os.FileMode) error {
161 name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name))))
162 if err := validateName(name); err != nil {
163 return err
164 }
165 // folder creation logic has to additionally check for folder name presence
166 bucketName, path := fs.splitName(name)
167 if bucketName == "" {
168 return ErrNoBucketInName
169 }
170 if path == "" {
171 // the API would throw "googleapi: Error 400: No object name, required", but this one is more consistent
172 return ErrEmptyObjectName
173 }
174
175 obj, err := fs.getObj(name)
176 if err != nil {
177 return err
178 }
179 w := obj.NewWriter(fs.ctx)
180 return w.Close()
181 }
182
183 func (fs *Fs) MkdirAll(path string, perm os.FileMode) error {
184 path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path))))
185 if err := validateName(path); err != nil {
186 return err
187 }
188 // folder creation logic has to additionally check for folder name presence
189 bucketName, splitPath := fs.splitName(path)
190 if bucketName == "" {
191 return ErrNoBucketInName
192 }
193 if splitPath == "" {
194 // the API would throw "googleapi: Error 400: No object name, required", but this one is more consistent
195 return ErrEmptyObjectName
196 }
197
198 root := ""
199 folders := strings.Split(path, fs.separator)
200 for i, f := range folders {
201 if f == "" && i != 0 {
202 continue // it's the last item - it should be empty
203 }
204 // Don't force a delimiter prefix
205 if root != "" {
206 root = root + fs.separator + f
207 } else {
208 // we have to have at least bucket name + folder name to create successfully
209 root = f
210 continue
211 }
212
213 if err := fs.Mkdir(root, perm); err != nil {
214 return err
215 }
216 }
217 return nil
218 }
219
220 func (fs *Fs) Open(name string) (*GcsFile, error) {
221 return fs.OpenFile(name, os.O_RDONLY, 0)
222 }
223
224 func (fs *Fs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) {
225 var file *GcsFile
226 var err error
227
228 name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
229 if err = validateName(name); err != nil {
230 return nil, err
231 }
232
233 f, found := fs.rawGcsObjects[name]
234 if found {
235 file = NewGcsFileFromOldFH(flag, fileMode, f.resource)
236 } else {
237 var obj stiface.ObjectHandle
238 obj, err = fs.getObj(name)
239 if err != nil {
240 return nil, err
241 }
242 file = NewGcsFile(fs.ctx, fs, obj, flag, fileMode, name)
243 }
244
245 if flag == os.O_RDONLY {
246 _, err = file.Stat()
247 if err != nil {
248 return nil, err
249 }
250 }
251
252 if flag&os.O_TRUNC != 0 {
253 err = file.resource.obj.Delete(fs.ctx)
254 if err != nil {
255 return nil, err
256 }
257 return fs.Create(name)
258 }
259
260 if flag&os.O_APPEND != 0 {
261 _, err = file.Seek(0, 2)
262 if err != nil {
263 return nil, err
264 }
265 }
266
267 if flag&os.O_CREATE != 0 {
268 _, err = file.Stat()
269 if err == nil { // the file actually exists
270 return nil, syscall.EPERM
271 }
272
273 _, err = file.WriteString("")
274 if err != nil {
275 return nil, err
276 }
277 }
278 return file, nil
279 }
280
281 func (fs *Fs) Remove(name string) error {
282 name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
283 if err := validateName(name); err != nil {
284 return err
285 }
286
287 obj, err := fs.getObj(name)
288 if err != nil {
289 return err
290 }
291 info, err := fs.Stat(name)
292 if err != nil {
293 return err
294 }
295 delete(fs.rawGcsObjects, name)
296
297 if info.IsDir() {
298 // it's a folder, we ha to check its contents - it cannot be removed, if not empty
299 var dir *GcsFile
300 dir, err = fs.Open(name)
301 if err != nil {
302 return err
303 }
304 var infos []os.FileInfo
305 infos, err = dir.Readdir(0)
306 if err != nil {
307 return err
308 }
309 if len(infos) > 0 {
310 return syscall.ENOTEMPTY
311 }
312
313 // it's an empty folder, we can continue
314 name = fs.ensureTrailingSeparator(name)
315 obj, err = fs.getObj(name)
316 if err != nil {
317 return err
318 }
319
320 return obj.Delete(fs.ctx)
321 }
322 return obj.Delete(fs.ctx)
323 }
324
325 func (fs *Fs) RemoveAll(path string) error {
326 path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path)))
327 if err := validateName(path); err != nil {
328 return err
329 }
330
331 pathInfo, err := fs.Stat(path)
332 if errors.Is(err, ErrFileNotFound) {
333 // return early if file doesn't exist
334 return nil
335 }
336 if err != nil {
337 return err
338 }
339
340 if !pathInfo.IsDir() {
341 return fs.Remove(path)
342 }
343
344 var dir *GcsFile
345 dir, err = fs.Open(path)
346 if err != nil {
347 return err
348 }
349
350 var infos []os.FileInfo
351 infos, err = dir.Readdir(0)
352 if err != nil {
353 return err
354 }
355 for _, info := range infos {
356 nameToRemove := fs.normSeparators(info.Name())
357 err = fs.RemoveAll(path + fs.separator + nameToRemove)
358 if err != nil {
359 return err
360 }
361 }
362
363 return fs.Remove(path)
364 }
365
366 func (fs *Fs) Rename(oldName, newName string) error {
367 oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName)))
368 if err := validateName(oldName); err != nil {
369 return err
370 }
371
372 newName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(newName)))
373 if err := validateName(newName); err != nil {
374 return err
375 }
376
377 src, err := fs.getObj(oldName)
378 if err != nil {
379 return err
380 }
381 dst, err := fs.getObj(newName)
382 if err != nil {
383 return err
384 }
385
386 if _, err = dst.CopierFrom(src).Run(fs.ctx); err != nil {
387 return err
388 }
389 delete(fs.rawGcsObjects, oldName)
390 return src.Delete(fs.ctx)
391 }
392
393 func (fs *Fs) Stat(name string) (os.FileInfo, error) {
394 name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
395 if err := validateName(name); err != nil {
396 return nil, err
397 }
398
399 return newFileInfo(name, fs, defaultFileMode)
400 }
401
402 func (fs *Fs) Chmod(_ string, _ os.FileMode) error {
403 return errors.New("method Chmod is not implemented in GCS")
404 }
405
406 func (fs *Fs) Chtimes(_ string, _, _ time.Time) error {
407 return errors.New("method Chtimes is not implemented. Create, Delete, Updated times are read only fields in GCS and set implicitly")
408 }
409
410 func (fs *Fs) Chown(_ string, _, _ int) error {
411 return errors.New("method Chown is not implemented for GCS")
412 }