From 61b665af132a79f9071329be2018ae983628dfa1 Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Sat, 31 Mar 2018 17:23:40 -0700 Subject: [PATCH] use cachingfs --- Gopkg.lock | 5 +- filesystem.go | 2 + .../github.com/hanwen/go-fuse/unionfs/autounion.go | 452 +++++++++ .../github.com/hanwen/go-fuse/unionfs/cachingfs.go | 151 +++ vendor/github.com/hanwen/go-fuse/unionfs/create.go | 36 + .../github.com/hanwen/go-fuse/unionfs/dircache.go | 125 +++ .../hanwen/go-fuse/unionfs/timedcache.go | 111 +++ .../github.com/hanwen/go-fuse/unionfs/unionfs.go | 1032 ++++++++++++++++++++ .../go-fuse/unionfs/unionfs_xattr_test_darwin.go | 37 + .../go-fuse/unionfs/unionfs_xattr_test_linux.go | 14 + 10 files changed, 1963 insertions(+), 2 deletions(-) create mode 100644 vendor/github.com/hanwen/go-fuse/unionfs/autounion.go create mode 100644 vendor/github.com/hanwen/go-fuse/unionfs/cachingfs.go create mode 100644 vendor/github.com/hanwen/go-fuse/unionfs/create.go create mode 100644 vendor/github.com/hanwen/go-fuse/unionfs/dircache.go create mode 100644 vendor/github.com/hanwen/go-fuse/unionfs/timedcache.go create mode 100644 vendor/github.com/hanwen/go-fuse/unionfs/unionfs.go create mode 100644 vendor/github.com/hanwen/go-fuse/unionfs/unionfs_xattr_test_darwin.go create mode 100644 vendor/github.com/hanwen/go-fuse/unionfs/unionfs_xattr_test_linux.go diff --git a/Gopkg.lock b/Gopkg.lock index 54682d3..65e172e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -14,7 +14,8 @@ "fuse", "fuse/nodefs", "fuse/pathfs", - "splice" + "splice", + "unionfs" ] revision = "a9ddcb8a4b609500fc59c89ccc9ee05f00a5fefd" @@ -60,6 +61,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "9bee1a4379bcd4b9176976a3cedac4a5c846c42ebdc54c623d3fb551c6921aa9" + inputs-digest = "3446597fa9ac971d950ce0953fbc4e46796f213d08a7781d54fe17858e2af33b" solver-name = "gps-cdcl" solver-version = 1 diff --git a/filesystem.go b/filesystem.go index 521750a..b8270eb 100644 --- a/filesystem.go +++ b/filesystem.go @@ -6,6 +6,7 @@ import ( "github.com/hanwen/go-fuse/fuse" "github.com/hanwen/go-fuse/fuse/nodefs" "github.com/hanwen/go-fuse/fuse/pathfs" + "github.com/hanwen/go-fuse/unionfs" "github.com/sirupsen/logrus" ) @@ -31,6 +32,7 @@ func NewWsFs(opts WsFsOpts, req chan RequestCallback, closer chan bool) *pathfs. if opts.ReadOnly { fs = pathfs.NewReadonlyFileSystem(fs) } + fs = unionfs.NewCachingFileSystem(fs, 1*time.Second) return pathfs.NewPathNodeFs(fs, nil) } diff --git a/vendor/github.com/hanwen/go-fuse/unionfs/autounion.go b/vendor/github.com/hanwen/go-fuse/unionfs/autounion.go new file mode 100644 index 0000000..db1214d --- /dev/null +++ b/vendor/github.com/hanwen/go-fuse/unionfs/autounion.go @@ -0,0 +1,452 @@ +// Copyright 2016 the Go-FUSE Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unionfs + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/hanwen/go-fuse/fuse" + "github.com/hanwen/go-fuse/fuse/nodefs" + "github.com/hanwen/go-fuse/fuse/pathfs" +) + +type knownFs struct { + unionFS pathfs.FileSystem + nodeFS *pathfs.PathNodeFs +} + +// Creates unions for all files under a given directory, +// walking the tree and looking for directories D which have a +// D/READONLY symlink. +// +// A union for A/B/C will placed under directory A-B-C. +type autoUnionFs struct { + pathfs.FileSystem + debug bool + + lock sync.RWMutex + zombies map[string]bool + knownFileSystems map[string]knownFs + nameRootMap map[string]string + root string + + nodeFs *pathfs.PathNodeFs + options *AutoUnionFsOptions +} + +type AutoUnionFsOptions struct { + UnionFsOptions + + nodefs.Options + pathfs.PathNodeFsOptions + + // If set, run updateKnownFses() after mounting. + UpdateOnMount bool + + // If set hides the _READONLY file. + HideReadonly bool + + // Expose this version in /status/gounionfs_version + Version string +} + +const ( + _READONLY = "READONLY" + _STATUS = "status" + _CONFIG = "config" + _DEBUG = "debug" + _ROOT = "root" + _VERSION = "gounionfs_version" + _SCAN_CONFIG = ".scan_config" +) + +func NewAutoUnionFs(directory string, options AutoUnionFsOptions) pathfs.FileSystem { + if options.HideReadonly { + options.HiddenFiles = append(options.HiddenFiles, _READONLY) + } + a := &autoUnionFs{ + knownFileSystems: make(map[string]knownFs), + nameRootMap: make(map[string]string), + zombies: make(map[string]bool), + options: &options, + FileSystem: pathfs.NewDefaultFileSystem(), + } + + directory, err := filepath.Abs(directory) + if err != nil { + panic("filepath.Abs returned err") + } + a.root = directory + return a +} + +func (fs *autoUnionFs) String() string { + return fmt.Sprintf("autoUnionFs(%s)", fs.root) +} + +func (fs *autoUnionFs) OnMount(nodeFs *pathfs.PathNodeFs) { + fs.nodeFs = nodeFs + if fs.options.UpdateOnMount { + time.AfterFunc(100*time.Millisecond, func() { fs.updateKnownFses() }) + } +} + +func (fs *autoUnionFs) addAutomaticFs(roots []string) { + relative := strings.TrimLeft(strings.Replace(roots[0], fs.root, "", -1), "/") + name := strings.Replace(relative, "/", "-", -1) + + if fs.getUnionFs(name) == nil { + fs.addFs(name, roots) + } +} + +func (fs *autoUnionFs) createFs(name string, roots []string) fuse.Status { + fs.lock.Lock() + defer fs.lock.Unlock() + + if fs.zombies[name] { + log.Printf("filesystem named %q is being removed", name) + return fuse.EBUSY + } + + for workspace, root := range fs.nameRootMap { + if root == roots[0] && workspace != name { + log.Printf("Already have a union FS for directory %s in workspace %s", + roots[0], workspace) + return fuse.EBUSY + } + } + + known := fs.knownFileSystems[name] + if known.unionFS != nil { + log.Println("Already have a workspace:", name) + return fuse.EBUSY + } + + ufs, err := NewUnionFsFromRoots(roots, &fs.options.UnionFsOptions, true) + if err != nil { + log.Println("Could not create UnionFs:", err) + return fuse.EPERM + } + + log.Printf("Adding workspace %v for roots %v", name, ufs.String()) + nfs := pathfs.NewPathNodeFs(ufs, &fs.options.PathNodeFsOptions) + code := fs.nodeFs.Mount(name, nfs.Root(), &fs.options.Options) + if code.Ok() { + fs.knownFileSystems[name] = knownFs{ + ufs, + nfs, + } + fs.nameRootMap[name] = roots[0] + } + return code +} + +func (fs *autoUnionFs) rmFs(name string) (code fuse.Status) { + fs.lock.Lock() + defer fs.lock.Unlock() + + if fs.zombies[name] { + return fuse.ENOENT + } + + known := fs.knownFileSystems[name] + if known.unionFS == nil { + return fuse.ENOENT + } + + root := fs.nameRootMap[name] + delete(fs.knownFileSystems, name) + delete(fs.nameRootMap, name) + fs.zombies[name] = true + fs.lock.Unlock() + code = fs.nodeFs.Unmount(name) + + fs.lock.Lock() + delete(fs.zombies, name) + if !code.Ok() { + // Reinstate. + log.Printf("Unmount failed for %s. Code %v", name, code) + fs.knownFileSystems[name] = known + fs.nameRootMap[name] = root + } + return code +} + +func (fs *autoUnionFs) addFs(name string, roots []string) (code fuse.Status) { + if name == _CONFIG || name == _STATUS || name == _SCAN_CONFIG { + return fuse.EINVAL + } + return fs.createFs(name, roots) +} + +func (fs *autoUnionFs) getRoots(path string) []string { + ro := filepath.Join(path, _READONLY) + fi, err := os.Lstat(ro) + fiDir, errDir := os.Stat(ro) + if err != nil || errDir != nil { + return nil + } + + if fi.Mode()&os.ModeSymlink != 0 && fiDir.IsDir() { + // TODO - should recurse and chain all READONLYs + // together. + return []string{path, ro} + } + return nil +} + +func (fs *autoUnionFs) visit(path string, fi os.FileInfo, err error) error { + if fi != nil && fi.IsDir() { + roots := fs.getRoots(path) + if roots != nil { + fs.addAutomaticFs(roots) + } + } + return nil +} + +func (fs *autoUnionFs) updateKnownFses() { + // We unroll the first level of entries in the root manually in order + // to allow symbolic links on that level. + directoryEntries, err := ioutil.ReadDir(fs.root) + if err == nil { + for _, dir := range directoryEntries { + if dir.IsDir() || dir.Mode()&os.ModeSymlink != 0 { + path := filepath.Join(fs.root, dir.Name()) + dir, _ = os.Stat(path) + fs.visit(path, dir, nil) + filepath.Walk(path, + func(path string, fi os.FileInfo, err error) error { + return fs.visit(path, fi, err) + }) + } + } + } +} + +func (fs *autoUnionFs) Readlink(path string, context *fuse.Context) (out string, code fuse.Status) { + comps := strings.Split(path, string(filepath.Separator)) + if comps[0] == _STATUS && comps[1] == _ROOT { + return fs.root, fuse.OK + } + + if comps[0] != _CONFIG { + return "", fuse.ENOENT + } + + name := comps[1] + + fs.lock.RLock() + defer fs.lock.RUnlock() + + root, ok := fs.nameRootMap[name] + if ok { + return root, fuse.OK + } + return "", fuse.ENOENT +} + +func (fs *autoUnionFs) getUnionFs(name string) pathfs.FileSystem { + fs.lock.RLock() + defer fs.lock.RUnlock() + return fs.knownFileSystems[name].unionFS +} + +func (fs *autoUnionFs) Symlink(pointedTo string, linkName string, context *fuse.Context) (code fuse.Status) { + comps := strings.Split(linkName, "/") + if len(comps) != 2 { + return fuse.EPERM + } + + if comps[0] == _CONFIG { + roots := fs.getRoots(pointedTo) + if roots == nil { + return fuse.Status(syscall.ENOTDIR) + } + + name := comps[1] + return fs.addFs(name, roots) + } + return fuse.EPERM +} + +func (fs *autoUnionFs) Unlink(path string, context *fuse.Context) (code fuse.Status) { + comps := strings.Split(path, "/") + if len(comps) != 2 { + return fuse.EPERM + } + + if comps[0] == _CONFIG && comps[1] != _SCAN_CONFIG { + code = fs.rmFs(comps[1]) + } else { + code = fuse.ENOENT + } + return code +} + +// Must define this, because ENOSYS will suspend all GetXAttr calls. +func (fs *autoUnionFs) GetXAttr(name string, attr string, context *fuse.Context) ([]byte, fuse.Status) { + return nil, fuse.ENOATTR +} + +func (fs *autoUnionFs) GetAttr(path string, context *fuse.Context) (*fuse.Attr, fuse.Status) { + a := &fuse.Attr{ + Owner: *fuse.CurrentOwner(), + } + if path == "" || path == _CONFIG || path == _STATUS { + a.Mode = fuse.S_IFDIR | 0755 + return a, fuse.OK + } + + if path == filepath.Join(_STATUS, _VERSION) { + a.Mode = fuse.S_IFREG | 0644 + a.Size = uint64(len(fs.options.Version)) + return a, fuse.OK + } + + if path == filepath.Join(_STATUS, _DEBUG) { + a.Mode = fuse.S_IFREG | 0644 + a.Size = uint64(len(fs.DebugData())) + return a, fuse.OK + } + + if path == filepath.Join(_STATUS, _ROOT) { + a.Mode = syscall.S_IFLNK | 0644 + return a, fuse.OK + } + + if path == filepath.Join(_CONFIG, _SCAN_CONFIG) { + a.Mode = fuse.S_IFREG | 0644 + return a, fuse.OK + } + comps := strings.Split(path, string(filepath.Separator)) + + if len(comps) > 1 && comps[0] == _CONFIG { + fs := fs.getUnionFs(comps[1]) + + if fs == nil { + return nil, fuse.ENOENT + } + + a.Mode = syscall.S_IFLNK | 0644 + return a, fuse.OK + } + + return nil, fuse.ENOENT +} + +func (fs *autoUnionFs) StatusDir() (stream []fuse.DirEntry, status fuse.Status) { + stream = make([]fuse.DirEntry, 0, 10) + stream = []fuse.DirEntry{ + {Name: _VERSION, Mode: fuse.S_IFREG | 0644}, + {Name: _DEBUG, Mode: fuse.S_IFREG | 0644}, + {Name: _ROOT, Mode: syscall.S_IFLNK | 0644}, + } + return stream, fuse.OK +} + +func (fs *autoUnionFs) DebugData() string { + conn := fs.nodeFs.Connector() + if conn.Server() == nil { + return "autoUnionFs.mountState not set" + } + setting := conn.Server().KernelSettings() + msg := fmt.Sprintf( + "Version: %v\n"+ + "Bufferpool: %v\n"+ + "Kernel: %v\n", + fs.options.Version, + conn.Server().DebugData(), + &setting) + + if conn != nil { + msg += fmt.Sprintf("Live inodes: %d\n", conn.InodeHandleCount()) + } + + return msg +} + +func (fs *autoUnionFs) Open(path string, flags uint32, context *fuse.Context) (nodefs.File, fuse.Status) { + if path == filepath.Join(_STATUS, _DEBUG) { + if flags&fuse.O_ANYWRITE != 0 { + return nil, fuse.EPERM + } + + return nodefs.NewDataFile([]byte(fs.DebugData())), fuse.OK + } + if path == filepath.Join(_STATUS, _VERSION) { + if flags&fuse.O_ANYWRITE != 0 { + return nil, fuse.EPERM + } + return nodefs.NewDataFile([]byte(fs.options.Version)), fuse.OK + } + if path == filepath.Join(_CONFIG, _SCAN_CONFIG) { + if flags&fuse.O_ANYWRITE != 0 { + fs.updateKnownFses() + } + return nodefs.NewDevNullFile(), fuse.OK + } + return nil, fuse.ENOENT +} + +func (fs *autoUnionFs) Truncate(name string, offset uint64, context *fuse.Context) (code fuse.Status) { + if name != filepath.Join(_CONFIG, _SCAN_CONFIG) { + log.Println("Huh? Truncating unsupported write file", name) + return fuse.EPERM + } + return fuse.OK +} + +func (fs *autoUnionFs) OpenDir(name string, context *fuse.Context) (stream []fuse.DirEntry, status fuse.Status) { + switch name { + case _STATUS: + return fs.StatusDir() + case _CONFIG: + case "/": + name = "" + case "": + default: + log.Printf("Argh! Don't know how to list dir %v", name) + return nil, fuse.ENOSYS + } + + fs.lock.RLock() + defer fs.lock.RUnlock() + + stream = make([]fuse.DirEntry, 0, len(fs.knownFileSystems)+5) + if name == _CONFIG { + for k := range fs.knownFileSystems { + stream = append(stream, fuse.DirEntry{ + Name: k, + Mode: syscall.S_IFLNK | 0644, + }) + } + } + + if name == "" { + stream = append(stream, fuse.DirEntry{ + Name: _CONFIG, + Mode: uint32(fuse.S_IFDIR | 0755), + }, + fuse.DirEntry{ + Name: _STATUS, + Mode: uint32(fuse.S_IFDIR | 0755), + }) + } + return stream, status +} + +func (fs *autoUnionFs) StatFs(name string) *fuse.StatfsOut { + return &fuse.StatfsOut{} +} diff --git a/vendor/github.com/hanwen/go-fuse/unionfs/cachingfs.go b/vendor/github.com/hanwen/go-fuse/unionfs/cachingfs.go new file mode 100644 index 0000000..7f97e01 --- /dev/null +++ b/vendor/github.com/hanwen/go-fuse/unionfs/cachingfs.go @@ -0,0 +1,151 @@ +// Copyright 2016 the Go-FUSE Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unionfs + +import ( + "fmt" + "log" + "strings" + "time" + + "github.com/hanwen/go-fuse/fuse" + "github.com/hanwen/go-fuse/fuse/nodefs" + "github.com/hanwen/go-fuse/fuse/pathfs" +) + +const _XATTRSEP = "@XATTR@" + +type attrResponse struct { + *fuse.Attr + fuse.Status +} + +type xattrResponse struct { + data []byte + fuse.Status +} + +type dirResponse struct { + entries []fuse.DirEntry + fuse.Status +} + +type linkResponse struct { + linkContent string + fuse.Status +} + +// Caches filesystem metadata. +type cachingFileSystem struct { + pathfs.FileSystem + + attributes *TimedCache + dirs *TimedCache + links *TimedCache + xattr *TimedCache +} + +func readDir(fs pathfs.FileSystem, name string) *dirResponse { + origStream, code := fs.OpenDir(name, nil) + + r := &dirResponse{nil, code} + if !code.Ok() { + return r + } + r.entries = origStream + return r +} + +func getAttr(fs pathfs.FileSystem, name string) *attrResponse { + a, code := fs.GetAttr(name, nil) + return &attrResponse{ + Attr: a, + Status: code, + } +} + +func getXAttr(fs pathfs.FileSystem, nameAttr string) *xattrResponse { + ns := strings.SplitN(nameAttr, _XATTRSEP, 2) + a, code := fs.GetXAttr(ns[0], ns[1], nil) + return &xattrResponse{ + data: a, + Status: code, + } +} + +func readLink(fs pathfs.FileSystem, name string) *linkResponse { + a, code := fs.Readlink(name, nil) + return &linkResponse{ + linkContent: a, + Status: code, + } +} + +func NewCachingFileSystem(fs pathfs.FileSystem, ttl time.Duration) pathfs.FileSystem { + c := new(cachingFileSystem) + c.FileSystem = fs + c.attributes = NewTimedCache(func(n string) (interface{}, bool) { + a := getAttr(fs, n) + return a, a.Ok() + }, ttl) + c.dirs = NewTimedCache(func(n string) (interface{}, bool) { + d := readDir(fs, n) + return d, d.Ok() + }, ttl) + c.links = NewTimedCache(func(n string) (interface{}, bool) { + l := readLink(fs, n) + return l, l.Ok() + }, ttl) + c.xattr = NewTimedCache(func(n string) (interface{}, bool) { + l := getXAttr(fs, n) + return l, l.Ok() + }, ttl) + return c +} + +func (fs *cachingFileSystem) DropCache() { + for _, c := range []*TimedCache{fs.attributes, fs.dirs, fs.links, fs.xattr} { + c.DropAll(nil) + } +} + +func (fs *cachingFileSystem) GetAttr(name string, context *fuse.Context) (*fuse.Attr, fuse.Status) { + if name == _DROP_CACHE { + return &fuse.Attr{ + Mode: fuse.S_IFREG | 0777, + }, fuse.OK + } + + r := fs.attributes.Get(name).(*attrResponse) + return r.Attr, r.Status +} + +func (fs *cachingFileSystem) GetXAttr(name string, attr string, context *fuse.Context) ([]byte, fuse.Status) { + key := name + _XATTRSEP + attr + r := fs.xattr.Get(key).(*xattrResponse) + return r.data, r.Status +} + +func (fs *cachingFileSystem) Readlink(name string, context *fuse.Context) (string, fuse.Status) { + r := fs.links.Get(name).(*linkResponse) + return r.linkContent, r.Status +} + +func (fs *cachingFileSystem) OpenDir(name string, context *fuse.Context) (stream []fuse.DirEntry, status fuse.Status) { + r := fs.dirs.Get(name).(*dirResponse) + return r.entries, r.Status +} + +func (fs *cachingFileSystem) String() string { + return fmt.Sprintf("cachingFileSystem(%v)", fs.FileSystem) +} + +func (fs *cachingFileSystem) Open(name string, flags uint32, context *fuse.Context) (f nodefs.File, status fuse.Status) { + if flags&fuse.O_ANYWRITE != 0 && name == _DROP_CACHE { + log.Println("Dropping cache for", fs) + fs.DropCache() + } + return fs.FileSystem.Open(name, flags, context) +} diff --git a/vendor/github.com/hanwen/go-fuse/unionfs/create.go b/vendor/github.com/hanwen/go-fuse/unionfs/create.go new file mode 100644 index 0000000..da65e83 --- /dev/null +++ b/vendor/github.com/hanwen/go-fuse/unionfs/create.go @@ -0,0 +1,36 @@ +// Copyright 2016 the Go-FUSE Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unionfs + +import ( + "os" + + "github.com/hanwen/go-fuse/fuse/pathfs" +) + +func NewUnionFsFromRoots(roots []string, opts *UnionFsOptions, roCaching bool) (pathfs.FileSystem, error) { + fses := make([]pathfs.FileSystem, 0) + for i, r := range roots { + var fs pathfs.FileSystem + fi, err := os.Stat(r) + if err != nil { + return nil, err + } + if fi.IsDir() { + fs = pathfs.NewLoopbackFileSystem(r) + } + if fs == nil { + return nil, err + + } + if i > 0 && roCaching { + fs = NewCachingFileSystem(fs, 0) + } + + fses = append(fses, fs) + } + + return NewUnionFs(fses, *opts) +} diff --git a/vendor/github.com/hanwen/go-fuse/unionfs/dircache.go b/vendor/github.com/hanwen/go-fuse/unionfs/dircache.go new file mode 100644 index 0000000..b37b78f --- /dev/null +++ b/vendor/github.com/hanwen/go-fuse/unionfs/dircache.go @@ -0,0 +1,125 @@ +// Copyright 2016 the Go-FUSE Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unionfs + +import ( + "sync" + "time" + + "github.com/hanwen/go-fuse/fuse" + "github.com/hanwen/go-fuse/fuse/pathfs" +) + +// newDirnameMap reads the contents of the given directory. On error, +// returns a nil map. This forces reloads in the dirCache until we +// succeed. +func newDirnameMap(fs pathfs.FileSystem, dir string) map[string]bool { + stream, code := fs.OpenDir(dir, nil) + if code == fuse.ENOENT { + // The directory not existing is not an error. + return map[string]bool{} + } + + if !code.Ok() { + return nil + } + + result := make(map[string]bool) + for _, e := range stream { + if e.Mode&fuse.S_IFREG != 0 { + result[e.Name] = true + } + } + return result +} + +// dirCache caches names in a directory for some time. +// +// If called when the cache is expired, the filenames are read afresh in +// the background. +type dirCache struct { + dir string + ttl time.Duration + fs pathfs.FileSystem + // Protects data below. + lock sync.RWMutex + + // If nil, you may call refresh() to schedule a new one. + names map[string]bool + updateRunning bool +} + +func (c *dirCache) setMap(newMap map[string]bool) { + c.lock.Lock() + defer c.lock.Unlock() + + c.names = newMap + c.updateRunning = false + _ = time.AfterFunc(c.ttl, + func() { c.DropCache() }) +} + +func (c *dirCache) DropCache() { + c.lock.Lock() + defer c.lock.Unlock() + c.names = nil +} + +// Try to refresh: if another update is already running, do nothing, +// otherwise, read the directory and set it. +func (c *dirCache) maybeRefresh() { + c.lock.Lock() + defer c.lock.Unlock() + if c.updateRunning { + return + } + c.updateRunning = true + go func() { + newmap := newDirnameMap(c.fs, c.dir) + c.setMap(newmap) + }() +} + +func (c *dirCache) RemoveEntry(name string) { + c.lock.Lock() + defer c.lock.Unlock() + if c.names == nil { + go c.maybeRefresh() + return + } + + delete(c.names, name) +} + +func (c *dirCache) AddEntry(name string) { + c.lock.Lock() + defer c.lock.Unlock() + if c.names == nil { + go c.maybeRefresh() + return + } + + c.names[name] = true +} + +func newDirCache(fs pathfs.FileSystem, dir string, ttl time.Duration) *dirCache { + dc := new(dirCache) + dc.dir = dir + dc.fs = fs + dc.ttl = ttl + return dc +} + +func (c *dirCache) HasEntry(name string) (mapPresent bool, found bool) { + c.lock.RLock() + defer c.lock.RUnlock() + + if c.names == nil { + go c.maybeRefresh() + return false, false + } + + return true, c.names[name] +} diff --git a/vendor/github.com/hanwen/go-fuse/unionfs/timedcache.go b/vendor/github.com/hanwen/go-fuse/unionfs/timedcache.go new file mode 100644 index 0000000..ab64492 --- /dev/null +++ b/vendor/github.com/hanwen/go-fuse/unionfs/timedcache.go @@ -0,0 +1,111 @@ +// Copyright 2016 the Go-FUSE Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unionfs + +import ( + "sync" + "time" +) + +type cacheEntry struct { + data interface{} + + // expiry is the absolute timestamp of the expiry. + expiry time.Time +} + +// TimedIntCache caches the result of fetch() for some time. It is +// thread-safe. Calls of fetch() do no happen inside a critical +// section, so when multiple concurrent Get()s happen for the same +// key, multiple fetch() calls may be issued for the same key. +type TimedCacheFetcher func(name string) (value interface{}, cacheable bool) +type TimedCache struct { + fetch TimedCacheFetcher + + // ttl is the duration of the cache. + ttl time.Duration + + cacheMapMutex sync.RWMutex + cacheMap map[string]*cacheEntry + + PurgeTimer *time.Timer +} + +// Creates a new cache with the given TTL. If TTL <= 0, the caching is +// indefinite. +func NewTimedCache(fetcher TimedCacheFetcher, ttl time.Duration) *TimedCache { + l := new(TimedCache) + l.ttl = ttl + l.fetch = fetcher + l.cacheMap = make(map[string]*cacheEntry) + return l +} + +func (c *TimedCache) Get(name string) interface{} { + c.cacheMapMutex.RLock() + info, ok := c.cacheMap[name] + c.cacheMapMutex.RUnlock() + + valid := ok && (c.ttl <= 0 || info.expiry.After(time.Now())) + if valid { + return info.data + } + return c.GetFresh(name) +} + +func (c *TimedCache) Set(name string, val interface{}) { + c.cacheMapMutex.Lock() + defer c.cacheMapMutex.Unlock() + + c.cacheMap[name] = &cacheEntry{ + data: val, + expiry: time.Now().Add(c.ttl), + } +} + +func (c *TimedCache) DropEntry(name string) { + c.cacheMapMutex.Lock() + defer c.cacheMapMutex.Unlock() + + delete(c.cacheMap, name) +} + +func (c *TimedCache) GetFresh(name string) interface{} { + data, ok := c.fetch(name) + if ok { + c.Set(name, data) + } + return data +} + +// Drop all expired entries. +func (c *TimedCache) Purge() { + keys := make([]string, 0, len(c.cacheMap)) + now := time.Now() + + c.cacheMapMutex.Lock() + defer c.cacheMapMutex.Unlock() + for k, v := range c.cacheMap { + if now.After(v.expiry) { + keys = append(keys, k) + } + } + for _, k := range keys { + delete(c.cacheMap, k) + } +} + +func (c *TimedCache) DropAll(names []string) { + c.cacheMapMutex.Lock() + defer c.cacheMapMutex.Unlock() + + if names == nil { + c.cacheMap = make(map[string]*cacheEntry, len(c.cacheMap)) + } else { + for _, nm := range names { + delete(c.cacheMap, nm) + } + } +} diff --git a/vendor/github.com/hanwen/go-fuse/unionfs/unionfs.go b/vendor/github.com/hanwen/go-fuse/unionfs/unionfs.go new file mode 100644 index 0000000..cc93e49 --- /dev/null +++ b/vendor/github.com/hanwen/go-fuse/unionfs/unionfs.go @@ -0,0 +1,1032 @@ +// Copyright 2016 the Go-FUSE Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unionfs + +import ( + "crypto/md5" + "fmt" + "log" + "os" + "path" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/hanwen/go-fuse/fuse" + "github.com/hanwen/go-fuse/fuse/nodefs" + "github.com/hanwen/go-fuse/fuse/pathfs" +) + +func filePathHash(path string) string { + dir, base := filepath.Split(path) + + h := md5.New() + h.Write([]byte(dir)) + return fmt.Sprintf("%x-%s", h.Sum(nil)[:8], base) +} + +/* + + UnionFs implements a user-space union file system, which is + stateless but efficient even if the writable branch is on NFS. + + + Assumptions: + + * It uses a list of branches, the first of which (index 0) is thought + to be writable, and the rest read-only. + + * It assumes that the number of deleted files is small relative to + the total tree size. + + + Implementation notes. + + * It overlays arbitrary writable FileSystems with any number of + readonly FileSystems. + + * Deleting a file will put a file named + /DELETIONS/HASH-OF-FULL-FILENAME into the writable overlay, + containing the full filename itself. + + This is optimized for NFS usage: we want to minimize the number of + NFS operations, which are slow. By putting all whiteouts in one + place, we can cheaply fetch the list of all deleted files. Even + without caching on our side, the kernel's negative dentry cache can + answer is-deleted queries quickly. + +*/ +type unionFS struct { + pathfs.FileSystem + + // The same, but as interfaces. + fileSystems []pathfs.FileSystem + + // A file-existence cache. + deletionCache *dirCache + + // A file -> branch cache. + branchCache *TimedCache + + // Map of files to hide. + hiddenFiles map[string]bool + + options *UnionFsOptions + nodeFs *pathfs.PathNodeFs +} + +type UnionFsOptions struct { + BranchCacheTTL time.Duration + DeletionCacheTTL time.Duration + DeletionDirName string + HiddenFiles []string +} + +const ( + _DROP_CACHE = ".drop_cache" +) + +func NewUnionFs(fileSystems []pathfs.FileSystem, options UnionFsOptions) (pathfs.FileSystem, error) { + g := &unionFS{ + options: &options, + fileSystems: fileSystems, + FileSystem: pathfs.NewDefaultFileSystem(), + } + + writable := g.fileSystems[0] + code := g.createDeletionStore() + if !code.Ok() { + return nil, fmt.Errorf("could not create deletion path %v: %v", options.DeletionDirName, code) + } + + g.deletionCache = newDirCache(writable, options.DeletionDirName, options.DeletionCacheTTL) + g.branchCache = NewTimedCache( + func(n string) (interface{}, bool) { return g.getBranchAttrNoCache(n), true }, + options.BranchCacheTTL) + + g.hiddenFiles = make(map[string]bool) + for _, name := range options.HiddenFiles { + g.hiddenFiles[name] = true + } + + return g, nil +} + +func (fs *unionFS) OnMount(nodeFs *pathfs.PathNodeFs) { + fs.nodeFs = nodeFs +} + +//////////////// +// Deal with all the caches. + +// The isDeleted() method tells us if a path has a marker in the deletion store. +// It may return an error code if the store could not be accessed. +func (fs *unionFS) isDeleted(name string) (deleted bool, code fuse.Status) { + marker := fs.deletionPath(name) + haveCache, found := fs.deletionCache.HasEntry(filepath.Base(marker)) + if haveCache { + return found, fuse.OK + } + + _, code = fs.fileSystems[0].GetAttr(marker, nil) + + if code == fuse.OK { + return true, code + } + if code == fuse.ENOENT { + return false, fuse.OK + } + + log.Printf("error accessing deletion marker %s: %v", marker, code) + return false, fuse.Status(syscall.EROFS) +} + +func (fs *unionFS) createDeletionStore() (code fuse.Status) { + writable := fs.fileSystems[0] + fi, code := writable.GetAttr(fs.options.DeletionDirName, nil) + if code == fuse.ENOENT { + code = writable.Mkdir(fs.options.DeletionDirName, 0755, nil) + if code.Ok() { + fi, code = writable.GetAttr(fs.options.DeletionDirName, nil) + } + } + + if !code.Ok() || !fi.IsDir() { + code = fuse.Status(syscall.EROFS) + } + + return code +} + +func (fs *unionFS) getBranch(name string) branchResult { + name = stripSlash(name) + r := fs.branchCache.Get(name) + return r.(branchResult) +} + +func (fs *unionFS) setBranch(name string, r branchResult) { + if !r.valid() { + log.Panicf("entry %q setting illegal branchResult %v", name, r) + } + fs.branchCache.Set(name, r) +} + +type branchResult struct { + attr *fuse.Attr + code fuse.Status + branch int +} + +func (r *branchResult) valid() bool { + return (r.branch >= 0 && r.attr != nil && r.code.Ok()) || + (r.branch < 0 && r.attr == nil && !r.code.Ok()) +} + +func (fs branchResult) String() string { + return fmt.Sprintf("{%v %v branch %d}", fs.attr, fs.code, fs.branch) +} + +func (fs *unionFS) getBranchAttrNoCache(name string) branchResult { + name = stripSlash(name) + + parent, base := path.Split(name) + parent = stripSlash(parent) + + parentBranch := 0 + if base != "" { + parentBranch = fs.getBranch(parent).branch + } + for i, fs := range fs.fileSystems { + if i < parentBranch { + continue + } + + a, s := fs.GetAttr(name, nil) + if s.Ok() { + if i > 0 { + // Needed to make hardlinks work. + a.Ino = 0 + } + return branchResult{ + attr: a, + code: s, + branch: i, + } + } else { + if s != fuse.ENOENT { + log.Printf("getattr: %v: Got error %v from branch %v", name, s, i) + } + } + } + return branchResult{nil, fuse.ENOENT, -1} +} + +//////////////// +// Deletion. + +func (fs *unionFS) deletionPath(name string) string { + return filepath.Join(fs.options.DeletionDirName, filePathHash(name)) +} + +func (fs *unionFS) removeDeletion(name string) { + marker := fs.deletionPath(name) + fs.deletionCache.RemoveEntry(path.Base(marker)) + + // os.Remove tries to be smart and issues a Remove() and + // Rmdir() sequentially. We want to skip the 2nd system call, + // so use syscall.Unlink() directly. + + code := fs.fileSystems[0].Unlink(marker, nil) + if !code.Ok() && code != fuse.ENOENT { + log.Printf("error unlinking %s: %v", marker, code) + } +} + +func (fs *unionFS) putDeletion(name string) (code fuse.Status) { + code = fs.createDeletionStore() + if !code.Ok() { + return code + } + + marker := fs.deletionPath(name) + fs.deletionCache.AddEntry(path.Base(marker)) + + // Is there a WriteStringToFileOrDie ? + writable := fs.fileSystems[0] + fi, code := writable.GetAttr(marker, nil) + if code.Ok() && fi.Size == uint64(len(name)) { + return fuse.OK + } + + var f nodefs.File + if code == fuse.ENOENT { + f, code = writable.Create(marker, uint32(os.O_TRUNC|os.O_WRONLY), 0644, nil) + } else { + writable.Chmod(marker, 0644, nil) + f, code = writable.Open(marker, uint32(os.O_TRUNC|os.O_WRONLY), nil) + } + if !code.Ok() { + log.Printf("could not create deletion file %v: %v", marker, code) + return fuse.EPERM + } + defer f.Release() + defer f.Flush() + n, code := f.Write([]byte(name), 0) + if int(n) != len(name) || !code.Ok() { + panic(fmt.Sprintf("Error for writing %v: %v, %v (exp %v) %v", name, marker, n, len(name), code)) + } + + return fuse.OK +} + +//////////////// +// Promotion. + +func (fs *unionFS) Promote(name string, srcResult branchResult, context *fuse.Context) (code fuse.Status) { + writable := fs.fileSystems[0] + sourceFs := fs.fileSystems[srcResult.branch] + + // Promote directories. + fs.promoteDirsTo(name) + + if srcResult.attr.IsRegular() { + code = pathfs.CopyFile(sourceFs, writable, name, name, context) + + if code.Ok() { + code = writable.Chmod(name, srcResult.attr.Mode&07777|0200, context) + } + if code.Ok() { + aTime := srcResult.attr.AccessTime() + mTime := srcResult.attr.ModTime() + code = writable.Utimens(name, &aTime, &mTime, context) + } + + files := fs.nodeFs.AllFiles(name, 0) + for _, fileWrapper := range files { + if !code.Ok() { + break + } + var uf *unionFsFile + f := fileWrapper.File + for f != nil { + ok := false + uf, ok = f.(*unionFsFile) + if ok { + break + } + f = f.InnerFile() + } + if uf == nil { + panic("no unionFsFile found inside") + } + + if uf.layer > 0 { + uf.layer = 0 + f := uf.File + uf.File, code = fs.fileSystems[0].Open(name, fileWrapper.OpenFlags, context) + f.Flush() + f.Release() + } + } + } else if srcResult.attr.IsSymlink() { + link := "" + link, code = sourceFs.Readlink(name, context) + if !code.Ok() { + log.Println("can't read link in source fs", name) + } else { + code = writable.Symlink(link, name, context) + } + } else if srcResult.attr.IsDir() { + code = writable.Mkdir(name, srcResult.attr.Mode&07777|0200, context) + } else { + log.Println("Unknown file type:", srcResult.attr) + return fuse.ENOSYS + } + + if !code.Ok() { + fs.branchCache.GetFresh(name) + return code + } else { + r := fs.getBranch(name) + r.branch = 0 + fs.setBranch(name, r) + } + + return fuse.OK +} + +//////////////////////////////////////////////////////////////// +// Below: implement interface for a FileSystem. + +func (fs *unionFS) Link(orig string, newName string, context *fuse.Context) (code fuse.Status) { + origResult := fs.getBranch(orig) + code = origResult.code + if code.Ok() && origResult.branch > 0 { + code = fs.Promote(orig, origResult, context) + } + if code.Ok() && origResult.branch > 0 { + // Hairy: for the link to be hooked up to the existing + // inode, PathNodeFs must see a client inode for the + // original. We force a refresh of the attribute (so + // the Ino is filled in.), and then force PathNodeFs + // to see the Inode number. + fs.branchCache.GetFresh(orig) + inode := fs.nodeFs.Node(orig) + var a fuse.Attr + inode.Node().GetAttr(&a, nil, nil) + } + if code.Ok() { + code = fs.promoteDirsTo(newName) + } + if code.Ok() { + code = fs.fileSystems[0].Link(orig, newName, context) + } + if code.Ok() { + fs.removeDeletion(newName) + fs.branchCache.GetFresh(newName) + } + return code +} + +func (fs *unionFS) Rmdir(path string, context *fuse.Context) (code fuse.Status) { + r := fs.getBranch(path) + if r.code != fuse.OK { + return r.code + } + if !r.attr.IsDir() { + return fuse.Status(syscall.ENOTDIR) + } + + stream, code := fs.OpenDir(path, context) + found := false + for _ = range stream { + found = true + } + if found { + return fuse.Status(syscall.ENOTEMPTY) + } + + if r.branch > 0 { + code = fs.putDeletion(path) + return code + } + code = fs.fileSystems[0].Rmdir(path, context) + if code != fuse.OK { + return code + } + + r = fs.branchCache.GetFresh(path).(branchResult) + if r.branch > 0 { + code = fs.putDeletion(path) + } + return code +} + +func (fs *unionFS) Mkdir(path string, mode uint32, context *fuse.Context) (code fuse.Status) { + deleted, code := fs.isDeleted(path) + if !code.Ok() { + return code + } + + if !deleted { + r := fs.getBranch(path) + if r.code != fuse.ENOENT { + return fuse.Status(syscall.EEXIST) + } + } + + code = fs.promoteDirsTo(path) + if code.Ok() { + code = fs.fileSystems[0].Mkdir(path, mode, context) + } + if code.Ok() { + fs.removeDeletion(path) + attr := &fuse.Attr{ + Mode: fuse.S_IFDIR | mode, + } + fs.setBranch(path, branchResult{attr, fuse.OK, 0}) + } + + var stream []fuse.DirEntry + stream, code = fs.OpenDir(path, context) + if code.Ok() { + // This shouldn't happen, but let's be safe. + for _, entry := range stream { + fs.putDeletion(filepath.Join(path, entry.Name)) + } + } + + return code +} + +func (fs *unionFS) Symlink(pointedTo string, linkName string, context *fuse.Context) (code fuse.Status) { + code = fs.promoteDirsTo(linkName) + if code.Ok() { + code = fs.fileSystems[0].Symlink(pointedTo, linkName, context) + } + if code.Ok() { + fs.removeDeletion(linkName) + fs.branchCache.GetFresh(linkName) + } + return code +} + +func (fs *unionFS) Truncate(path string, size uint64, context *fuse.Context) (code fuse.Status) { + if path == _DROP_CACHE { + return fuse.OK + } + + r := fs.getBranch(path) + if r.branch > 0 { + code = fs.Promote(path, r, context) + r.branch = 0 + } + + if code.Ok() { + code = fs.fileSystems[0].Truncate(path, size, context) + } + if code.Ok() { + r.attr.Size = size + now := time.Now() + r.attr.SetTimes(nil, &now, &now) + fs.setBranch(path, r) + } + return code +} + +func (fs *unionFS) Utimens(name string, atime *time.Time, mtime *time.Time, context *fuse.Context) (code fuse.Status) { + name = stripSlash(name) + r := fs.getBranch(name) + + code = r.code + if code.Ok() && r.branch > 0 { + code = fs.Promote(name, r, context) + r.branch = 0 + } + if code.Ok() { + code = fs.fileSystems[0].Utimens(name, atime, mtime, context) + } + if code.Ok() { + now := time.Now() + r.attr.SetTimes(atime, mtime, &now) + fs.setBranch(name, r) + } + return code +} + +func (fs *unionFS) Chown(name string, uid uint32, gid uint32, context *fuse.Context) (code fuse.Status) { + name = stripSlash(name) + r := fs.getBranch(name) + if r.attr == nil || r.code != fuse.OK { + return r.code + } + + if os.Geteuid() != 0 { + return fuse.EPERM + } + + if r.attr.Uid != uid || r.attr.Gid != gid { + if r.branch > 0 { + code := fs.Promote(name, r, context) + if code != fuse.OK { + return code + } + r.branch = 0 + } + fs.fileSystems[0].Chown(name, uid, gid, context) + } + r.attr.Uid = uid + r.attr.Gid = gid + now := time.Now() + r.attr.SetTimes(nil, nil, &now) + fs.setBranch(name, r) + return fuse.OK +} + +func (fs *unionFS) Chmod(name string, mode uint32, context *fuse.Context) (code fuse.Status) { + name = stripSlash(name) + r := fs.getBranch(name) + if r.attr == nil { + return r.code + } + if r.code != fuse.OK { + return r.code + } + + permMask := uint32(07777) + + // Always be writable. + oldMode := r.attr.Mode & permMask + + if oldMode != mode { + if r.branch > 0 { + code := fs.Promote(name, r, context) + if code != fuse.OK { + return code + } + r.branch = 0 + } + fs.fileSystems[0].Chmod(name, mode, context) + } + r.attr.Mode = (r.attr.Mode &^ permMask) | mode + now := time.Now() + r.attr.SetTimes(nil, nil, &now) + fs.setBranch(name, r) + return fuse.OK +} + +func (fs *unionFS) Access(name string, mode uint32, context *fuse.Context) (code fuse.Status) { + // We always allow writing. + mode = mode &^ fuse.W_OK + if name == "" || name == _DROP_CACHE { + return fuse.OK + } + r := fs.getBranch(name) + if r.branch >= 0 { + return fs.fileSystems[r.branch].Access(name, mode, context) + } + return fuse.ENOENT +} + +func (fs *unionFS) Unlink(name string, context *fuse.Context) (code fuse.Status) { + r := fs.getBranch(name) + if r.branch == 0 { + code = fs.fileSystems[0].Unlink(name, context) + if code != fuse.OK { + return code + } + r = fs.branchCache.GetFresh(name).(branchResult) + } + + if r.branch > 0 { + // It would be nice to do the putDeletion async. + code = fs.putDeletion(name) + } + return code +} + +func (fs *unionFS) Readlink(name string, context *fuse.Context) (out string, code fuse.Status) { + r := fs.getBranch(name) + if r.branch >= 0 { + return fs.fileSystems[r.branch].Readlink(name, context) + } + return "", fuse.ENOENT +} + +func stripSlash(fn string) string { + return strings.TrimRight(fn, string(filepath.Separator)) +} + +func (fs *unionFS) promoteDirsTo(filename string) fuse.Status { + dirName, _ := filepath.Split(filename) + dirName = stripSlash(dirName) + + var todo []string + var results []branchResult + for dirName != "" { + r := fs.getBranch(dirName) + + if !r.code.Ok() { + log.Println("path component does not exist", filename, dirName) + } + if !r.attr.IsDir() { + log.Println("path component is not a directory.", dirName, r) + return fuse.EPERM + } + if r.branch == 0 { + break + } + todo = append(todo, dirName) + results = append(results, r) + dirName, _ = filepath.Split(dirName) + dirName = stripSlash(dirName) + } + + for i := range todo { + j := len(todo) - i - 1 + d := todo[j] + r := results[j] + code := fs.fileSystems[0].Mkdir(d, r.attr.Mode&07777|0200, nil) + if code != fuse.OK { + log.Println("Error creating dir leading to path", d, code, fs.fileSystems[0]) + return fuse.EPERM + } + + aTime := r.attr.AccessTime() + mTime := r.attr.ModTime() + fs.fileSystems[0].Utimens(d, &aTime, &mTime, nil) + r.branch = 0 + fs.setBranch(d, r) + } + return fuse.OK +} + +func (fs *unionFS) Create(name string, flags uint32, mode uint32, context *fuse.Context) (fuseFile nodefs.File, code fuse.Status) { + writable := fs.fileSystems[0] + + code = fs.promoteDirsTo(name) + if code != fuse.OK { + return nil, code + } + fuseFile, code = writable.Create(name, flags, mode, context) + if code.Ok() { + fuseFile = fs.newUnionFsFile(fuseFile, 0) + fs.removeDeletion(name) + + now := time.Now() + a := fuse.Attr{ + Mode: fuse.S_IFREG | mode, + } + a.SetTimes(nil, &now, &now) + fs.setBranch(name, branchResult{&a, fuse.OK, 0}) + } + return fuseFile, code +} + +func (fs *unionFS) GetAttr(name string, context *fuse.Context) (a *fuse.Attr, s fuse.Status) { + _, hidden := fs.hiddenFiles[name] + if hidden { + return nil, fuse.ENOENT + } + if name == _DROP_CACHE { + return &fuse.Attr{ + Mode: fuse.S_IFREG | 0777, + }, fuse.OK + } + if name == fs.options.DeletionDirName { + return nil, fuse.ENOENT + } + isDel, s := fs.isDeleted(name) + if !s.Ok() { + return nil, s + } + + if isDel { + return nil, fuse.ENOENT + } + r := fs.getBranch(name) + if r.branch < 0 { + return nil, fuse.ENOENT + } + fi := *r.attr + // Make everything appear writable. + fi.Mode |= 0200 + return &fi, r.code +} + +func (fs *unionFS) GetXAttr(name string, attr string, context *fuse.Context) ([]byte, fuse.Status) { + if name == _DROP_CACHE { + return nil, fuse.ENOATTR + } + r := fs.getBranch(name) + if r.branch >= 0 { + return fs.fileSystems[r.branch].GetXAttr(name, attr, context) + } + return nil, fuse.ENOENT +} + +func (fs *unionFS) OpenDir(directory string, context *fuse.Context) (stream []fuse.DirEntry, status fuse.Status) { + dirBranch := fs.getBranch(directory) + if dirBranch.branch < 0 { + return nil, fuse.ENOENT + } + + // We could try to use the cache, but we have a delay, so + // might as well get the fresh results async. + var wg sync.WaitGroup + var deletions map[string]bool + + wg.Add(1) + go func() { + deletions = newDirnameMap(fs.fileSystems[0], fs.options.DeletionDirName) + wg.Done() + }() + + entries := make([]map[string]uint32, len(fs.fileSystems)) + for i := range fs.fileSystems { + entries[i] = make(map[string]uint32) + } + + statuses := make([]fuse.Status, len(fs.fileSystems)) + for i, l := range fs.fileSystems { + if i >= dirBranch.branch { + wg.Add(1) + go func(j int, pfs pathfs.FileSystem) { + ch, s := pfs.OpenDir(directory, context) + statuses[j] = s + for _, v := range ch { + entries[j][v.Name] = v.Mode + } + wg.Done() + }(i, l) + } + } + + wg.Wait() + if deletions == nil { + _, code := fs.fileSystems[0].GetAttr(fs.options.DeletionDirName, context) + if code == fuse.ENOENT { + deletions = map[string]bool{} + } else { + return nil, fuse.Status(syscall.EROFS) + } + } + + results := entries[0] + + // TODO(hanwen): should we do anything with the return + // statuses? + for i, m := range entries { + if statuses[i] != fuse.OK { + continue + } + if i == 0 { + // We don't need to further process the first + // branch: it has no deleted files. + continue + } + for k, v := range m { + _, ok := results[k] + if ok { + continue + } + + deleted := deletions[filePathHash(filepath.Join(directory, k))] + if !deleted { + results[k] = v + } + } + } + if directory == "" { + delete(results, fs.options.DeletionDirName) + for name, _ := range fs.hiddenFiles { + delete(results, name) + } + } + + stream = make([]fuse.DirEntry, 0, len(results)) + for k, v := range results { + stream = append(stream, fuse.DirEntry{ + Name: k, + Mode: v, + }) + } + return stream, fuse.OK +} + +// recursivePromote promotes path, and if a directory, everything +// below that directory. It returns a list of all promoted paths, in +// full, including the path itself. +func (fs *unionFS) recursivePromote(path string, pathResult branchResult, context *fuse.Context) (names []string, code fuse.Status) { + names = []string{} + if pathResult.branch > 0 { + code = fs.Promote(path, pathResult, context) + } + + if code.Ok() { + names = append(names, path) + } + + if code.Ok() && pathResult.attr != nil && pathResult.attr.IsDir() { + var stream []fuse.DirEntry + stream, code = fs.OpenDir(path, context) + for _, e := range stream { + if !code.Ok() { + break + } + subnames := []string{} + p := filepath.Join(path, e.Name) + r := fs.getBranch(p) + subnames, code = fs.recursivePromote(p, r, context) + names = append(names, subnames...) + } + } + + if !code.Ok() { + names = nil + } + return names, code +} + +func (fs *unionFS) renameDirectory(srcResult branchResult, srcDir string, dstDir string, context *fuse.Context) (code fuse.Status) { + names := []string{} + if code.Ok() { + names, code = fs.recursivePromote(srcDir, srcResult, context) + } + if code.Ok() { + code = fs.promoteDirsTo(dstDir) + } + + if code.Ok() { + writable := fs.fileSystems[0] + code = writable.Rename(srcDir, dstDir, context) + } + + if code.Ok() { + for _, srcName := range names { + relative := strings.TrimLeft(srcName[len(srcDir):], string(filepath.Separator)) + dst := filepath.Join(dstDir, relative) + fs.removeDeletion(dst) + + srcResult := fs.getBranch(srcName) + srcResult.branch = 0 + fs.setBranch(dst, srcResult) + + srcResult = fs.branchCache.GetFresh(srcName).(branchResult) + if srcResult.branch > 0 { + code = fs.putDeletion(srcName) + } + } + } + return code +} + +func (fs *unionFS) Rename(src string, dst string, context *fuse.Context) (code fuse.Status) { + srcResult := fs.getBranch(src) + code = srcResult.code + if code.Ok() { + code = srcResult.code + } + + if srcResult.attr.IsDir() { + return fs.renameDirectory(srcResult, src, dst, context) + } + + if code.Ok() && srcResult.branch > 0 { + code = fs.Promote(src, srcResult, context) + } + if code.Ok() { + code = fs.promoteDirsTo(dst) + } + if code.Ok() { + code = fs.fileSystems[0].Rename(src, dst, context) + } + + if code.Ok() { + fs.removeDeletion(dst) + // Rename is racy; avoid racing with unionFsFile.Release(). + fs.branchCache.DropEntry(dst) + + srcResult := fs.branchCache.GetFresh(src) + if srcResult.(branchResult).branch > 0 { + code = fs.putDeletion(src) + } + } + return code +} + +func (fs *unionFS) DropBranchCache(names []string) { + fs.branchCache.DropAll(names) +} + +func (fs *unionFS) DropDeletionCache() { + fs.deletionCache.DropCache() +} + +func (fs *unionFS) DropSubFsCaches() { + for _, fs := range fs.fileSystems { + a, code := fs.GetAttr(_DROP_CACHE, nil) + if code.Ok() && a.IsRegular() { + f, _ := fs.Open(_DROP_CACHE, uint32(os.O_WRONLY), nil) + if f != nil { + f.Flush() + f.Release() + } + } + } +} + +func (fs *unionFS) Open(name string, flags uint32, context *fuse.Context) (fuseFile nodefs.File, status fuse.Status) { + if name == _DROP_CACHE { + if flags&fuse.O_ANYWRITE != 0 { + log.Println("Forced cache drop on", fs) + fs.DropBranchCache(nil) + fs.DropDeletionCache() + fs.DropSubFsCaches() + fs.nodeFs.ForgetClientInodes() + } + return nodefs.NewDevNullFile(), fuse.OK + } + r := fs.getBranch(name) + if r.branch < 0 { + // This should not happen, as a GetAttr() should have + // already verified existence. + log.Println("UnionFs: open of non-existent file:", name) + return nil, fuse.ENOENT + } + if flags&fuse.O_ANYWRITE != 0 && r.branch > 0 { + code := fs.Promote(name, r, context) + if code != fuse.OK { + return nil, code + } + r.branch = 0 + now := time.Now() + r.attr.SetTimes(nil, &now, nil) + fs.setBranch(name, r) + } + fuseFile, status = fs.fileSystems[r.branch].Open(name, uint32(flags), context) + if fuseFile != nil { + fuseFile = fs.newUnionFsFile(fuseFile, r.branch) + } + return fuseFile, status +} + +func (fs *unionFS) String() string { + names := []string{} + for _, fs := range fs.fileSystems { + names = append(names, fs.String()) + } + return fmt.Sprintf("UnionFs(%v)", names) +} + +func (fs *unionFS) StatFs(name string) *fuse.StatfsOut { + return fs.fileSystems[0].StatFs("") +} + +type unionFsFile struct { + nodefs.File + ufs *unionFS + node *nodefs.Inode + layer int +} + +func (fs *unionFsFile) String() string { + return fmt.Sprintf("unionFsFile(%s)", fs.File.String()) +} + +func (fs *unionFS) newUnionFsFile(f nodefs.File, branch int) *unionFsFile { + return &unionFsFile{ + File: f, + ufs: fs, + layer: branch, + } +} + +func (fs *unionFsFile) InnerFile() (file nodefs.File) { + return fs.File +} + +// We can't hook on Release. Release has no response, so it is not +// ordered wrt any following calls. +func (fs *unionFsFile) Flush() (code fuse.Status) { + code = fs.File.Flush() + path := fs.ufs.nodeFs.Path(fs.node) + fs.ufs.branchCache.GetFresh(path) + return code +} + +func (fs *unionFsFile) SetInode(node *nodefs.Inode) { + fs.node = node +} + +func (fs *unionFsFile) GetAttr(out *fuse.Attr) fuse.Status { + code := fs.File.GetAttr(out) + if code.Ok() { + out.Mode |= 0200 + } + return code +} diff --git a/vendor/github.com/hanwen/go-fuse/unionfs/unionfs_xattr_test_darwin.go b/vendor/github.com/hanwen/go-fuse/unionfs/unionfs_xattr_test_darwin.go new file mode 100644 index 0000000..e67133a --- /dev/null +++ b/vendor/github.com/hanwen/go-fuse/unionfs/unionfs_xattr_test_darwin.go @@ -0,0 +1,37 @@ +// Copyright 2016 the Go-FUSE Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unionfs + +import ( + "syscall" + "unsafe" +) + +// Darwin doesn't have support for syscall.Getxattr() so we pull it into its own file and implement it by hand on Darwin. +func Getxattr(path string, attr string, dest []byte) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(dest) > 0 { + _p2 = unsafe.Pointer(&dest[0]) + } else { + var _zero uintptr + _p2 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall6(syscall.SYS_GETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(dest)), 0, 0) + sz = int(r0) + if e1 != 0 { + err = e1 + } + return +} diff --git a/vendor/github.com/hanwen/go-fuse/unionfs/unionfs_xattr_test_linux.go b/vendor/github.com/hanwen/go-fuse/unionfs/unionfs_xattr_test_linux.go new file mode 100644 index 0000000..6b2568d --- /dev/null +++ b/vendor/github.com/hanwen/go-fuse/unionfs/unionfs_xattr_test_linux.go @@ -0,0 +1,14 @@ +// Copyright 2016 the Go-FUSE Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unionfs + +import ( + "syscall" +) + +// Darwin doesn't have support for syscall.Getxattr() so we pull it into its own file and implement it by hand on Darwin. +func Getxattr(path string, attr string, dest []byte) (sz int, err error) { + return syscall.Getxattr(path, attr, dest) +}