From 357478d1d15b6360e01adfee4a3476fc38e0e2a7 Mon Sep 17 00:00:00 2001 From: Andrew Dunham Date: Wed, 29 Apr 2015 13:26:31 -0700 Subject: [PATCH] Don't create a util package - move the functionality to 'internal' --- cmd/cayley/cayley.go | 10 ++--- integration/integration_test.go | 8 ++-- internal/load.go | 82 ++++++++++++++++++++++++++++++++++++++++ util/load.go | 83 ----------------------------------------- 4 files changed, 91 insertions(+), 92 deletions(-) create mode 100644 internal/load.go delete mode 100644 util/load.go diff --git a/cmd/cayley/cayley.go b/cmd/cayley/cayley.go index f5f8b6e..77c6cde 100644 --- a/cmd/cayley/cayley.go +++ b/cmd/cayley/cayley.go @@ -29,7 +29,7 @@ import ( "github.com/google/cayley/db" "github.com/google/cayley/graph" "github.com/google/cayley/http" - "github.com/google/cayley/util" + "github.com/google/cayley/internal" // Load all supported backends. _ "github.com/google/cayley/graph/bolt" @@ -185,7 +185,7 @@ func main() { if err != nil { break } - err = util.Load(handle.QuadWriter, cfg, *quadFile, *quadType) + err = internal.Load(handle.QuadWriter, cfg, *quadFile, *quadType) if err != nil { break } @@ -197,7 +197,7 @@ func main() { if err != nil { break } - err = util.Load(handle.QuadWriter, cfg, *quadFile, *quadType) + err = internal.Load(handle.QuadWriter, cfg, *quadFile, *quadType) if err != nil { break } @@ -210,7 +210,7 @@ func main() { break } if !graph.IsPersistent(cfg.DatabaseType) { - err = util.Load(handle.QuadWriter, cfg, "", *quadType) + err = internal.Load(handle.QuadWriter, cfg, "", *quadType) if err != nil { break } @@ -226,7 +226,7 @@ func main() { break } if !graph.IsPersistent(cfg.DatabaseType) { - err = util.Load(handle.QuadWriter, cfg, "", *quadType) + err = internal.Load(handle.QuadWriter, cfg, "", *quadType) if err != nil { break } diff --git a/integration/integration_test.go b/integration/integration_test.go index 1015a5f..c6a39ed 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -28,9 +28,9 @@ import ( "github.com/google/cayley/config" "github.com/google/cayley/db" "github.com/google/cayley/graph" + "github.com/google/cayley/internal" "github.com/google/cayley/quad" "github.com/google/cayley/query/gremlin" - "github.com/google/cayley/util" // Load all supported backends. _ "github.com/google/cayley/graph/bolt" @@ -460,7 +460,7 @@ func prepare(t testing.TB) { } if needsLoad { - err = util.Load(handle.QuadWriter, cfg, "../data/30kmoviedata.nq.gz", "cquad") + err = internal.Load(handle.QuadWriter, cfg, "../data/30kmoviedata.nq.gz", "cquad") if err != nil { t.Fatalf("Failed to load %q: %v", cfg.DatabasePath, err) } @@ -477,7 +477,7 @@ func deletePrepare(t testing.TB) { if err != nil { t.Fatalf("Failed to remove %q: %v", cfg.DatabasePath, err) } - err = util.Load(handle.QuadWriter, cfg, "", "cquad") + err = internal.Load(handle.QuadWriter, cfg, "", "cquad") if err != nil { t.Fatalf("Failed to load %q: %v", cfg.DatabasePath, err) } @@ -486,7 +486,7 @@ func deletePrepare(t testing.TB) { } func removeAll(qw graph.QuadWriter, cfg *config.Config, path, typ string) error { - return util.DecompressAndLoad(qw, cfg, path, typ, remove) + return internal.DecompressAndLoad(qw, cfg, path, typ, remove) } func remove(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error { diff --git a/internal/load.go b/internal/load.go new file mode 100644 index 0000000..57b46da --- /dev/null +++ b/internal/load.go @@ -0,0 +1,82 @@ +package internal + +import ( + "fmt" + "io" + client "net/http" + "net/url" + "os" + "path/filepath" + + "github.com/google/cayley/config" + "github.com/google/cayley/db" + "github.com/google/cayley/graph" + "github.com/google/cayley/quad" + "github.com/google/cayley/quad/cquads" + "github.com/google/cayley/quad/nquads" +) + +// Load loads a graph from the given path and write it to qw. See +// DecompressAndLoad for more information. +func Load(qw graph.QuadWriter, cfg *config.Config, path, typ string) error { + return DecompressAndLoad(qw, cfg, path, typ, db.Load) +} + +// DecompressAndLoad will load or fetch a graph from the given path, decompress +// it, and then call the given load function to process the decompressed graph. +// If no loadFn is provided, db.Load is called. +func DecompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string, loadFn func(graph.QuadWriter, *config.Config, quad.Unmarshaler) error) error { + var r io.Reader + + if path == "" { + path = cfg.DatabasePath + } + if path == "" { + return nil + } + u, err := url.Parse(path) + if err != nil || u.Scheme == "file" || u.Scheme == "" { + // Don't alter relative URL path or non-URL path parameter. + if u.Scheme != "" && err == nil { + // Recovery heuristic for mistyping "file://path/to/file". + path = filepath.Join(u.Host, u.Path) + } + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("could not open file %q: %v", path, err) + } + defer f.Close() + r = f + } else { + res, err := client.Get(path) + if err != nil { + return fmt.Errorf("could not get resource <%s>: %v", u, err) + } + defer res.Body.Close() + r = res.Body + } + + r, err = Decompressor(r) + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + var dec quad.Unmarshaler + switch typ { + case "cquad": + dec = cquads.NewDecoder(r) + case "nquad": + dec = nquads.NewDecoder(r) + default: + return fmt.Errorf("unknown quad format %q", typ) + } + + if loadFn != nil { + return loadFn(qw, cfg, dec) + } + + return db.Load(qw, cfg, dec) +} diff --git a/util/load.go b/util/load.go deleted file mode 100644 index 07f9bf9..0000000 --- a/util/load.go +++ /dev/null @@ -1,83 +0,0 @@ -package util - -import ( - "fmt" - "io" - client "net/http" - "net/url" - "os" - "path/filepath" - - "github.com/google/cayley/config" - "github.com/google/cayley/db" - "github.com/google/cayley/graph" - "github.com/google/cayley/internal" - "github.com/google/cayley/quad" - "github.com/google/cayley/quad/cquads" - "github.com/google/cayley/quad/nquads" -) - -// Load loads a graph from the given path and write it to qw. See -// DecompressAndLoad for more information. -func Load(qw graph.QuadWriter, cfg *config.Config, path, typ string) error { - return DecompressAndLoad(qw, cfg, path, typ, db.Load) -} - -// DecompressAndLoad will load or fetch a graph from the given path, decompress -// it, and then call the given load function to process the decompressed graph. -// If no loadFn is provided, db.Load is called. -func DecompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string, loadFn func(graph.QuadWriter, *config.Config, quad.Unmarshaler) error) error { - var r io.Reader - - if path == "" { - path = cfg.DatabasePath - } - if path == "" { - return nil - } - u, err := url.Parse(path) - if err != nil || u.Scheme == "file" || u.Scheme == "" { - // Don't alter relative URL path or non-URL path parameter. - if u.Scheme != "" && err == nil { - // Recovery heuristic for mistyping "file://path/to/file". - path = filepath.Join(u.Host, u.Path) - } - f, err := os.Open(path) - if err != nil { - return fmt.Errorf("could not open file %q: %v", path, err) - } - defer f.Close() - r = f - } else { - res, err := client.Get(path) - if err != nil { - return fmt.Errorf("could not get resource <%s>: %v", u, err) - } - defer res.Body.Close() - r = res.Body - } - - r, err = internal.Decompressor(r) - if err != nil { - if err == io.EOF { - return nil - } - return err - } - - var dec quad.Unmarshaler - switch typ { - case "cquad": - dec = cquads.NewDecoder(r) - case "nquad": - dec = nquads.NewDecoder(r) - default: - return fmt.Errorf("unknown quad format %q", typ) - } - - if loadFn != nil { - return loadFn(qw, cfg, dec) - } - - return db.Load(qw, cfg, dec) -}