diff --git a/cayley.go b/cmd/cayley/cayley.go similarity index 75% rename from cayley.go rename to cmd/cayley/cayley.go index 41b3244..f5f8b6e 100644 --- a/cayley.go +++ b/cmd/cayley/cayley.go @@ -19,11 +19,7 @@ package main import ( "flag" "fmt" - "io" - client "net/http" - "net/url" "os" - "path/filepath" "runtime" "time" @@ -33,10 +29,7 @@ import ( "github.com/google/cayley/db" "github.com/google/cayley/graph" "github.com/google/cayley/http" - "github.com/google/cayley/internal" - "github.com/google/cayley/quad" - "github.com/google/cayley/quad/cquads" - "github.com/google/cayley/quad/nquads" + "github.com/google/cayley/util" // Load all supported backends. _ "github.com/google/cayley/graph/bolt" @@ -192,7 +185,7 @@ func main() { if err != nil { break } - err = load(handle.QuadWriter, cfg, *quadFile, *quadType) + err = util.Load(handle.QuadWriter, cfg, *quadFile, *quadType) if err != nil { break } @@ -204,7 +197,7 @@ func main() { if err != nil { break } - err = load(handle.QuadWriter, cfg, *quadFile, *quadType) + err = util.Load(handle.QuadWriter, cfg, *quadFile, *quadType) if err != nil { break } @@ -217,7 +210,7 @@ func main() { break } if !graph.IsPersistent(cfg.DatabaseType) { - err = load(handle.QuadWriter, cfg, "", *quadType) + err = util.Load(handle.QuadWriter, cfg, "", *quadType) if err != nil { break } @@ -233,7 +226,7 @@ func main() { break } if !graph.IsPersistent(cfg.DatabaseType) { - err = load(handle.QuadWriter, cfg, "", *quadType) + err = util.Load(handle.QuadWriter, cfg, "", *quadType) if err != nil { break } @@ -251,63 +244,3 @@ func main() { glog.Errorln(err) } } - -func load(qw graph.QuadWriter, cfg *config.Config, path, typ string) error { - return decompressAndLoad(qw, cfg, path, typ, db.Load) -} - -func decompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string, loadFn func(graph.QuadWriter, *config.Config, quad.Unmarshaler) error) error { - var r io.Reader - - if path == "" { - path = cfg.DatabasePath - } - if path == "" { - return nil - } - u, err := url.Parse(path) - if err != nil || u.Scheme == "file" || u.Scheme == "" { - // Don't alter relative URL path or non-URL path parameter. - if u.Scheme != "" && err == nil { - // Recovery heuristic for mistyping "file://path/to/file". - path = filepath.Join(u.Host, u.Path) - } - f, err := os.Open(path) - if err != nil { - return fmt.Errorf("could not open file %q: %v", path, err) - } - defer f.Close() - r = f - } else { - res, err := client.Get(path) - if err != nil { - return fmt.Errorf("could not get resource <%s>: %v", u, err) - } - defer res.Body.Close() - r = res.Body - } - - r, err = internal.Decompressor(r) - if err != nil { - if err == io.EOF { - return nil - } - return err - } - - var dec quad.Unmarshaler - switch typ { - case "cquad": - dec = cquads.NewDecoder(r) - case "nquad": - dec = nquads.NewDecoder(r) - default: - return fmt.Errorf("unknown quad format %q", typ) - } - - if loadFn != nil { - return loadFn(qw, cfg, dec) - } - - return db.Load(qw, cfg, dec) -} diff --git a/cayley_test.go b/integration/integration_test.go similarity index 98% rename from cayley_test.go rename to integration/integration_test.go index 95a2c84..1015a5f 100644 --- a/cayley_test.go +++ b/integration/integration_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package integration import ( "flag" @@ -30,6 +30,16 @@ import ( "github.com/google/cayley/graph" "github.com/google/cayley/quad" "github.com/google/cayley/query/gremlin" + "github.com/google/cayley/util" + + // Load all supported backends. + _ "github.com/google/cayley/graph/bolt" + _ "github.com/google/cayley/graph/leveldb" + _ "github.com/google/cayley/graph/memstore" + _ "github.com/google/cayley/graph/mongo" + + // Load writer registry + _ "github.com/google/cayley/writer" ) var backend = flag.String("backend", "memstore", "Which backend to test. Loads test data to /tmp if not present.") @@ -415,7 +425,7 @@ func prepare(t testing.TB) { cfg.DatabaseType = *backend switch *backend { case "memstore": - cfg.DatabasePath = "data/30kmoviedata.nq.gz" + cfg.DatabasePath = "../data/30kmoviedata.nq.gz" case "leveldb", "bolt": cfg.DatabasePath = "/tmp/cayley_test_" + *backend cfg.DatabaseOptions = map[string]interface{}{ @@ -450,7 +460,7 @@ func prepare(t testing.TB) { } if needsLoad { - err = load(handle.QuadWriter, cfg, "data/30kmoviedata.nq.gz", "cquad") + err = util.Load(handle.QuadWriter, cfg, "../data/30kmoviedata.nq.gz", "cquad") if err != nil { t.Fatalf("Failed to load %q: %v", cfg.DatabasePath, err) } @@ -467,7 +477,7 @@ func deletePrepare(t testing.TB) { if err != nil { t.Fatalf("Failed to remove %q: %v", cfg.DatabasePath, err) } - err = load(handle.QuadWriter, cfg, "", "cquad") + err = util.Load(handle.QuadWriter, cfg, "", "cquad") if err != nil { t.Fatalf("Failed to load %q: %v", cfg.DatabasePath, err) } @@ -476,7 +486,7 @@ func deletePrepare(t testing.TB) { } func removeAll(qw graph.QuadWriter, cfg *config.Config, path, typ string) error { - return decompressAndLoad(qw, cfg, path, typ, remove) + return util.DecompressAndLoad(qw, cfg, path, typ, remove) } func remove(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error { diff --git a/util/load.go b/util/load.go new file mode 100644 index 0000000..07f9bf9 --- /dev/null +++ b/util/load.go @@ -0,0 +1,83 @@ +package util + +import ( + "fmt" + "io" + client "net/http" + "net/url" + "os" + "path/filepath" + + "github.com/google/cayley/config" + "github.com/google/cayley/db" + "github.com/google/cayley/graph" + "github.com/google/cayley/internal" + "github.com/google/cayley/quad" + "github.com/google/cayley/quad/cquads" + "github.com/google/cayley/quad/nquads" +) + +// Load loads a graph from the given path and write it to qw. See +// DecompressAndLoad for more information. +func Load(qw graph.QuadWriter, cfg *config.Config, path, typ string) error { + return DecompressAndLoad(qw, cfg, path, typ, db.Load) +} + +// DecompressAndLoad will load or fetch a graph from the given path, decompress +// it, and then call the given load function to process the decompressed graph. +// If no loadFn is provided, db.Load is called. +func DecompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string, loadFn func(graph.QuadWriter, *config.Config, quad.Unmarshaler) error) error { + var r io.Reader + + if path == "" { + path = cfg.DatabasePath + } + if path == "" { + return nil + } + u, err := url.Parse(path) + if err != nil || u.Scheme == "file" || u.Scheme == "" { + // Don't alter relative URL path or non-URL path parameter. + if u.Scheme != "" && err == nil { + // Recovery heuristic for mistyping "file://path/to/file". + path = filepath.Join(u.Host, u.Path) + } + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("could not open file %q: %v", path, err) + } + defer f.Close() + r = f + } else { + res, err := client.Get(path) + if err != nil { + return fmt.Errorf("could not get resource <%s>: %v", u, err) + } + defer res.Body.Close() + r = res.Body + } + + r, err = internal.Decompressor(r) + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + var dec quad.Unmarshaler + switch typ { + case "cquad": + dec = cquads.NewDecoder(r) + case "nquad": + dec = nquads.NewDecoder(r) + default: + return fmt.Errorf("unknown quad format %q", typ) + } + + if loadFn != nil { + return loadFn(qw, cfg, dec) + } + + return db.Load(qw, cfg, dec) +}