Refactor location of code
1. The 'cayley' command now lives under cmd/ 2. Integration tests live under integration/ 3. Some utility functions used in both the integration tests and the 'cayley' command were split to their own 'util' package.
This commit is contained in:
Andrew Dunham
parent
7f1607693a
commit
e9e768ce36
5 changed files with 987 additions and 961 deletions
|
|
@ -19,11 +19,7 @@ package main
|
|||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
client "net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
|
|
@ -33,10 +29,7 @@ import (
|
|||
"github.com/google/cayley/db"
|
||||
"github.com/google/cayley/graph"
|
||||
"github.com/google/cayley/http"
|
||||
"github.com/google/cayley/internal"
|
||||
"github.com/google/cayley/quad"
|
||||
"github.com/google/cayley/quad/cquads"
|
||||
"github.com/google/cayley/quad/nquads"
|
||||
"github.com/google/cayley/util"
|
||||
|
||||
// Load all supported backends.
|
||||
_ "github.com/google/cayley/graph/bolt"
|
||||
|
|
@ -192,7 +185,7 @@ func main() {
|
|||
if err != nil {
|
||||
break
|
||||
}
|
||||
err = load(handle.QuadWriter, cfg, *quadFile, *quadType)
|
||||
err = util.Load(handle.QuadWriter, cfg, *quadFile, *quadType)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
|
@ -204,7 +197,7 @@ func main() {
|
|||
if err != nil {
|
||||
break
|
||||
}
|
||||
err = load(handle.QuadWriter, cfg, *quadFile, *quadType)
|
||||
err = util.Load(handle.QuadWriter, cfg, *quadFile, *quadType)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
|
@ -217,7 +210,7 @@ func main() {
|
|||
break
|
||||
}
|
||||
if !graph.IsPersistent(cfg.DatabaseType) {
|
||||
err = load(handle.QuadWriter, cfg, "", *quadType)
|
||||
err = util.Load(handle.QuadWriter, cfg, "", *quadType)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
|
@ -233,7 +226,7 @@ func main() {
|
|||
break
|
||||
}
|
||||
if !graph.IsPersistent(cfg.DatabaseType) {
|
||||
err = load(handle.QuadWriter, cfg, "", *quadType)
|
||||
err = util.Load(handle.QuadWriter, cfg, "", *quadType)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
|
@ -251,63 +244,3 @@ func main() {
|
|||
glog.Errorln(err)
|
||||
}
|
||||
}
|
||||
|
||||
func load(qw graph.QuadWriter, cfg *config.Config, path, typ string) error {
|
||||
return decompressAndLoad(qw, cfg, path, typ, db.Load)
|
||||
}
|
||||
|
||||
func decompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string, loadFn func(graph.QuadWriter, *config.Config, quad.Unmarshaler) error) error {
|
||||
var r io.Reader
|
||||
|
||||
if path == "" {
|
||||
path = cfg.DatabasePath
|
||||
}
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
u, err := url.Parse(path)
|
||||
if err != nil || u.Scheme == "file" || u.Scheme == "" {
|
||||
// Don't alter relative URL path or non-URL path parameter.
|
||||
if u.Scheme != "" && err == nil {
|
||||
// Recovery heuristic for mistyping "file://path/to/file".
|
||||
path = filepath.Join(u.Host, u.Path)
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open file %q: %v", path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
r = f
|
||||
} else {
|
||||
res, err := client.Get(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get resource <%s>: %v", u, err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
r = res.Body
|
||||
}
|
||||
|
||||
r, err = internal.Decompressor(r)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var dec quad.Unmarshaler
|
||||
switch typ {
|
||||
case "cquad":
|
||||
dec = cquads.NewDecoder(r)
|
||||
case "nquad":
|
||||
dec = nquads.NewDecoder(r)
|
||||
default:
|
||||
return fmt.Errorf("unknown quad format %q", typ)
|
||||
}
|
||||
|
||||
if loadFn != nil {
|
||||
return loadFn(qw, cfg, dec)
|
||||
}
|
||||
|
||||
return db.Load(qw, cfg, dec)
|
||||
}
|
||||
|
|
@ -12,7 +12,7 @@
|
|||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
package integration
|
||||
|
||||
import (
|
||||
"flag"
|
||||
|
|
@ -30,6 +30,16 @@ import (
|
|||
"github.com/google/cayley/graph"
|
||||
"github.com/google/cayley/quad"
|
||||
"github.com/google/cayley/query/gremlin"
|
||||
"github.com/google/cayley/util"
|
||||
|
||||
// Load all supported backends.
|
||||
_ "github.com/google/cayley/graph/bolt"
|
||||
_ "github.com/google/cayley/graph/leveldb"
|
||||
_ "github.com/google/cayley/graph/memstore"
|
||||
_ "github.com/google/cayley/graph/mongo"
|
||||
|
||||
// Load writer registry
|
||||
_ "github.com/google/cayley/writer"
|
||||
)
|
||||
|
||||
var backend = flag.String("backend", "memstore", "Which backend to test. Loads test data to /tmp if not present.")
|
||||
|
|
@ -415,7 +425,7 @@ func prepare(t testing.TB) {
|
|||
cfg.DatabaseType = *backend
|
||||
switch *backend {
|
||||
case "memstore":
|
||||
cfg.DatabasePath = "data/30kmoviedata.nq.gz"
|
||||
cfg.DatabasePath = "../data/30kmoviedata.nq.gz"
|
||||
case "leveldb", "bolt":
|
||||
cfg.DatabasePath = "/tmp/cayley_test_" + *backend
|
||||
cfg.DatabaseOptions = map[string]interface{}{
|
||||
|
|
@ -450,7 +460,7 @@ func prepare(t testing.TB) {
|
|||
}
|
||||
|
||||
if needsLoad {
|
||||
err = load(handle.QuadWriter, cfg, "data/30kmoviedata.nq.gz", "cquad")
|
||||
err = util.Load(handle.QuadWriter, cfg, "../data/30kmoviedata.nq.gz", "cquad")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to load %q: %v", cfg.DatabasePath, err)
|
||||
}
|
||||
|
|
@ -467,7 +477,7 @@ func deletePrepare(t testing.TB) {
|
|||
if err != nil {
|
||||
t.Fatalf("Failed to remove %q: %v", cfg.DatabasePath, err)
|
||||
}
|
||||
err = load(handle.QuadWriter, cfg, "", "cquad")
|
||||
err = util.Load(handle.QuadWriter, cfg, "", "cquad")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to load %q: %v", cfg.DatabasePath, err)
|
||||
}
|
||||
|
|
@ -476,7 +486,7 @@ func deletePrepare(t testing.TB) {
|
|||
}
|
||||
|
||||
func removeAll(qw graph.QuadWriter, cfg *config.Config, path, typ string) error {
|
||||
return decompressAndLoad(qw, cfg, path, typ, remove)
|
||||
return util.DecompressAndLoad(qw, cfg, path, typ, remove)
|
||||
}
|
||||
|
||||
func remove(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error {
|
||||
83
util/load.go
Normal file
83
util/load.go
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
client "net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/google/cayley/config"
|
||||
"github.com/google/cayley/db"
|
||||
"github.com/google/cayley/graph"
|
||||
"github.com/google/cayley/internal"
|
||||
"github.com/google/cayley/quad"
|
||||
"github.com/google/cayley/quad/cquads"
|
||||
"github.com/google/cayley/quad/nquads"
|
||||
)
|
||||
|
||||
// Load loads a graph from the given path and write it to qw. See
|
||||
// DecompressAndLoad for more information.
|
||||
func Load(qw graph.QuadWriter, cfg *config.Config, path, typ string) error {
|
||||
return DecompressAndLoad(qw, cfg, path, typ, db.Load)
|
||||
}
|
||||
|
||||
// DecompressAndLoad will load or fetch a graph from the given path, decompress
|
||||
// it, and then call the given load function to process the decompressed graph.
|
||||
// If no loadFn is provided, db.Load is called.
|
||||
func DecompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string, loadFn func(graph.QuadWriter, *config.Config, quad.Unmarshaler) error) error {
|
||||
var r io.Reader
|
||||
|
||||
if path == "" {
|
||||
path = cfg.DatabasePath
|
||||
}
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
u, err := url.Parse(path)
|
||||
if err != nil || u.Scheme == "file" || u.Scheme == "" {
|
||||
// Don't alter relative URL path or non-URL path parameter.
|
||||
if u.Scheme != "" && err == nil {
|
||||
// Recovery heuristic for mistyping "file://path/to/file".
|
||||
path = filepath.Join(u.Host, u.Path)
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open file %q: %v", path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
r = f
|
||||
} else {
|
||||
res, err := client.Get(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get resource <%s>: %v", u, err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
r = res.Body
|
||||
}
|
||||
|
||||
r, err = internal.Decompressor(r)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var dec quad.Unmarshaler
|
||||
switch typ {
|
||||
case "cquad":
|
||||
dec = cquads.NewDecoder(r)
|
||||
case "nquad":
|
||||
dec = nquads.NewDecoder(r)
|
||||
default:
|
||||
return fmt.Errorf("unknown quad format %q", typ)
|
||||
}
|
||||
|
||||
if loadFn != nil {
|
||||
return loadFn(qw, cfg, dec)
|
||||
}
|
||||
|
||||
return db.Load(qw, cfg, dec)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue