barak/cayley
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #245 from andrew-d/andrew-refactor-packages

Browse source 
Refactor packages
This commit is contained in:
Barak Michener 2015-04-30 22:57:58 -04:00
commit ca8c1912de
6 changed files with 989 additions and 961 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -4,3 +4,6 @@ main
*.peg.go
cayley.cfg
.cayley_history
# The built binary
/cayley

75
cayley.go → cmd/cayley/cayley.go
View file

@ -19,11 +19,7 @@ package main
import (
"flag"
"fmt"
"io"
client "net/http"
"net/url"
"os"
"path/filepath"
"runtime"
"time"
@ -34,9 +30,6 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/http"
"github.com/google/cayley/internal"
"github.com/google/cayley/quad"
"github.com/google/cayley/quad/cquads"
"github.com/google/cayley/quad/nquads"
// Load all supported backends.
_ "github.com/google/cayley/graph/bolt"
@ -192,7 +185,7 @@ func main() {
if err != nil {
break
}
err = load(handle.QuadWriter, cfg, *quadFile, *quadType)
err = internal.Load(handle.QuadWriter, cfg, *quadFile, *quadType)
if err != nil {
break
}
@ -204,7 +197,7 @@ func main() {
if err != nil {
break
}
err = load(handle.QuadWriter, cfg, *quadFile, *quadType)
err = internal.Load(handle.QuadWriter, cfg, *quadFile, *quadType)
if err != nil {
break
}
@ -217,7 +210,7 @@ func main() {
break
}
if !graph.IsPersistent(cfg.DatabaseType) {
err = load(handle.QuadWriter, cfg, "", *quadType)
err = internal.Load(handle.QuadWriter, cfg, "", *quadType)
if err != nil {
break
}
@ -233,7 +226,7 @@ func main() {
break
}
if !graph.IsPersistent(cfg.DatabaseType) {
err = load(handle.QuadWriter, cfg, "", *quadType)
err = internal.Load(handle.QuadWriter, cfg, "", *quadType)
if err != nil {
break
}
@ -251,63 +244,3 @@ func main() {
glog.Errorln(err)
}
}
func load(qw graph.QuadWriter, cfg *config.Config, path, typ string) error {
return decompressAndLoad(qw, cfg, path, typ, db.Load)
}
func decompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string, loadFn func(graph.QuadWriter, *config.Config, quad.Unmarshaler) error) error {
var r io.Reader
if path == "" {
path = cfg.DatabasePath
}
if path == "" {
return nil
}
u, err := url.Parse(path)
if err != nil || u.Scheme == "file" || u.Scheme == "" {
// Don't alter relative URL path or non-URL path parameter.
if u.Scheme != "" && err == nil {
// Recovery heuristic for mistyping "file://path/to/file".
path = filepath.Join(u.Host, u.Path)
}
f, err := os.Open(path)
if err != nil {
return fmt.Errorf("could not open file %q: %v", path, err)
}
defer f.Close()
r = f
} else {
res, err := client.Get(path)
if err != nil {
return fmt.Errorf("could not get resource <%s>: %v", u, err)
}
defer res.Body.Close()
r = res.Body
}
r, err = internal.Decompressor(r)
if err != nil {
if err == io.EOF {
return nil
}
return err
}
var dec quad.Unmarshaler
switch typ {
case "cquad":
dec = cquads.NewDecoder(r)
case "nquad":
dec = nquads.NewDecoder(r)
default:
return fmt.Errorf("unknown quad format %q", typ)
}
if loadFn != nil {
return loadFn(qw, cfg, dec)
}
return db.Load(qw, cfg, dec)
}

20
cayley_test.go → integration/integration_test.go
View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package integration
import (
"flag"
@ -28,8 +28,18 @@ import (
"github.com/google/cayley/config"
"github.com/google/cayley/db"
"github.com/google/cayley/graph"
"github.com/google/cayley/internal"
"github.com/google/cayley/quad"
"github.com/google/cayley/query/gremlin"
// Load all supported backends.
_ "github.com/google/cayley/graph/bolt"
_ "github.com/google/cayley/graph/leveldb"
_ "github.com/google/cayley/graph/memstore"
_ "github.com/google/cayley/graph/mongo"
// Load writer registry
_ "github.com/google/cayley/writer"
)
var backend = flag.String("backend", "memstore", "Which backend to test. Loads test data to /tmp if not present.")
@ -415,7 +425,7 @@ func prepare(t testing.TB) {
cfg.DatabaseType = *backend
switch *backend {
case "memstore":
cfg.DatabasePath = "data/30kmoviedata.nq.gz"
cfg.DatabasePath = "../data/30kmoviedata.nq.gz"
case "leveldb", "bolt":
cfg.DatabasePath = "/tmp/cayley_test_" + *backend
cfg.DatabaseOptions = map[string]interface{}{
@ -450,7 +460,7 @@ func prepare(t testing.TB) {
}
if needsLoad {
err = load(handle.QuadWriter, cfg, "data/30kmoviedata.nq.gz", "cquad")
err = internal.Load(handle.QuadWriter, cfg, "../data/30kmoviedata.nq.gz", "cquad")
if err != nil {
t.Fatalf("Failed to load %q: %v", cfg.DatabasePath, err)
}
@ -467,7 +477,7 @@ func deletePrepare(t testing.TB) {
if err != nil {
t.Fatalf("Failed to remove %q: %v", cfg.DatabasePath, err)
}
err = load(handle.QuadWriter, cfg, "", "cquad")
err = internal.Load(handle.QuadWriter, cfg, "", "cquad")
if err != nil {
t.Fatalf("Failed to load %q: %v", cfg.DatabasePath, err)
}
@ -476,7 +486,7 @@ func deletePrepare(t testing.TB) {
}
func removeAll(qw graph.QuadWriter, cfg *config.Config, path, typ string) error {
return decompressAndLoad(qw, cfg, path, typ, remove)
return internal.DecompressAndLoad(qw, cfg, path, typ, remove)
}
func remove(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error {

82
internal/load.go Normal file
View file

@ -0,0 +1,82 @@
package internal
import (
"fmt"
"io"
client "net/http"
"net/url"
"os"
"path/filepath"
"github.com/google/cayley/config"
"github.com/google/cayley/db"
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
"github.com/google/cayley/quad/cquads"
"github.com/google/cayley/quad/nquads"
)
// Load loads a graph from the given path and write it to qw. See
// DecompressAndLoad for more information.
func Load(qw graph.QuadWriter, cfg *config.Config, path, typ string) error {
return DecompressAndLoad(qw, cfg, path, typ, db.Load)
}
// DecompressAndLoad will load or fetch a graph from the given path, decompress
// it, and then call the given load function to process the decompressed graph.
// If no loadFn is provided, db.Load is called.
func DecompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string, loadFn func(graph.QuadWriter, *config.Config, quad.Unmarshaler) error) error {
var r io.Reader
if path == "" {
path = cfg.DatabasePath
}
if path == "" {
return nil
}
u, err := url.Parse(path)
if err != nil || u.Scheme == "file" || u.Scheme == "" {
// Don't alter relative URL path or non-URL path parameter.
if u.Scheme != "" && err == nil {
// Recovery heuristic for mistyping "file://path/to/file".
path = filepath.Join(u.Host, u.Path)
}
f, err := os.Open(path)
if err != nil {
return fmt.Errorf("could not open file %q: %v", path, err)
}
defer f.Close()
r = f
} else {
res, err := client.Get(path)
if err != nil {
return fmt.Errorf("could not get resource <%s>: %v", u, err)
}
defer res.Body.Close()
r = res.Body
}
r, err = Decompressor(r)
if err != nil {
if err == io.EOF {
return nil
}
return err
}
var dec quad.Unmarshaler
switch typ {
case "cquad":
dec = cquads.NewDecoder(r)
case "nquad":
dec = nquads.NewDecoder(r)
default:
return fmt.Errorf("unknown quad format %q", typ)
}
if loadFn != nil {
return loadFn(qw, cfg, dec)
}
return db.Load(qw, cfg, dec)
}