Reorganize to go get will work

This makes almost no changes to source, but touches every almost file.

Also fixes error in gremlin test code.

Makefile

@@ -1,29 +0,0 @@
-# Copyright 2014 The Cayley Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-default:	build
-
-build:
-	./make.sh build
-
-deps:
-	./make.sh deps
-
-test:
-	ls ./src | grep -v "\." | sed 's/\///g' | xargs go test -cover
-
-convey:
-	./bin/goconvey --depth=2
-
-	

activate.sh

@@ -1,9 +0,0 @@
-# Absolute path this script is in. /home/user/bin
-cd "`dirname '${BASH_SOURCE:-$0}'`"
-SCRIPTPATH="`pwd`"
-echo $dir
-cd - > /dev/null
-
-export GOPATH=$SCRIPTPATH
-#export GOOS="linux"
-#export GOARCH="amd64"

appengine/cayley-appengine.go

@@ -15,22 +15,24 @@
 package cayleyappengine
 
 import (
-	"cayley_config"
-	"cayley_http"
-	"github.com/barakmich/glog"
-	"graph"
-	"graph_memstore"
-	"nquads"
 	"os"
+
+	"github.com/barakmich/glog"
+
+	cfg "github.com/google/cayley/config"
+	"github.com/google/cayley/graph"
+	"github.com/google/cayley/graph/memstore"
+	"github.com/google/cayley/http"
+	"github.com/google/cayley/nquads"
 )
 
 func init() {
 	glog.SetToStderr(true)
-	config := cayley_config.ParseConfigFromFile("cayley_appengine.cfg")
+	config := cfg.ParseConfigFromFile("cayley_appengine.cfg")
-	ts := graph_memstore.NewMemTripleStore()
+	ts := memstore.NewMemTripleStore()
 	glog.Errorln(config)
 	LoadTriplesFromFileInto(ts, config.DatabasePath, config.LoadSize)
-	cayley_http.SetupRoutes(ts, config)
+	http.SetupRoutes(ts, config)
 }
 
 func ReadTriplesFromFile(c chan *graph.Triple, tripleFile string) {

graph/memstore/llrb-iterator.go

@@ -12,14 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_memstore
+package memstore
 
 import (
 	"fmt"
-	"github.com/petar/GoLLRB/llrb"
-	"graph"
 	"math"
 	"strings"
+
+	"github.com/petar/GoLLRB/llrb"
+
+	"github.com/google/cayley/graph"
 )
 
 type LlrbIterator struct {

graph/memstore/memstore-all-iterator.go

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_memstore
+package memstore
 
 import (
-	"graph"
+	"github.com/google/cayley/graph"
 )
 
 type MemstoreAllIterator struct {

graph/memstore/memtriplestore-iterator-optimize.go

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_memstore
+package memstore
 
 import (
-	"graph"
+	"github.com/google/cayley/graph"
 )
 
 func (ts *MemTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {

graph/memstore/memtriplestore.go

@@ -12,13 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_memstore
+package memstore
 
 import (
 	"fmt"
+
 	"github.com/barakmich/glog"
+	"github.com/google/cayley/graph"
+
 	"github.com/petar/GoLLRB/llrb"
-	"graph"
 )
 
 type TripleDirectionIndex struct {

graph/memstore/memtriplestore_test.go

@@ -12,13 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_memstore
+package memstore
 
 import (
-	. "github.com/smartystreets/goconvey/convey"
-	"graph"
 	"sort"
 	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+
+	"github.com/google/cayley/graph"
 )
 
 func TestMemstore(t *testing.T) {

graph/memstore/testing_memstore.go

@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_memstore
+package memstore
 
-import "graph"
+import "github.com/google/cayley/graph"
 
 //    +---+                        +---+
 //    | A |-------               ->| F |<--

graph/mongo/lru.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_mongo
+package mongo
 
 import (
 	"container/list"

graph/mongo/mongo-iterator.go

@@ -12,15 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_mongo
+package mongo
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/barakmich/glog"
-	"graph"
 	"labix.org/v2/mgo"
 	"labix.org/v2/mgo/bson"
-	"strings"
+
+	"github.com/google/cayley/graph"
 )
 
 type MongoIterator struct {

graph/mongo/mongo-triplestore-iterator-optimize.go

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_mongo
+package mongo
 
 import (
-	"graph"
+	"github.com/google/cayley/graph"
 )
 
 func (ts *MongoTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {

graph/mongo/mongo-triplestore.go

@@ -12,17 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_mongo
+package mongo
 
 import (
 	"crypto/sha1"
 	"encoding/hex"
-	"github.com/barakmich/glog"
-	"graph"
 	"hash"
+	"log"
+
 	"labix.org/v2/mgo"
 	"labix.org/v2/mgo/bson"
-	"log"
+
+	"github.com/barakmich/glog"
+	"github.com/google/cayley/graph"
 )
 
 const DefaultDBName = "cayley"

graph/sexp/parser.go

@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_sexp
+package sexp
 
 import (
 	"github.com/badgerodon/peg"
-	"graph"
+
+	"github.com/google/cayley/graph"
 )
 
 func BuildIteratorTreeForQuery(ts graph.TripleStore, query string) graph.Iterator {

graph/sexp/parser_test.go

@@ -12,13 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_sexp
+package sexp
 
 import (
-	. "github.com/smartystreets/goconvey/convey"
-	"graph"
-	"graph_memstore"
 	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+
+	"github.com/google/cayley/graph"
+	"github.com/google/cayley/graph/memstore"
 )
 
 func TestBadParse(t *testing.T) {
@@ -30,7 +32,7 @@ func TestBadParse(t *testing.T) {
 
 func TestParseSexpWithMemstore(t *testing.T) {
 	Convey("With a Memstore", t, func() {
-		ts := graph_memstore.NewMemTripleStore()
+		ts := memstore.NewMemTripleStore()
 
 		Convey("It should parse an empty query", func() {
 			it := BuildIteratorTreeForQuery(ts, "()")
@@ -62,7 +64,7 @@ func TestParseSexpWithMemstore(t *testing.T) {
 }
 
 func TestTreeConstraintParse(t *testing.T) {
-	ts := graph_memstore.NewMemTripleStore()
+	ts := memstore.NewMemTripleStore()
 	ts.AddTriple(graph.MakeTriple("i", "like", "food", ""))
 	ts.AddTriple(graph.MakeTriple("food", "is", "good", ""))
 	query := "(\"i\"\n" +
@@ -82,7 +84,7 @@ func TestTreeConstraintParse(t *testing.T) {
 }
 
 func TestTreeConstraintTagParse(t *testing.T) {
-	ts := graph_memstore.NewMemTripleStore()
+	ts := memstore.NewMemTripleStore()
 	ts.AddTriple(graph.MakeTriple("i", "like", "food", ""))
 	ts.AddTriple(graph.MakeTriple("food", "is", "good", ""))
 	query := "(\"i\"\n" +
@@ -102,7 +104,7 @@ func TestTreeConstraintTagParse(t *testing.T) {
 }
 
 func TestMultipleConstraintParse(t *testing.T) {
-	ts := graph_memstore.NewMemTripleStore()
+	ts := memstore.NewMemTripleStore()
 	ts.AddTriple(graph.MakeTriple("i", "like", "food", ""))
 	ts.AddTriple(graph.MakeTriple("i", "like", "beer", ""))
 	ts.AddTriple(graph.MakeTriple("you", "like", "beer", ""))

graph/sexp/sexp-session.go

@@ -12,15 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package graph_sexp
+package sexp
 
 // Defines a running session of the sexp query language.
 
 import (
 	"errors"
 	"fmt"
-	"graph"
 	"sort"
+
+	"github.com/google/cayley/graph"
 )
 
 type SexpSession struct {

gremlin/gremlin-build-iterator.go

@@ -15,10 +15,12 @@
 package gremlin
 
 import (
+	"strconv"
+
 	"github.com/barakmich/glog"
 	"github.com/robertkrimen/otto"
-	"graph"
+
-	"strconv"
+	"github.com/google/cayley/graph"
 )
 
 func getStrings(obj *otto.Object, field string) []string {

gremlin/gremlin-finals.go

@@ -17,7 +17,8 @@ package gremlin
 import (
 	"github.com/barakmich/glog"
 	"github.com/robertkrimen/otto"
-	"graph"
+
+	"github.com/google/cayley/graph"
 )
 
 const GremlinTopResultTag = "id"

gremlin/gremlin-functional_test.go

@@ -15,10 +15,12 @@
 package gremlin
 
 import (
-	. "github.com/smartystreets/goconvey/convey"
-	"graph_memstore"
 	"sort"
 	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+
+	"github.com/google/cayley/graph/memstore"
 )
 
 //    +---+                        +---+
@@ -34,8 +36,8 @@ import (
 //
 
 func buildTripleStore() *GremlinSession {
-	ts := graph_memstore.MakeTestingMemstore()
+	ts := memstore.MakeTestingMemstore()
-	return NewGremlinSession(ts, -1)
+	return NewGremlinSession(ts, -1, false)
 }
 
 func shouldBeUnordered(actual interface{}, expected ...interface{}) string {

gremlin/gremlin-session.go

@@ -17,10 +17,12 @@ package gremlin
 import (
 	"errors"
 	"fmt"
-	"github.com/robertkrimen/otto"
-	"graph"
 	"sort"
 	"time"
+
+	"github.com/robertkrimen/otto"
+
+	"github.com/google/cayley/graph"
 )
 
 type GremlinSession struct {

http/cayley-http-docs.go

@@ -12,15 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package cayley_http
+package http
 
 import (
 	"fmt"
-	"github.com/julienschmidt/httprouter"
-	"github.com/russross/blackfriday"
 	"io/ioutil"
 	"net/http"
 	"os"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/russross/blackfriday"
 )
 
 type DocRequestHandler struct {

http/cayley-http-query.go

@@ -12,17 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package cayley_http
+package http
 
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/julienschmidt/httprouter"
-	"graph"
-	"gremlin"
 	"io/ioutil"
-	"mql"
 	"net/http"
+
+	"github.com/julienschmidt/httprouter"
+
+	"github.com/google/cayley/graph"
+	"github.com/google/cayley/gremlin"
+	"github.com/google/cayley/mql"
 )
 
 type SuccessQueryWrapper struct {

http/cayley-http-write.go

@@ -12,19 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package cayley_http
+package http
 
 import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/barakmich/glog"
-	"github.com/julienschmidt/httprouter"
-	"graph"
 	"io/ioutil"
 	"net/http"
-	"nquads"
 	"strconv"
+
+	"github.com/barakmich/glog"
+	"github.com/julienschmidt/httprouter"
+
+	"github.com/google/cayley/graph"
+	"github.com/google/cayley/nquads"
 )
 
 func ParseJsonToTripleList(jsonBody []byte) ([]*graph.Triple, error) {

http/cayley-http.go

@@ -12,17 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package cayley_http
+package http
 
 import (
-	cfg "cayley_config"
 	"fmt"
-	"github.com/barakmich/glog"
-	"github.com/julienschmidt/httprouter"
-	"graph"
 	"html/template"
 	"net/http"
 	"time"
+
+	"github.com/barakmich/glog"
+	"github.com/julienschmidt/httprouter"
+
+	cfg "github.com/google/cayley/config"
+	"github.com/google/cayley/graph"
 )
 
 type ResponseHandler func(http.ResponseWriter, *http.Request, httprouter.Params) int

http/cayley-http_test.go

@@ -12,11 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package cayley_http
+package http
 
 import (
-	. "github.com/smartystreets/goconvey/convey"
 	"testing"
+	. "github.com/smartystreets/goconvey/convey"
 )
 
 func TestParseJSONOkay(t *testing.T) {

init.go

@@ -15,9 +15,9 @@
 package cayley
 
 import (
-	cfg "cayley_config"
+	cfg "github.com/google/cayley/config"
-	"graph_leveldb"
+	"github.com/google/cayley/graph/leveldb"
-	"graph_mongo"
+	"github.com/google/cayley/graph/mongo"
 )
 
 func CayleyInit(config *cfg.CayleyConfig, triplePath string) bool {
@@ -25,9 +25,9 @@ func CayleyInit(config *cfg.CayleyConfig, triplePath string) bool {
 	dbpath := config.DatabasePath
 	switch config.DatabaseType {
 	case "mongo", "mongodb":
-		created = graph_mongo.CreateNewMongoGraph(dbpath, config.DatabaseOptions)
+		created = mongo.CreateNewMongoGraph(dbpath, config.DatabaseOptions)
 	case "leveldb":
-		created = graph_leveldb.CreateNewLevelDB(dbpath)
+		created = leveldb.CreateNewLevelDB(dbpath)
 	case "mem":
 		return true
 	}

load.go

@@ -15,19 +15,21 @@
 package cayley
 
 import (
-	cfg "cayley_config"
-	"github.com/barakmich/glog"
-	"graph"
-	"graph_mongo"
-	"nquads"
 	"os"
+
+	"github.com/barakmich/glog"
+
+	cfg "github.com/google/cayley/config"
+	"github.com/google/cayley/graph"
+	"github.com/google/cayley/graph/mongo"
+	"github.com/google/cayley/nquads"
 )
 
 func CayleyLoad(ts graph.TripleStore, config *cfg.CayleyConfig, triplePath string, firstTime bool) {
 	switch config.DatabaseType {
 	case "mongo", "mongodb":
 		if firstTime {
-			loadMongo(ts.(*graph_mongo.MongoTripleStore), triplePath)
+			loadMongo(ts.(*mongo.MongoTripleStore), triplePath)
 		} else {
 			LoadTriplesFromFileInto(ts, triplePath, config.LoadSize)
 		}
@@ -41,7 +43,7 @@ func CayleyLoad(ts graph.TripleStore, config *cfg.CayleyConfig, triplePath strin
 
 }
 
-func loadMongo(ts *graph_mongo.MongoTripleStore, path string) {
+func loadMongo(ts *mongo.MongoTripleStore, path string) {
 	tChan := make(chan *graph.Triple)
 	go ReadTriplesFromFile(tChan, path)
 	ts.BulkLoad(tChan)

make.sh

@@ -1,65 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2014 The Cayley Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-set -e
-
-cd "`dirname '$0'`"
-SCRIPTPATH="`pwd`"
-cd - > /dev/null
-
-export GOPATH=$SCRIPTPATH
-export GOBIN=
-
-function deps {
-echo "Fetching dependencies to $SCRIPTPATH..."
-printf "                  (00/15)\r"
-  go get -u -t github.com/smartystreets/goconvey
-printf "#                 (01/15)\r"
-  go get -u github.com/badgerodon/peg
-printf "##                (02/15)\r"
-  go get -u github.com/barakmich/glog
-printf "####              (03/15)\r"
-  go get -u github.com/julienschmidt/httprouter
-printf "#####             (04/15)\r"
-  go get -u github.com/petar/GoLLRB/llrb
-printf "######            (05/15)\r"
-  go get -u github.com/robertkrimen/otto
-printf "#######           (06/15)\r"
-  go get -u github.com/stretchrcom/testify
-printf "########          (07/15)\r"
-  go get -u github.com/syndtr/goleveldb/leveldb
-printf "#########         (08/15)\r"
-  go get -u github.com/syndtr/goleveldb/leveldb/cache
-printf "##########        (09/15)\r"
-  go get -u github.com/syndtr/goleveldb/leveldb/iterator
-printf "###########       (10/15)\r"
-  go get -u github.com/syndtr/goleveldb/leveldb/opt
-printf "############      (11/15)\r"
-  go get -u github.com/syndtr/goleveldb/leveldb/util
-printf "#############     (12/15)\r"
-  go get -u labix.org/v2/mgo
-printf "##############    (13/15)\r"
-  go get -u labix.org/v2/mgo/bson
-printf "###############   (14/15)\r"
-  go get -u github.com/russross/blackfriday
-printf "################  (15/15)\r"
-printf "\n"
-}
-
-function build {
-  go build cayley
-}
-
-$1

mql/mql-build-iterator.go

@@ -17,10 +17,11 @@ package mql
 import (
 	"errors"
 	"fmt"
-	"graph"
 	"log"
 	"math"
 	"strings"
+
+	"github.com/google/cayley/graph"
 )
 
 func (m *MqlQuery) buildFixed(s string) graph.Iterator {

mql/mql-fill.go

@@ -15,8 +15,9 @@
 package mql
 
 import (
-	"graph"
 	"sort"
+
+	"github.com/google/cayley/graph"
 )
 
 func (m *MqlQuery) treeifyResult(tags map[string]graph.TSVal) map[MqlResultPath]string {

mql/mql-functional_test.go

@@ -16,9 +16,11 @@ package mql
 
 import (
 	"encoding/json"
-	. "github.com/smartystreets/goconvey/convey"
-	"graph_memstore"
 	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+
+	"github.com/google/cayley/graph/memstore"
 )
 
 //    +---+                        +---+
@@ -34,7 +36,7 @@ import (
 //
 
 func buildTripleStore() *MqlSession {
-	ts := graph_memstore.MakeTestingMemstore()
+	ts := memstore.MakeTestingMemstore()
 	return NewMqlSession(ts)
 }
 

mql/mql-query.go

@@ -16,8 +16,9 @@ package mql
 
 import (
 	"fmt"
-	"graph"
 	"strings"
+
+	"github.com/google/cayley/graph"
 )
 
 type MqlPath string

mql/mql-session.go

@@ -17,9 +17,11 @@ package mql
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/barakmich/glog"
-	"graph"
 	"sort"
+
+	"github.com/barakmich/glog"
+
+	"github.com/google/cayley/graph"
 )
 
 type MqlSession struct {

nquads/nquads.go

@@ -16,10 +16,12 @@ package nquads
 
 import (
 	"bufio"
-	"github.com/barakmich/glog"
-	"graph"
 	"io"
 	"strings"
+
+	"github.com/barakmich/glog"
+
+	"github.com/google/cayley/graph"
 )
 
 func isWhitespace(s uint8) bool {

nquads/nquads_test.go

@@ -15,9 +15,11 @@
 package nquads
 
 import (
-	. "github.com/smartystreets/goconvey/convey"
-	"graph"
 	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+
+	"github.com/google/cayley/graph"
 )
 
 func TestParsingNTriples(t *testing.T) {

open.go

@@ -15,24 +15,24 @@
 package cayley
 
 import (
-	"graph"
-
-	cfg "cayley_config"
 	"github.com/barakmich/glog"
-	"graph_leveldb"
+
-	"graph_memstore"
+	cfg "github.com/google/cayley/config"
-	"graph_mongo"
+	"github.com/google/cayley/graph"
+	"github.com/google/cayley/graph/leveldb"
+	"github.com/google/cayley/graph/memstore"
+	"github.com/google/cayley/graph/mongo"
 )
 
 func OpenTSFromConfig(config *cfg.CayleyConfig) graph.TripleStore {
 	glog.Infof("Opening database \"%s\" at %s", config.DatabaseType, config.DatabasePath)
 	switch config.DatabaseType {
 	case "mongo", "mongodb":
-		return graph_mongo.NewMongoTripleStore(config.DatabasePath, config.DatabaseOptions)
+		return mongo.NewMongoTripleStore(config.DatabasePath, config.DatabaseOptions)
 	case "leveldb":
-		return graph_leveldb.NewDefaultLevelDBTripleStore(config.DatabasePath, config.DatabaseOptions)
+		return leveldb.NewDefaultLevelDBTripleStore(config.DatabasePath, config.DatabaseOptions)
 	case "mem":
-		ts := graph_memstore.NewMemTripleStore()
+		ts := memstore.NewMemTripleStore()
 		CayleyLoad(ts, config, config.DatabasePath, true)
 		return ts
 	}

repl.go

@@ -16,17 +16,18 @@ package cayley
 
 import (
 	"bufio"
-	cfg "cayley_config"
 	"fmt"
-	"graph"
-	sexp "graph_sexp"
-	"gremlin"
 	"io"
-	"mql"
-	"nquads"
 	"os"
 	"strings"
 	"time"
+
+	cfg "github.com/google/cayley/config"
+	"github.com/google/cayley/graph"
+	"github.com/google/cayley/graph/sexp"
+	"github.com/google/cayley/gremlin"
+	"github.com/google/cayley/mql"
+	"github.com/google/cayley/nquads"
 )
 
 func trace(s string) (string, time.Time) {

src/cayley/main.go

@@ -1,87 +0,0 @@
-// Copyright 2014 The Cayley Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	cayley "cayley_cmd"
-	cfg "cayley_config"
-	cayley_http "cayley_http"
-	"flag"
-	"fmt"
-	"github.com/barakmich/glog"
-	"graph"
-	"os"
-	"runtime"
-)
-
-var tripleFile = flag.String("triples", "", "Triple File to load before going to REPL.")
-var cpuprofile = flag.String("prof", "", "Output profiling file.")
-var queryLanguage = flag.String("query_lang", "gremlin", "Use this parser as the query language.")
-var configFile = flag.String("config", "", "Path to an explicit configuration file.")
-
-func Usage() {
-	fmt.Println("Cayley is a graph store and graph query layer.\n")
-	fmt.Println("Usage:")
-	fmt.Println("  cayley COMMAND [flags]\n")
-	fmt.Println("Commands:")
-	fmt.Println("  init\tCreate an empty database.")
-	fmt.Println("  load\tBulk-load a triple file into the database.")
-	fmt.Println("  http\tServe an HTTP endpoint on the given host and port.")
-	fmt.Println("  repl\tDrop into a REPL of the given query language.")
-	fmt.Println("\nFlags:")
-	flag.Parse()
-	flag.PrintDefaults()
-}
-
-func main() {
-	// No command? It's time for usage.
-	if len(os.Args) == 1 {
-		Usage()
-		os.Exit(1)
-	}
-	cmd := os.Args[1]
-	newargs := make([]string, 0)
-	newargs = append(newargs, os.Args[0])
-	newargs = append(newargs, os.Args[2:]...)
-	os.Args = newargs
-	flag.Parse()
-	var ts graph.TripleStore
-	config := cfg.ParseConfigFromFlagsAndFile(*configFile)
-	if os.Getenv("GOMAXPROCS") == "" {
-		runtime.GOMAXPROCS(runtime.NumCPU())
-		glog.Infoln("Setting GOMAXPROCS to", runtime.NumCPU())
-	} else {
-		glog.Infoln("GOMAXPROCS currently", os.Getenv("GOMAXPROCS"), " -- not adjusting")
-	}
-	switch cmd {
-	case "init":
-		cayley.CayleyInit(config, *tripleFile)
-	case "load":
-		ts = cayley.OpenTSFromConfig(config)
-		cayley.CayleyLoad(ts, config, *tripleFile, false)
-		ts.Close()
-	case "repl":
-		ts = cayley.OpenTSFromConfig(config)
-		cayley.CayleyRepl(ts, *queryLanguage, config)
-		ts.Close()
-	case "http":
-		ts = cayley.OpenTSFromConfig(config)
-		cayley_http.CayleyHTTP(ts, config)
-		ts.Close()
-	default:
-		fmt.Println("No command", cmd)
-		flag.Usage()
-	}
-}

src/graph_leveldb/leveldb-all-iterator.go

@@ -1,132 +0,0 @@
-// Copyright 2014 The Cayley Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package graph_leveldb
-
-import (
-	"bytes"
-	"fmt"
-	leveldb_it "github.com/syndtr/goleveldb/leveldb/iterator"
-	leveldb_opt "github.com/syndtr/goleveldb/leveldb/opt"
-	"graph"
-	"strings"
-)
-
-type LevelDBAllIterator struct {
-	graph.BaseIterator
-	prefix []byte
-	dir    string
-	open   bool
-	it     leveldb_it.Iterator
-	ts     *LevelDBTripleStore
-	ro     *leveldb_opt.ReadOptions
-}
-
-func NewLevelDBAllIterator(prefix, dir string, ts *LevelDBTripleStore) *LevelDBAllIterator {
-	var it LevelDBAllIterator
-	graph.BaseIteratorInit(&it.BaseIterator)
-	it.ro = &leveldb_opt.ReadOptions{}
-	it.ro.DontFillCache = true
-	it.it = ts.db.NewIterator(nil, it.ro)
-	it.prefix = []byte(prefix)
-	it.dir = dir
-	it.open = true
-	it.ts = ts
-	it.it.Seek(it.prefix)
-	if !it.it.Valid() {
-		it.open = false
-		it.it.Release()
-	}
-	return &it
-}
-
-func (a *LevelDBAllIterator) Reset() {
-	if !a.open {
-		a.it = a.ts.db.NewIterator(nil, a.ro)
-		a.open = true
-	}
-	a.it.Seek(a.prefix)
-	if !a.it.Valid() {
-		a.open = false
-		a.it.Release()
-	}
-}
-
-func (a *LevelDBAllIterator) Clone() graph.Iterator {
-	out := NewLevelDBAllIterator(string(a.prefix), a.dir, a.ts)
-	out.CopyTagsFrom(a)
-	return out
-}
-
-func (a *LevelDBAllIterator) Next() (graph.TSVal, bool) {
-	if !a.open {
-		a.Last = nil
-		return nil, false
-	}
-	var out []byte
-	out = make([]byte, len(a.it.Key()))
-	copy(out, a.it.Key())
-	a.it.Next()
-	if !a.it.Valid() {
-		a.Close()
-	}
-	if !bytes.HasPrefix(out, a.prefix) {
-		a.Close()
-		return nil, false
-	}
-	a.Last = out
-	return out, true
-}
-
-func (a *LevelDBAllIterator) Check(v graph.TSVal) bool {
-	a.Last = v
-	return true
-}
-
-func (lit *LevelDBAllIterator) Close() {
-	if lit.open {
-		lit.it.Release()
-		lit.open = false
-	}
-}
-
-func (a *LevelDBAllIterator) Size() (int64, bool) {
-	size, err := a.ts.GetApproximateSizeForPrefix(a.prefix)
-	if err == nil {
-		return size, false
-	}
-	// INT64_MAX
-	return int64(^uint64(0) >> 1), false
-}
-
-func (lit *LevelDBAllIterator) DebugString(indent int) string {
-	size, _ := lit.Size()
-	return fmt.Sprintf("%s(%s tags: %v leveldb size:%d %s %p)", strings.Repeat(" ", indent), lit.Type(), lit.Tags(), size, lit.dir, lit)
-}
-
-func (lit *LevelDBAllIterator) Type() string { return "all" }
-func (lit *LevelDBAllIterator) Sorted() bool { return false }
-
-func (lit *LevelDBAllIterator) Optimize() (graph.Iterator, bool) {
-	return lit, false
-}
-
-func (lit *LevelDBAllIterator) GetStats() *graph.IteratorStats {
-	s, _ := lit.Size()
-	return &graph.IteratorStats{
-		CheckCost: 1,
-		NextCost:  2,
-		Size:      s,
-	}
-}

src/graph_leveldb/leveldb-iterator.go

@@ -1,210 +0,0 @@
-// Copyright 2014 The Cayley Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package graph_leveldb
-
-import (
-	"bytes"
-	_ "encoding/binary"
-	"fmt"
-	leveldb_it "github.com/syndtr/goleveldb/leveldb/iterator"
-	leveldb_opt "github.com/syndtr/goleveldb/leveldb/opt"
-	"graph"
-	"strings"
-)
-
-type LevelDBIterator struct {
-	graph.BaseIterator
-	nextPrefix     []byte
-	checkId        []byte
-	dir            string
-	open           bool
-	it             leveldb_it.Iterator
-	ts             *LevelDBTripleStore
-	ro             *leveldb_opt.ReadOptions
-	originalPrefix string
-}
-
-func NewLevelDBIterator(prefix, dir string, value graph.TSVal, ts *LevelDBTripleStore) *LevelDBIterator {
-	var it LevelDBIterator
-	graph.BaseIteratorInit(&it.BaseIterator)
-	it.checkId = value.([]byte)
-	it.dir = dir
-	it.originalPrefix = prefix
-	it.nextPrefix = make([]byte, 0, 2+ts.hasher.Size())
-	it.nextPrefix = append(it.nextPrefix, []byte(prefix)...)
-	it.nextPrefix = append(it.nextPrefix, []byte(it.checkId[1:])...)
-	it.ro = &leveldb_opt.ReadOptions{}
-	it.ro.DontFillCache = true
-	it.it = ts.db.NewIterator(nil, it.ro)
-	it.open = true
-	it.ts = ts
-	ok := it.it.Seek(it.nextPrefix)
-	if !ok {
-		it.open = false
-		it.it.Release()
-	}
-	return &it
-}
-
-func (lit *LevelDBIterator) Reset() {
-	if !lit.open {
-		lit.it = lit.ts.db.NewIterator(nil, lit.ro)
-		lit.open = true
-	}
-	ok := lit.it.Seek(lit.nextPrefix)
-	if !ok {
-		lit.open = false
-		lit.it.Release()
-	}
-}
-
-func (lit *LevelDBIterator) Clone() graph.Iterator {
-	out := NewLevelDBIterator(lit.originalPrefix, lit.dir, lit.checkId, lit.ts)
-	out.CopyTagsFrom(lit)
-	return out
-}
-
-func (lit *LevelDBIterator) Close() {
-	if lit.open {
-		lit.it.Release()
-		lit.open = false
-	}
-}
-
-func (lit *LevelDBIterator) Next() (graph.TSVal, bool) {
-	if lit.it == nil {
-		lit.Last = nil
-		return nil, false
-	}
-	if !lit.open {
-		lit.Last = nil
-		return nil, false
-	}
-	if !lit.it.Valid() {
-		lit.Last = nil
-		lit.Close()
-		return nil, false
-	}
-	if bytes.HasPrefix(lit.it.Key(), lit.nextPrefix) {
-		out := make([]byte, len(lit.it.Key()))
-		copy(out, lit.it.Key())
-		lit.Last = out
-		ok := lit.it.Next()
-		if !ok {
-			lit.Close()
-		}
-		return out, true
-	}
-	lit.Close()
-	lit.Last = nil
-	return nil, false
-}
-
-func GetPositionFromPrefix(prefix []byte, dir string, ts *LevelDBTripleStore) int {
-	if bytes.Equal(prefix, []byte("sp")) {
-		switch dir {
-		case "s":
-			return 2
-		case "p":
-			return ts.hasher.Size() + 2
-		case "o":
-			return 2*ts.hasher.Size() + 2
-		case "c":
-			return -1
-		}
-	}
-	if bytes.Equal(prefix, []byte("po")) {
-		switch dir {
-		case "s":
-			return 2*ts.hasher.Size() + 2
-		case "p":
-			return 2
-		case "o":
-			return ts.hasher.Size() + 2
-		case "c":
-			return -1
-		}
-	}
-	if bytes.Equal(prefix, []byte("os")) {
-		switch dir {
-		case "s":
-			return ts.hasher.Size() + 2
-		case "p":
-			return 2*ts.hasher.Size() + 2
-		case "o":
-			return 2
-		case "c":
-			return -1
-		}
-	}
-	if bytes.Equal(prefix, []byte("cp")) {
-		switch dir {
-		case "s":
-			return 2*ts.hasher.Size() + 2
-		case "p":
-			return ts.hasher.Size() + 2
-		case "o":
-			return 3*ts.hasher.Size() + 2
-		case "c":
-			return 2
-		}
-	}
-	panic("Notreached")
-}
-
-func (lit *LevelDBIterator) Check(v graph.TSVal) bool {
-	val := v.([]byte)
-	if val[0] == 'z' {
-		return false
-	}
-	offset := GetPositionFromPrefix(val[0:2], lit.dir, lit.ts)
-	if offset != -1 {
-		if bytes.HasPrefix(val[offset:], lit.checkId[1:]) {
-			return true
-		}
-	} else {
-		nameForDir := lit.ts.GetTriple(v).Get(lit.dir)
-		hashForDir := lit.ts.GetIdFor(nameForDir).([]byte)
-		if bytes.Equal(hashForDir, lit.checkId) {
-			return true
-		}
-	}
-	return false
-}
-
-func (lit *LevelDBIterator) Size() (int64, bool) {
-	return lit.ts.GetSizeFor(lit.checkId), true
-}
-
-func (lit *LevelDBIterator) DebugString(indent int) string {
-	size, _ := lit.Size()
-	return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), lit.Type(), lit.GetUid(), lit.Tags(), lit.dir, size, lit.ts.GetNameFor(lit.checkId))
-}
-
-func (lit *LevelDBIterator) Type() string { return "leveldb" }
-func (lit *LevelDBIterator) Sorted() bool { return false }
-
-func (lit *LevelDBIterator) Optimize() (graph.Iterator, bool) {
-	return lit, false
-}
-
-func (lit *LevelDBIterator) GetStats() *graph.IteratorStats {
-	s, _ := lit.Size()
-	return &graph.IteratorStats{
-		CheckCost: 1,
-		NextCost:  2,
-		Size:      s,
-	}
-}

src/graph_leveldb/leveldb-triplestore-iterator-optimize.go

@@ -1,53 +0,0 @@
-// Copyright 2014 The Cayley Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package graph_leveldb
-
-import (
-	"graph"
-)
-
-func (ts *LevelDBTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
-	switch it.Type() {
-	case "linksto":
-		return ts.optimizeLinksTo(it.(*graph.LinksToIterator))
-
-	}
-	return it, false
-}
-
-func (ts *LevelDBTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) {
-	l := it.GetSubIterators()
-	if l.Len() != 1 {
-		return it, false
-	}
-	primaryIt := l.Front().Value.(graph.Iterator)
-	if primaryIt.Type() == "fixed" {
-		size, _ := primaryIt.Size()
-		if size == 1 {
-			val, ok := primaryIt.Next()
-			if !ok {
-				panic("Sizes lie")
-			}
-			newIt := ts.GetTripleIterator(it.Direction(), val)
-			newIt.CopyTagsFrom(it)
-			for _, tag := range primaryIt.Tags() {
-				newIt.AddFixedTag(tag, val)
-			}
-			it.Close()
-			return newIt, true
-		}
-	}
-	return it, false
-}

src/graph_leveldb/leveldb-triplestore.go

@@ -1,427 +0,0 @@
-// Copyright 2014 The Cayley Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package graph_leveldb
-
-import (
-	"bytes"
-	"crypto/sha1"
-	"encoding/binary"
-	"encoding/json"
-	"fmt"
-	"github.com/barakmich/glog"
-	"github.com/syndtr/goleveldb/leveldb"
-	leveldb_cache "github.com/syndtr/goleveldb/leveldb/cache"
-	leveldb_opt "github.com/syndtr/goleveldb/leveldb/opt"
-	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
-	"graph"
-	"hash"
-)
-
-const DefaultCacheSize = 2
-const DefaultWriteBufferSize = 20
-
-type LevelDBTripleStore struct {
-	dbOpts    *leveldb_opt.Options
-	db        *leveldb.DB
-	path      string
-	open      bool
-	size      int64
-	hasher    hash.Hash
-	writeopts *leveldb_opt.WriteOptions
-	readopts  *leveldb_opt.ReadOptions
-}
-
-func CreateNewLevelDB(path string) bool {
-	opts := &leveldb_opt.Options{}
-	db, err := leveldb.OpenFile(path, opts)
-	if err != nil {
-		glog.Errorln("Error: couldn't create database", err)
-		return false
-	}
-	defer db.Close()
-	ts := &LevelDBTripleStore{}
-	ts.db = db
-	ts.writeopts = &leveldb_opt.WriteOptions{
-		Sync: true,
-	}
-	ts.Close()
-	return true
-}
-
-func NewDefaultLevelDBTripleStore(path string, options graph.OptionsDict) *LevelDBTripleStore {
-	var ts LevelDBTripleStore
-	ts.path = path
-	cache_size := DefaultCacheSize
-	if val, ok := options.GetIntKey("cache_size_mb"); ok {
-		cache_size = val
-	}
-	ts.dbOpts = &leveldb_opt.Options{
-		BlockCache: leveldb_cache.NewLRUCache(cache_size * leveldb_opt.MiB),
-	}
-	ts.dbOpts.ErrorIfMissing = true
-
-	write_buffer_mb := DefaultWriteBufferSize
-	if val, ok := options.GetIntKey("write_buffer_mb"); ok {
-		write_buffer_mb = val
-	}
-	ts.dbOpts.WriteBuffer = write_buffer_mb * leveldb_opt.MiB
-	ts.hasher = sha1.New()
-	ts.writeopts = &leveldb_opt.WriteOptions{
-		Sync: false,
-	}
-	ts.readopts = &leveldb_opt.ReadOptions{}
-	db, err := leveldb.OpenFile(ts.path, ts.dbOpts)
-	if err != nil {
-		panic("Error, couldn't open! " + err.Error())
-	}
-	ts.db = db
-	glog.Infoln(ts.GetStats())
-	ts.getSize()
-	return &ts
-}
-
-func (ts *LevelDBTripleStore) GetStats() string {
-	out := ""
-	stats, err := ts.db.GetProperty("leveldb.stats")
-	if err == nil {
-		out += fmt.Sprintln("Stats: ", stats)
-	}
-	out += fmt.Sprintln("Size: ", ts.size)
-	return out
-}
-
-func (ts *LevelDBTripleStore) Size() int64 {
-	return ts.size
-}
-
-func (ts *LevelDBTripleStore) createKeyFor(dir1, dir2, dir3 string, triple *graph.Triple) []byte {
-	key := make([]byte, 0, 2+(ts.hasher.Size()*3))
-	key = append(key, []byte(dir1+dir2)...)
-	key = append(key, ts.convertStringToByteHash(triple.Get(dir1))...)
-	key = append(key, ts.convertStringToByteHash(triple.Get(dir2))...)
-	key = append(key, ts.convertStringToByteHash(triple.Get(dir3))...)
-	return key
-}
-
-func (ts *LevelDBTripleStore) createProvKeyFor(dir1, dir2, dir3 string, triple *graph.Triple) []byte {
-	key := make([]byte, 0, 2+(ts.hasher.Size()*4))
-	key = append(key, []byte("c"+dir1)...)
-	key = append(key, ts.convertStringToByteHash(triple.Get("c"))...)
-	key = append(key, ts.convertStringToByteHash(triple.Get(dir1))...)
-	key = append(key, ts.convertStringToByteHash(triple.Get(dir2))...)
-	key = append(key, ts.convertStringToByteHash(triple.Get(dir3))...)
-	return key
-}
-
-func (ts *LevelDBTripleStore) createValueKeyFor(s string) []byte {
-	key := make([]byte, 0, 1+ts.hasher.Size())
-	key = append(key, []byte("z")...)
-	key = append(key, ts.convertStringToByteHash(s)...)
-	return key
-}
-
-func (ts *LevelDBTripleStore) AddTriple(t *graph.Triple) {
-	batch := &leveldb.Batch{}
-	ts.buildWrite(batch, t)
-	err := ts.db.Write(batch, ts.writeopts)
-	if err != nil {
-		glog.Errorf("Couldn't write to DB for triple %s", t.ToString())
-		return
-	}
-	ts.size++
-}
-
-func (ts *LevelDBTripleStore) RemoveTriple(t *graph.Triple) {
-	_, err := ts.db.Get(ts.createKeyFor("s", "p", "o", t), ts.readopts)
-	if err != nil && err != leveldb.ErrNotFound {
-		glog.Errorf("Couldn't access DB to confirm deletion")
-		return
-	}
-	if err == leveldb.ErrNotFound {
-		// No such triple in the database, forget about it.
-		return
-	}
-	batch := &leveldb.Batch{}
-	batch.Delete(ts.createKeyFor("s", "p", "o", t))
-	batch.Delete(ts.createKeyFor("o", "s", "p", t))
-	batch.Delete(ts.createKeyFor("p", "o", "s", t))
-	ts.UpdateValueKeyBy(t.Get("s"), -1, batch)
-	ts.UpdateValueKeyBy(t.Get("p"), -1, batch)
-	ts.UpdateValueKeyBy(t.Get("o"), -1, batch)
-	if t.Get("c") != "" {
-		batch.Delete(ts.createProvKeyFor("p", "s", "o", t))
-		ts.UpdateValueKeyBy(t.Get("c"), -1, batch)
-	}
-	err = ts.db.Write(batch, nil)
-	if err != nil {
-		glog.Errorf("Couldn't delete triple %s", t.ToString())
-		return
-	}
-	ts.size--
-}
-
-func (ts *LevelDBTripleStore) buildTripleWrite(batch *leveldb.Batch, t *graph.Triple) {
-	bytes, err := json.Marshal(*t)
-	if err != nil {
-		glog.Errorf("Couldn't write to buffer for triple %s\n  %s\n", t.ToString(), err)
-		return
-	}
-	batch.Put(ts.createKeyFor("s", "p", "o", t), bytes)
-	batch.Put(ts.createKeyFor("o", "s", "p", t), bytes)
-	batch.Put(ts.createKeyFor("p", "o", "s", t), bytes)
-	if t.Get("c") != "" {
-		batch.Put(ts.createProvKeyFor("p", "s", "o", t), bytes)
-	}
-}
-
-func (ts *LevelDBTripleStore) buildWrite(batch *leveldb.Batch, t *graph.Triple) {
-	ts.buildTripleWrite(batch, t)
-	ts.UpdateValueKeyBy(t.Get("s"), 1, nil)
-	ts.UpdateValueKeyBy(t.Get("p"), 1, nil)
-	ts.UpdateValueKeyBy(t.Get("o"), 1, nil)
-	if t.Get("c") != "" {
-		ts.UpdateValueKeyBy(t.Get("c"), 1, nil)
-	}
-}
-
-type ValueData struct {
-	Name string
-	Size int64
-}
-
-func (ts *LevelDBTripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.Batch) {
-	value := &ValueData{name, int64(amount)}
-	key := ts.createValueKeyFor(name)
-	b, err := ts.db.Get(key, ts.readopts)
-
-	// Error getting the node from the database.
-	if err != nil && err != leveldb.ErrNotFound {
-		glog.Errorf("Error reading Value %s from the DB\n", name)
-		return
-	}
-
-	// Node exists in the database -- unmarshal and update.
-	if b != nil && err != leveldb.ErrNotFound {
-		err = json.Unmarshal(b, value)
-		if err != nil {
-			glog.Errorln("Error: couldn't reconstruct value ", err)
-			return
-		}
-		value.Size += int64(amount)
-	}
-
-	// Are we deleting something?
-	if amount < 0 {
-		if value.Size <= 0 {
-			if batch == nil {
-				ts.db.Delete(key, ts.writeopts)
-			} else {
-				batch.Delete(key)
-			}
-			return
-		}
-	}
-
-	// Repackage and rewrite.
-	bytes, err := json.Marshal(&value)
-	if err != nil {
-		glog.Errorf("Couldn't write to buffer for value %s\n %s", name, err)
-		return
-	}
-	if batch == nil {
-		ts.db.Put(key, bytes, ts.writeopts)
-	} else {
-		batch.Put(key, bytes)
-	}
-}
-
-func (ts *LevelDBTripleStore) AddTripleSet(t_s []*graph.Triple) {
-	batch := &leveldb.Batch{}
-	newTs := len(t_s)
-	resizeMap := make(map[string]int)
-	for _, t := range t_s {
-		ts.buildTripleWrite(batch, t)
-		resizeMap[t.Sub]++
-		resizeMap[t.Pred]++
-		resizeMap[t.Obj]++
-		if t.Provenance != "" {
-			resizeMap[t.Provenance]++
-		}
-	}
-	for k, v := range resizeMap {
-		ts.UpdateValueKeyBy(k, v, batch)
-	}
-	err := ts.db.Write(batch, ts.writeopts)
-	if err != nil {
-		glog.Errorf("Couldn't write to DB for tripleset")
-		return
-	}
-	ts.size += int64(newTs)
-}
-
-func (ldbts *LevelDBTripleStore) Close() {
-	buf := new(bytes.Buffer)
-	err := binary.Write(buf, binary.LittleEndian, ldbts.size)
-	if err == nil {
-		werr := ldbts.db.Put([]byte("__size"), buf.Bytes(), ldbts.writeopts)
-		if werr != nil {
-			glog.Errorf("Couldn't write size before closing!")
-		}
-	} else {
-		glog.Errorf("Couldn't convert size before closing!")
-	}
-	ldbts.db.Close()
-	ldbts.open = false
-}
-
-func (ts *LevelDBTripleStore) GetTriple(k graph.TSVal) *graph.Triple {
-	var triple graph.Triple
-	b, err := ts.db.Get(k.([]byte), ts.readopts)
-	if err != nil && err != leveldb.ErrNotFound {
-		glog.Errorln("Error: couldn't get triple from DB")
-		return &graph.Triple{}
-	}
-	if err == leveldb.ErrNotFound {
-		// No harm, no foul.
-		return &graph.Triple{}
-	}
-	err = json.Unmarshal(b, &triple)
-	if err != nil {
-		glog.Errorln("Error: couldn't reconstruct triple")
-		return &graph.Triple{}
-	}
-	return &triple
-}
-
-func (ts *LevelDBTripleStore) convertStringToByteHash(s string) []byte {
-	ts.hasher.Reset()
-	key := make([]byte, 0, ts.hasher.Size())
-	ts.hasher.Write([]byte(s))
-	key = ts.hasher.Sum(key)
-	return key
-}
-
-func (ts *LevelDBTripleStore) GetIdFor(s string) graph.TSVal {
-	return ts.createValueKeyFor(s)
-}
-
-func (ts *LevelDBTripleStore) getValueData(value_key []byte) ValueData {
-	var out ValueData
-	if glog.V(3) {
-		glog.V(3).Infof("%s %v\n", string(value_key[0]), value_key)
-	}
-	b, err := ts.db.Get(value_key, ts.readopts)
-	if err != nil && err != leveldb.ErrNotFound {
-		glog.Errorln("Error: couldn't get value from DB")
-		return out
-	}
-	if b != nil && err != leveldb.ErrNotFound {
-		err = json.Unmarshal(b, &out)
-		if err != nil {
-			glog.Errorln("Error: couldn't reconstruct value")
-			return ValueData{}
-		}
-	}
-	return out
-}
-
-func (ts *LevelDBTripleStore) GetNameFor(k graph.TSVal) string {
-	if k == nil {
-		glog.V(2).Infoln("k was nil")
-		return ""
-	}
-	return ts.getValueData(k.([]byte)).Name
-}
-
-func (ts *LevelDBTripleStore) GetSizeFor(k graph.TSVal) int64 {
-	if k == nil {
-		return 0
-	}
-	return int64(ts.getValueData(k.([]byte)).Size)
-}
-
-func (ts *LevelDBTripleStore) getSize() {
-	var size int64
-	b, err := ts.db.Get([]byte("__size"), ts.readopts)
-	if err != nil && err != leveldb.ErrNotFound {
-		panic("Couldn't read size " + err.Error())
-	}
-	if err == leveldb.ErrNotFound {
-		// Must be a new database. Cool
-		ts.size = 0
-		return
-	}
-	buf := bytes.NewBuffer(b)
-	err = binary.Read(buf, binary.LittleEndian, &size)
-	if err != nil {
-		glog.Errorln("Error: couldn't parse size")
-	}
-	ts.size = size
-}
-
-func (ts *LevelDBTripleStore) GetApproximateSizeForPrefix(pre []byte) (int64, error) {
-	limit := make([]byte, len(pre))
-	copy(limit, pre)
-	end := len(limit) - 1
-	limit[end]++
-	ranges := make([]leveldb_util.Range, 1)
-	ranges[0].Start = pre
-	ranges[0].Limit = limit
-	sizes, err := ts.db.GetApproximateSizes(ranges)
-	if err == nil {
-		return (int64(sizes[0]) >> 6) + 1, nil
-	}
-	return 0, nil
-}
-
-func (ts *LevelDBTripleStore) GetTripleIterator(dir string, val graph.TSVal) graph.Iterator {
-	switch dir {
-	case "s":
-		return NewLevelDBIterator("sp", "s", val, ts)
-	case "p":
-		return NewLevelDBIterator("po", "p", val, ts)
-	case "o":
-		return NewLevelDBIterator("os", "o", val, ts)
-	case "c":
-		return NewLevelDBIterator("cp", "c", val, ts)
-	}
-	panic("Notreached " + dir)
-}
-
-func (ts *LevelDBTripleStore) GetNodesAllIterator() graph.Iterator {
-	return NewLevelDBAllIterator("z", "v", ts)
-}
-
-func (ts *LevelDBTripleStore) GetTriplesAllIterator() graph.Iterator {
-	return NewLevelDBAllIterator("po", "p", ts)
-}
-
-func (ts *LevelDBTripleStore) GetTripleDirection(val graph.TSVal, direction string) graph.TSVal {
-	v := val.([]uint8)
-	offset := GetPositionFromPrefix(v[0:2], direction, ts)
-	if offset != -1 {
-		return append([]byte("z"), v[offset:offset+ts.hasher.Size()]...)
-	} else {
-		return ts.GetTriple(val).Get(direction)
-	}
-}
-
-func compareBytes(a, b graph.TSVal) bool {
-	return bytes.Equal(a.([]uint8), b.([]uint8))
-}
-
-func (ts *LevelDBTripleStore) MakeFixed() *graph.FixedIterator {
-	return graph.NewFixedIteratorWithCompare(compareBytes)
-}

src/graph_leveldb/leveldb_test.go

@@ -1,433 +0,0 @@
-// Copyright 2014 The Cayley Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package graph_leveldb
-
-import (
-	. "github.com/smartystreets/goconvey/convey"
-	"graph"
-	"io/ioutil"
-	"os"
-	"sort"
-	"testing"
-)
-
-func makeTripleSet() []*graph.Triple {
-	tripleSet := []*graph.Triple{
-		graph.MakeTriple("A", "follows", "B", ""),
-		graph.MakeTriple("C", "follows", "B", ""),
-		graph.MakeTriple("C", "follows", "D", ""),
-		graph.MakeTriple("D", "follows", "B", ""),
-		graph.MakeTriple("B", "follows", "F", ""),
-		graph.MakeTriple("F", "follows", "G", ""),
-		graph.MakeTriple("D", "follows", "G", ""),
-		graph.MakeTriple("E", "follows", "F", ""),
-		graph.MakeTriple("B", "status", "cool", "status_graph"),
-		graph.MakeTriple("D", "status", "cool", "status_graph"),
-		graph.MakeTriple("G", "status", "cool", "status_graph"),
-	}
-	return tripleSet
-}
-
-func extractTripleFromIterator(ts graph.TripleStore, it graph.Iterator) []string {
-	var output []string
-	for {
-		val, ok := it.Next()
-		if !ok {
-			break
-		}
-		output = append(output, ts.GetTriple(val).ToString())
-	}
-	return output
-}
-
-func extractValuesFromIterator(ts graph.TripleStore, it graph.Iterator) []string {
-	var output []string
-	for {
-		val, ok := it.Next()
-		if !ok {
-			break
-		}
-		output = append(output, ts.GetNameFor(val))
-	}
-	return output
-}
-
-func TestCreateDatabase(t *testing.T) {
-
-	Convey("Given a database path", t, func() {
-		tmpDir, err := ioutil.TempDir(os.TempDir(), "cayley_test")
-		t.Log(tmpDir)
-		if err != nil {
-			t.Fatal("Cannot use ioutil.", err)
-		}
-
-		Convey("Creates a database", func() {
-			ok := CreateNewLevelDB(tmpDir)
-			So(ok, ShouldBeTrue)
-			Convey("And has good defaults for a new database", func() {
-				ts := NewDefaultLevelDBTripleStore(tmpDir, nil)
-				So(ts, ShouldNotBeNil)
-				So(ts.Size(), ShouldEqual, 0)
-				ts.Close()
-			})
-		})
-
-		Convey("Fails if it cannot create the database", func() {
-			ok := CreateNewLevelDB("/dev/null/some terrible path")
-			So(ok, ShouldBeFalse)
-			So(func() { NewDefaultLevelDBTripleStore("/dev/null/some terrible path", nil) }, ShouldPanic)
-		})
-
-		Reset(func() {
-			os.RemoveAll(tmpDir)
-		})
-
-	})
-
-}
-
-func TestLoadDatabase(t *testing.T) {
-	var ts *LevelDBTripleStore
-
-	Convey("Given a created database path", t, func() {
-		tmpDir, _ := ioutil.TempDir(os.TempDir(), "cayley_test")
-		t.Log(tmpDir)
-		ok := CreateNewLevelDB(tmpDir)
-		So(ok, ShouldBeTrue)
-		ts = NewDefaultLevelDBTripleStore(tmpDir, nil)
-
-		Convey("Can load a single triple", func() {
-			ts.AddTriple(graph.MakeTriple("Something", "points_to", "Something Else", "context"))
-			So(ts.GetNameFor(ts.GetIdFor("Something")), ShouldEqual, "Something")
-			So(ts.Size(), ShouldEqual, 1)
-		})
-
-		Convey("Can load many triples", func() {
-
-			ts.AddTripleSet(makeTripleSet())
-			So(ts.Size(), ShouldEqual, 11)
-			So(ts.GetSizeFor(ts.GetIdFor("B")), ShouldEqual, 5)
-
-			Convey("Can delete triples", func() {
-				ts.RemoveTriple(graph.MakeTriple("A", "follows", "B", ""))
-				So(ts.Size(), ShouldEqual, 10)
-				So(ts.GetSizeFor(ts.GetIdFor("B")), ShouldEqual, 4)
-			})
-		})
-
-		Reset(func() {
-			ts.Close()
-			os.RemoveAll(tmpDir)
-		})
-
-	})
-
-}
-
-func TestAllIterator(t *testing.T) {
-	var ts *LevelDBTripleStore
-
-	Convey("Given a prepared database", t, func() {
-		tmpDir, _ := ioutil.TempDir(os.TempDir(), "cayley_test")
-		t.Log(tmpDir)
-		defer os.RemoveAll(tmpDir)
-		ok := CreateNewLevelDB(tmpDir)
-		So(ok, ShouldBeTrue)
-		ts = NewDefaultLevelDBTripleStore(tmpDir, nil)
-		ts.AddTripleSet(makeTripleSet())
-		var it graph.Iterator
-
-		Convey("Can create an all iterator for nodes", func() {
-			it = ts.GetNodesAllIterator()
-			So(it, ShouldNotBeNil)
-
-			Convey("Has basics", func() {
-				size, accurate := it.Size()
-				So(size, ShouldBeBetween, 0, 20)
-				So(accurate, ShouldBeFalse)
-				So(it.Type(), ShouldEqual, "all")
-				re_it, ok := it.Optimize()
-				So(ok, ShouldBeFalse)
-				So(re_it, ShouldPointTo, it)
-			})
-
-			Convey("Iterates all nodes", func() {
-				expected := []string{
-					"A",
-					"B",
-					"C",
-					"D",
-					"E",
-					"F",
-					"G",
-					"follows",
-					"status",
-					"cool",
-					"status_graph",
-				}
-				sort.Strings(expected)
-				actual := extractValuesFromIterator(ts, it)
-				sort.Strings(actual)
-				So(actual, ShouldResemble, expected)
-				it.Reset()
-				actual = extractValuesFromIterator(ts, it)
-				sort.Strings(actual)
-				So(actual, ShouldResemble, expected)
-
-			})
-
-			Convey("Contains a couple nodes", func() {
-				So(it.Check(ts.GetIdFor("A")), ShouldBeTrue)
-				So(it.Check(ts.GetIdFor("cool")), ShouldBeTrue)
-				//So(it.Check(ts.GetIdFor("baller")), ShouldBeFalse)
-			})
-
-			Reset(func() {
-				it.Reset()
-			})
-		})
-
-		Convey("Can create an all iterator for edges", func() {
-			it := ts.GetTriplesAllIterator()
-			So(it, ShouldNotBeNil)
-			Convey("Has basics", func() {
-				size, accurate := it.Size()
-				So(size, ShouldBeBetween, 0, 20)
-				So(accurate, ShouldBeFalse)
-				So(it.Type(), ShouldEqual, "all")
-				re_it, ok := it.Optimize()
-				So(ok, ShouldBeFalse)
-				So(re_it, ShouldPointTo, it)
-			})
-
-			Convey("Iterates an edge", func() {
-				edge_val, _ := it.Next()
-				triple := ts.GetTriple(edge_val)
-				set := makeTripleSet()
-				var string_set []string
-				for _, t := range set {
-					string_set = append(string_set, t.ToString())
-				}
-				So(triple.ToString(), ShouldBeIn, string_set)
-			})
-
-			Reset(func() {
-				ts.Close()
-			})
-		})
-	})
-
-}
-
-func TestSetIterator(t *testing.T) {
-	var ts *LevelDBTripleStore
-	var tmpDir string
-
-	Convey("Given a prepared database", t, func() {
-		tmpDir, _ = ioutil.TempDir(os.TempDir(), "cayley_test")
-		t.Log(tmpDir)
-		defer os.RemoveAll(tmpDir)
-		ok := CreateNewLevelDB(tmpDir)
-		So(ok, ShouldBeTrue)
-		ts = NewDefaultLevelDBTripleStore(tmpDir, nil)
-		ts.AddTripleSet(makeTripleSet())
-		var it graph.Iterator
-
-		Convey("Can create a subject iterator", func() {
-			it = ts.GetTripleIterator("s", ts.GetIdFor("C"))
-
-			Convey("Containing the right things", func() {
-				expected := []string{
-					graph.MakeTriple("C", "follows", "B", "").ToString(),
-					graph.MakeTriple("C", "follows", "D", "").ToString(),
-				}
-				actual := extractTripleFromIterator(ts, it)
-				sort.Strings(actual)
-				sort.Strings(expected)
-				So(actual, ShouldResemble, expected)
-			})
-
-			Convey("And checkable", func() {
-				and := graph.NewAndIterator()
-				and.AddSubIterator(ts.GetTriplesAllIterator())
-				and.AddSubIterator(it)
-
-				expected := []string{
-					graph.MakeTriple("C", "follows", "B", "").ToString(),
-					graph.MakeTriple("C", "follows", "D", "").ToString(),
-				}
-				actual := extractTripleFromIterator(ts, and)
-				sort.Strings(actual)
-				sort.Strings(expected)
-				So(actual, ShouldResemble, expected)
-			})
-			Reset(func() {
-				it.Reset()
-			})
-
-		})
-
-		Convey("Can create an object iterator", func() {
-			it = ts.GetTripleIterator("o", ts.GetIdFor("F"))
-
-			Convey("Containing the right things", func() {
-				expected := []string{
-					graph.MakeTriple("B", "follows", "F", "").ToString(),
-					graph.MakeTriple("E", "follows", "F", "").ToString(),
-				}
-				actual := extractTripleFromIterator(ts, it)
-				sort.Strings(actual)
-				sort.Strings(expected)
-				So(actual, ShouldResemble, expected)
-			})
-
-			Convey("Mutually and-checkable", func() {
-				and := graph.NewAndIterator()
-				and.AddSubIterator(ts.GetTripleIterator("s", ts.GetIdFor("B")))
-				and.AddSubIterator(it)
-
-				expected := []string{
-					graph.MakeTriple("B", "follows", "F", "").ToString(),
-				}
-				actual := extractTripleFromIterator(ts, and)
-				sort.Strings(actual)
-				sort.Strings(expected)
-				So(actual, ShouldResemble, expected)
-			})
-
-		})
-
-		Convey("Can create a predicate iterator", func() {
-			it = ts.GetTripleIterator("p", ts.GetIdFor("status"))
-
-			Convey("Containing the right things", func() {
-				expected := []string{
-					graph.MakeTriple("B", "status", "cool", "status_graph").ToString(),
-					graph.MakeTriple("D", "status", "cool", "status_graph").ToString(),
-					graph.MakeTriple("G", "status", "cool", "status_graph").ToString(),
-				}
-				actual := extractTripleFromIterator(ts, it)
-				sort.Strings(actual)
-				sort.Strings(expected)
-				So(actual, ShouldResemble, expected)
-			})
-
-		})
-
-		Convey("Can create a provenance iterator", func() {
-			it = ts.GetTripleIterator("c", ts.GetIdFor("status_graph"))
-
-			Convey("Containing the right things", func() {
-				expected := []string{
-					graph.MakeTriple("B", "status", "cool", "status_graph").ToString(),
-					graph.MakeTriple("D", "status", "cool", "status_graph").ToString(),
-					graph.MakeTriple("G", "status", "cool", "status_graph").ToString(),
-				}
-				actual := extractTripleFromIterator(ts, it)
-				sort.Strings(actual)
-				sort.Strings(expected)
-				So(actual, ShouldResemble, expected)
-			})
-
-			Convey("Can be cross-checked", func() {
-				and := graph.NewAndIterator()
-				// Order is important
-				and.AddSubIterator(ts.GetTripleIterator("s", ts.GetIdFor("B")))
-				and.AddSubIterator(it)
-
-				expected := []string{
-					graph.MakeTriple("B", "status", "cool", "status_graph").ToString(),
-				}
-				actual := extractTripleFromIterator(ts, and)
-				So(actual, ShouldResemble, expected)
-			})
-
-			Convey("Can check against other iterators", func() {
-				and := graph.NewAndIterator()
-				// Order is important
-				and.AddSubIterator(it)
-				and.AddSubIterator(ts.GetTripleIterator("s", ts.GetIdFor("B")))
-
-				expected := []string{
-					graph.MakeTriple("B", "status", "cool", "status_graph").ToString(),
-				}
-				actual := extractTripleFromIterator(ts, and)
-				So(actual, ShouldResemble, expected)
-			})
-			Reset(func() {
-				it.Reset()
-			})
-
-		})
-
-		Reset(func() {
-			ts.Close()
-		})
-
-	})
-
-}
-
-func TestOptimize(t *testing.T) {
-	var ts *LevelDBTripleStore
-	var lto graph.Iterator
-	var tmpDir string
-
-	Convey("Given a prepared database", t, func() {
-		tmpDir, _ = ioutil.TempDir(os.TempDir(), "cayley_test")
-		t.Log(tmpDir)
-		defer os.RemoveAll(tmpDir)
-		ok := CreateNewLevelDB(tmpDir)
-		So(ok, ShouldBeTrue)
-		ts = NewDefaultLevelDBTripleStore(tmpDir, nil)
-		ts.AddTripleSet(makeTripleSet())
-
-		Convey("With an linksto-fixed pair", func() {
-			fixed := ts.MakeFixed()
-			fixed.AddValue(ts.GetIdFor("F"))
-			fixed.AddTag("internal")
-			lto = graph.NewLinksToIterator(ts, fixed, "o")
-
-			Convey("Creates an appropriate iterator", func() {
-				oldIt := lto.Clone()
-				newIt, ok := lto.Optimize()
-				So(ok, ShouldBeTrue)
-				So(newIt.Type(), ShouldEqual, "leveldb")
-
-				Convey("Containing the right things", func() {
-					afterOp := extractTripleFromIterator(ts, newIt)
-					beforeOp := extractTripleFromIterator(ts, oldIt)
-					sort.Strings(afterOp)
-					sort.Strings(beforeOp)
-					So(afterOp, ShouldResemble, beforeOp)
-				})
-
-				Convey("With the correct tags", func() {
-					oldIt.Next()
-					newIt.Next()
-					oldResults := make(map[string]graph.TSVal)
-					oldIt.TagResults(&oldResults)
-					newResults := make(map[string]graph.TSVal)
-					oldIt.TagResults(&newResults)
-					So(newResults, ShouldResemble, oldResults)
-				})
-
-			})
-
-		})
-
-	})
-
-}