diff --git a/Makefile b/Makefile deleted file mode 100644 index 3a28444..0000000 --- a/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2014 The Cayley Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -default: build - -build: - ./make.sh build - -deps: - ./make.sh deps - -test: - ls ./src | grep -v "\." | sed 's/\///g' | xargs go test -cover - -convey: - ./bin/goconvey --depth=2 - - diff --git a/activate.sh b/activate.sh deleted file mode 100644 index 6d8cabd..0000000 --- a/activate.sh +++ /dev/null @@ -1,9 +0,0 @@ -# Absolute path this script is in. /home/user/bin -cd "`dirname '${BASH_SOURCE:-$0}'`" -SCRIPTPATH="`pwd`" -echo $dir -cd - > /dev/null - -export GOPATH=$SCRIPTPATH -#export GOOS="linux" -#export GOARCH="amd64" diff --git a/appengine/cayley-appengine.go b/appengine/cayley-appengine.go index 77a0e23..c915932 100644 --- a/appengine/cayley-appengine.go +++ b/appengine/cayley-appengine.go @@ -15,22 +15,24 @@ package cayleyappengine import ( - "cayley_config" - "cayley_http" - "github.com/barakmich/glog" - "graph" - "graph_memstore" - "nquads" "os" + + "github.com/barakmich/glog" + + cfg "github.com/google/cayley/config" + "github.com/google/cayley/graph" + "github.com/google/cayley/graph/memstore" + "github.com/google/cayley/http" + "github.com/google/cayley/nquads" ) func init() { glog.SetToStderr(true) - config := cayley_config.ParseConfigFromFile("cayley_appengine.cfg") - ts := graph_memstore.NewMemTripleStore() + config := cfg.ParseConfigFromFile("cayley_appengine.cfg") + ts := memstore.NewMemTripleStore() glog.Errorln(config) LoadTriplesFromFileInto(ts, config.DatabasePath, config.LoadSize) - cayley_http.SetupRoutes(ts, config) + http.SetupRoutes(ts, config) } func ReadTriplesFromFile(c chan *graph.Triple, tripleFile string) { diff --git a/src/cayley_config/cayley-config.go b/config/config.go similarity index 100% rename from src/cayley_config/cayley-config.go rename to config/config.go diff --git a/src/graph/all-iterator.go b/graph/all-iterator.go similarity index 100% rename from src/graph/all-iterator.go rename to graph/all-iterator.go diff --git a/src/graph/and-iterator-optimize.go b/graph/and-iterator-optimize.go similarity index 100% rename from src/graph/and-iterator-optimize.go rename to graph/and-iterator-optimize.go diff --git a/src/graph/and-iterator-optimize_test.go b/graph/and-iterator-optimize_test.go similarity index 100% rename from src/graph/and-iterator-optimize_test.go rename to graph/and-iterator-optimize_test.go diff --git a/src/graph/and-iterator.go b/graph/and-iterator.go similarity index 100% rename from src/graph/and-iterator.go rename to graph/and-iterator.go diff --git a/src/graph/and-iterator_test.go b/graph/and-iterator_test.go similarity index 100% rename from src/graph/and-iterator_test.go rename to graph/and-iterator_test.go diff --git a/src/graph/fixed-iterator.go b/graph/fixed-iterator.go similarity index 100% rename from src/graph/fixed-iterator.go rename to graph/fixed-iterator.go diff --git a/src/graph/hasa-iterator.go b/graph/hasa-iterator.go similarity index 100% rename from src/graph/hasa-iterator.go rename to graph/hasa-iterator.go diff --git a/src/graph/iterator.go b/graph/iterator.go similarity index 100% rename from src/graph/iterator.go rename to graph/iterator.go diff --git a/src/graph/linksto-iterator.go b/graph/linksto-iterator.go similarity index 100% rename from src/graph/linksto-iterator.go rename to graph/linksto-iterator.go diff --git a/src/graph/linksto-iterator_test.go b/graph/linksto-iterator_test.go similarity index 100% rename from src/graph/linksto-iterator_test.go rename to graph/linksto-iterator_test.go diff --git a/src/graph_memstore/llrb-iterator.go b/graph/memstore/llrb-iterator.go similarity index 98% rename from src/graph_memstore/llrb-iterator.go rename to graph/memstore/llrb-iterator.go index fed4272..692a3c6 100644 --- a/src/graph_memstore/llrb-iterator.go +++ b/graph/memstore/llrb-iterator.go @@ -12,14 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_memstore +package memstore import ( "fmt" - "github.com/petar/GoLLRB/llrb" - "graph" "math" "strings" + + "github.com/petar/GoLLRB/llrb" + + "github.com/google/cayley/graph" ) type LlrbIterator struct { diff --git a/src/graph_memstore/memstore-all-iterator.go b/graph/memstore/memstore-all-iterator.go similarity index 95% rename from src/graph_memstore/memstore-all-iterator.go rename to graph/memstore/memstore-all-iterator.go index 21245a0..99cf734 100644 --- a/src/graph_memstore/memstore-all-iterator.go +++ b/graph/memstore/memstore-all-iterator.go @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_memstore +package memstore import ( - "graph" + "github.com/google/cayley/graph" ) type MemstoreAllIterator struct { diff --git a/src/graph_memstore/memtriplestore-iterator-optimize.go b/graph/memstore/memtriplestore-iterator-optimize.go similarity index 96% rename from src/graph_memstore/memtriplestore-iterator-optimize.go rename to graph/memstore/memtriplestore-iterator-optimize.go index 2fd913b..7c895fc 100644 --- a/src/graph_memstore/memtriplestore-iterator-optimize.go +++ b/graph/memstore/memtriplestore-iterator-optimize.go @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_memstore +package memstore import ( - "graph" + "github.com/google/cayley/graph" ) func (ts *MemTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { diff --git a/src/graph_memstore/memtriplestore.go b/graph/memstore/memtriplestore.go similarity index 99% rename from src/graph_memstore/memtriplestore.go rename to graph/memstore/memtriplestore.go index 83f5c17..7c7882e 100644 --- a/src/graph_memstore/memtriplestore.go +++ b/graph/memstore/memtriplestore.go @@ -12,13 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_memstore +package memstore import ( "fmt" + "github.com/barakmich/glog" + "github.com/google/cayley/graph" + "github.com/petar/GoLLRB/llrb" - "graph" ) type TripleDirectionIndex struct { diff --git a/src/graph_memstore/memtriplestore_test.go b/graph/memstore/memtriplestore_test.go similarity index 98% rename from src/graph_memstore/memtriplestore_test.go rename to graph/memstore/memtriplestore_test.go index 0321028..71d7016 100644 --- a/src/graph_memstore/memtriplestore_test.go +++ b/graph/memstore/memtriplestore_test.go @@ -12,13 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_memstore +package memstore import ( - . "github.com/smartystreets/goconvey/convey" - "graph" "sort" "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/google/cayley/graph" ) func TestMemstore(t *testing.T) { diff --git a/src/graph_memstore/testing_memstore.go b/graph/memstore/testing_memstore.go similarity index 96% rename from src/graph_memstore/testing_memstore.go rename to graph/memstore/testing_memstore.go index d32d838..b33c95f 100644 --- a/src/graph_memstore/testing_memstore.go +++ b/graph/memstore/testing_memstore.go @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_memstore +package memstore -import "graph" +import "github.com/google/cayley/graph" // +---+ +---+ // | A |------- ->| F |<-- diff --git a/src/graph/mock_ts.go b/graph/mock_ts.go similarity index 100% rename from src/graph/mock_ts.go rename to graph/mock_ts.go diff --git a/src/graph_mongo/lru.go b/graph/mongo/lru.go similarity index 98% rename from src/graph_mongo/lru.go rename to graph/mongo/lru.go index c856a5a..90220e4 100644 --- a/src/graph_mongo/lru.go +++ b/graph/mongo/lru.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_mongo +package mongo import ( "container/list" diff --git a/src/graph_mongo/mongo-iterator.go b/graph/mongo/mongo-iterator.go similarity index 98% rename from src/graph_mongo/mongo-iterator.go rename to graph/mongo/mongo-iterator.go index 6d7f84a..addcfb0 100644 --- a/src/graph_mongo/mongo-iterator.go +++ b/graph/mongo/mongo-iterator.go @@ -12,15 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_mongo +package mongo import ( "fmt" + "strings" + "github.com/barakmich/glog" - "graph" "labix.org/v2/mgo" "labix.org/v2/mgo/bson" - "strings" + + "github.com/google/cayley/graph" ) type MongoIterator struct { diff --git a/src/graph_mongo/mongo-triplestore-iterator-optimize.go b/graph/mongo/mongo-triplestore-iterator-optimize.go similarity index 96% rename from src/graph_mongo/mongo-triplestore-iterator-optimize.go rename to graph/mongo/mongo-triplestore-iterator-optimize.go index ea35602..d10bc22 100644 --- a/src/graph_mongo/mongo-triplestore-iterator-optimize.go +++ b/graph/mongo/mongo-triplestore-iterator-optimize.go @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_mongo +package mongo import ( - "graph" + "github.com/google/cayley/graph" ) func (ts *MongoTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { diff --git a/src/graph_mongo/mongo-triplestore.go b/graph/mongo/mongo-triplestore.go similarity index 99% rename from src/graph_mongo/mongo-triplestore.go rename to graph/mongo/mongo-triplestore.go index fa4ea25..917ab4d 100644 --- a/src/graph_mongo/mongo-triplestore.go +++ b/graph/mongo/mongo-triplestore.go @@ -12,17 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_mongo +package mongo import ( "crypto/sha1" "encoding/hex" - "github.com/barakmich/glog" - "graph" "hash" + "log" + "labix.org/v2/mgo" "labix.org/v2/mgo/bson" - "log" + + "github.com/barakmich/glog" + "github.com/google/cayley/graph" ) const DefaultDBName = "cayley" diff --git a/src/graph/optional-iterator.go b/graph/optional-iterator.go similarity index 100% rename from src/graph/optional-iterator.go rename to graph/optional-iterator.go diff --git a/src/graph/or-iterator.go b/graph/or-iterator.go similarity index 100% rename from src/graph/or-iterator.go rename to graph/or-iterator.go diff --git a/src/graph/or-iterator_test.go b/graph/or-iterator_test.go similarity index 100% rename from src/graph/or-iterator_test.go rename to graph/or-iterator_test.go diff --git a/src/graph/query-shape.go b/graph/query-shape.go similarity index 100% rename from src/graph/query-shape.go rename to graph/query-shape.go diff --git a/src/graph/query-shape_test.go b/graph/query-shape_test.go similarity index 100% rename from src/graph/query-shape_test.go rename to graph/query-shape_test.go diff --git a/src/graph/result-tree-evaluator.go b/graph/result-tree-evaluator.go similarity index 100% rename from src/graph/result-tree-evaluator.go rename to graph/result-tree-evaluator.go diff --git a/src/graph/result-tree-evaluator_test.go b/graph/result-tree-evaluator_test.go similarity index 100% rename from src/graph/result-tree-evaluator_test.go rename to graph/result-tree-evaluator_test.go diff --git a/src/graph/session.go b/graph/session.go similarity index 100% rename from src/graph/session.go rename to graph/session.go diff --git a/src/graph_sexp/parser.go b/graph/sexp/parser.go similarity index 99% rename from src/graph_sexp/parser.go rename to graph/sexp/parser.go index 529b854..2efcf49 100644 --- a/src/graph_sexp/parser.go +++ b/graph/sexp/parser.go @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_sexp +package sexp import ( "github.com/badgerodon/peg" - "graph" + + "github.com/google/cayley/graph" ) func BuildIteratorTreeForQuery(ts graph.TripleStore, query string) graph.Iterator { diff --git a/src/graph_sexp/parser_test.go b/graph/sexp/parser_test.go similarity index 93% rename from src/graph_sexp/parser_test.go rename to graph/sexp/parser_test.go index 90233c5..f72239c 100644 --- a/src/graph_sexp/parser_test.go +++ b/graph/sexp/parser_test.go @@ -12,13 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_sexp +package sexp import ( - . "github.com/smartystreets/goconvey/convey" - "graph" - "graph_memstore" "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/google/cayley/graph" + "github.com/google/cayley/graph/memstore" ) func TestBadParse(t *testing.T) { @@ -30,7 +32,7 @@ func TestBadParse(t *testing.T) { func TestParseSexpWithMemstore(t *testing.T) { Convey("With a Memstore", t, func() { - ts := graph_memstore.NewMemTripleStore() + ts := memstore.NewMemTripleStore() Convey("It should parse an empty query", func() { it := BuildIteratorTreeForQuery(ts, "()") @@ -62,7 +64,7 @@ func TestParseSexpWithMemstore(t *testing.T) { } func TestTreeConstraintParse(t *testing.T) { - ts := graph_memstore.NewMemTripleStore() + ts := memstore.NewMemTripleStore() ts.AddTriple(graph.MakeTriple("i", "like", "food", "")) ts.AddTriple(graph.MakeTriple("food", "is", "good", "")) query := "(\"i\"\n" + @@ -82,7 +84,7 @@ func TestTreeConstraintParse(t *testing.T) { } func TestTreeConstraintTagParse(t *testing.T) { - ts := graph_memstore.NewMemTripleStore() + ts := memstore.NewMemTripleStore() ts.AddTriple(graph.MakeTriple("i", "like", "food", "")) ts.AddTriple(graph.MakeTriple("food", "is", "good", "")) query := "(\"i\"\n" + @@ -102,7 +104,7 @@ func TestTreeConstraintTagParse(t *testing.T) { } func TestMultipleConstraintParse(t *testing.T) { - ts := graph_memstore.NewMemTripleStore() + ts := memstore.NewMemTripleStore() ts.AddTriple(graph.MakeTriple("i", "like", "food", "")) ts.AddTriple(graph.MakeTriple("i", "like", "beer", "")) ts.AddTriple(graph.MakeTriple("you", "like", "beer", "")) diff --git a/src/graph_sexp/sexp-session.go b/graph/sexp/sexp-session.go similarity index 98% rename from src/graph_sexp/sexp-session.go rename to graph/sexp/sexp-session.go index 161e5ae..7065e02 100644 --- a/src/graph_sexp/sexp-session.go +++ b/graph/sexp/sexp-session.go @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package graph_sexp +package sexp // Defines a running session of the sexp query language. import ( "errors" "fmt" - "graph" "sort" + + "github.com/google/cayley/graph" ) type SexpSession struct { diff --git a/src/graph/triple.go b/graph/triple.go similarity index 100% rename from src/graph/triple.go rename to graph/triple.go diff --git a/src/graph/triplestore.go b/graph/triplestore.go similarity index 100% rename from src/graph/triplestore.go rename to graph/triplestore.go diff --git a/src/graph/value-comparison-iterator.go b/graph/value-comparison-iterator.go similarity index 100% rename from src/graph/value-comparison-iterator.go rename to graph/value-comparison-iterator.go diff --git a/src/graph/value-comparison-iterator_test.go b/graph/value-comparison-iterator_test.go similarity index 100% rename from src/graph/value-comparison-iterator_test.go rename to graph/value-comparison-iterator_test.go diff --git a/src/gremlin/gremlin-build-iterator.go b/gremlin/gremlin-build-iterator.go similarity index 99% rename from src/gremlin/gremlin-build-iterator.go rename to gremlin/gremlin-build-iterator.go index 823c568..b6e9a45 100644 --- a/src/gremlin/gremlin-build-iterator.go +++ b/gremlin/gremlin-build-iterator.go @@ -15,10 +15,12 @@ package gremlin import ( + "strconv" + "github.com/barakmich/glog" "github.com/robertkrimen/otto" - "graph" - "strconv" + + "github.com/google/cayley/graph" ) func getStrings(obj *otto.Object, field string) []string { diff --git a/src/gremlin/gremlin-env.go b/gremlin/gremlin-env.go similarity index 100% rename from src/gremlin/gremlin-env.go rename to gremlin/gremlin-env.go diff --git a/src/gremlin/gremlin-finals.go b/gremlin/gremlin-finals.go similarity index 99% rename from src/gremlin/gremlin-finals.go rename to gremlin/gremlin-finals.go index 08bdd6d..f8c978d 100644 --- a/src/gremlin/gremlin-finals.go +++ b/gremlin/gremlin-finals.go @@ -17,7 +17,8 @@ package gremlin import ( "github.com/barakmich/glog" "github.com/robertkrimen/otto" - "graph" + + "github.com/google/cayley/graph" ) const GremlinTopResultTag = "id" diff --git a/src/gremlin/gremlin-functional_test.go b/gremlin/gremlin-functional_test.go similarity index 98% rename from src/gremlin/gremlin-functional_test.go rename to gremlin/gremlin-functional_test.go index f516965..f6c65fb 100644 --- a/src/gremlin/gremlin-functional_test.go +++ b/gremlin/gremlin-functional_test.go @@ -15,10 +15,12 @@ package gremlin import ( - . "github.com/smartystreets/goconvey/convey" - "graph_memstore" "sort" "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/google/cayley/graph/memstore" ) // +---+ +---+ @@ -34,8 +36,8 @@ import ( // func buildTripleStore() *GremlinSession { - ts := graph_memstore.MakeTestingMemstore() - return NewGremlinSession(ts, -1) + ts := memstore.MakeTestingMemstore() + return NewGremlinSession(ts, -1, false) } func shouldBeUnordered(actual interface{}, expected ...interface{}) string { diff --git a/src/gremlin/gremlin-session.go b/gremlin/gremlin-session.go similarity index 99% rename from src/gremlin/gremlin-session.go rename to gremlin/gremlin-session.go index 25b3fbb..a0b0483 100644 --- a/src/gremlin/gremlin-session.go +++ b/gremlin/gremlin-session.go @@ -17,10 +17,12 @@ package gremlin import ( "errors" "fmt" - "github.com/robertkrimen/otto" - "graph" "sort" "time" + + "github.com/robertkrimen/otto" + + "github.com/google/cayley/graph" ) type GremlinSession struct { diff --git a/src/gremlin/gremlin-traversals.go b/gremlin/gremlin-traversals.go similarity index 100% rename from src/gremlin/gremlin-traversals.go rename to gremlin/gremlin-traversals.go diff --git a/src/gremlin/gremlin_test.nt b/gremlin/gremlin_test.nt similarity index 100% rename from src/gremlin/gremlin_test.nt rename to gremlin/gremlin_test.nt diff --git a/src/cayley_http/cayley-http-docs.go b/http/cayley-http-docs.go similarity index 99% rename from src/cayley_http/cayley-http-docs.go rename to http/cayley-http-docs.go index d90e8d9..45e747d 100644 --- a/src/cayley_http/cayley-http-docs.go +++ b/http/cayley-http-docs.go @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cayley_http +package http import ( "fmt" - "github.com/julienschmidt/httprouter" - "github.com/russross/blackfriday" "io/ioutil" "net/http" "os" + + "github.com/julienschmidt/httprouter" + "github.com/russross/blackfriday" ) type DocRequestHandler struct { diff --git a/src/cayley_http/cayley-http-query.go b/http/cayley-http-query.go similarity index 97% rename from src/cayley_http/cayley-http-query.go rename to http/cayley-http-query.go index c3a91e6..4a99dd0 100644 --- a/src/cayley_http/cayley-http-query.go +++ b/http/cayley-http-query.go @@ -12,17 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cayley_http +package http import ( "encoding/json" "fmt" - "github.com/julienschmidt/httprouter" - "graph" - "gremlin" "io/ioutil" - "mql" "net/http" + + "github.com/julienschmidt/httprouter" + + "github.com/google/cayley/graph" + "github.com/google/cayley/gremlin" + "github.com/google/cayley/mql" ) type SuccessQueryWrapper struct { diff --git a/src/cayley_http/cayley-http-write.go b/http/cayley-http-write.go similarity index 97% rename from src/cayley_http/cayley-http-write.go rename to http/cayley-http-write.go index a6fdcb6..20fb60d 100644 --- a/src/cayley_http/cayley-http-write.go +++ b/http/cayley-http-write.go @@ -12,19 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cayley_http +package http import ( "encoding/json" "errors" "fmt" - "github.com/barakmich/glog" - "github.com/julienschmidt/httprouter" - "graph" "io/ioutil" "net/http" - "nquads" "strconv" + + "github.com/barakmich/glog" + "github.com/julienschmidt/httprouter" + + "github.com/google/cayley/graph" + "github.com/google/cayley/nquads" ) func ParseJsonToTripleList(jsonBody []byte) ([]*graph.Triple, error) { diff --git a/src/cayley_http/cayley-http.go b/http/cayley-http.go similarity index 97% rename from src/cayley_http/cayley-http.go rename to http/cayley-http.go index 35acab1..68d50bb 100644 --- a/src/cayley_http/cayley-http.go +++ b/http/cayley-http.go @@ -12,17 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cayley_http +package http import ( - cfg "cayley_config" "fmt" - "github.com/barakmich/glog" - "github.com/julienschmidt/httprouter" - "graph" "html/template" "net/http" "time" + + "github.com/barakmich/glog" + "github.com/julienschmidt/httprouter" + + cfg "github.com/google/cayley/config" + "github.com/google/cayley/graph" ) type ResponseHandler func(http.ResponseWriter, *http.Request, httprouter.Params) int diff --git a/src/cayley_http/cayley-http_test.go b/http/cayley-http_test.go similarity index 98% rename from src/cayley_http/cayley-http_test.go rename to http/cayley-http_test.go index 941b6da..d59749f 100644 --- a/src/cayley_http/cayley-http_test.go +++ b/http/cayley-http_test.go @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cayley_http +package http import ( - . "github.com/smartystreets/goconvey/convey" "testing" + . "github.com/smartystreets/goconvey/convey" ) func TestParseJSONOkay(t *testing.T) { diff --git a/src/cayley_cmd/cayley-init.go b/init.go similarity index 81% rename from src/cayley_cmd/cayley-init.go rename to init.go index 74226c3..54a66cd 100644 --- a/src/cayley_cmd/cayley-init.go +++ b/init.go @@ -15,9 +15,9 @@ package cayley import ( - cfg "cayley_config" - "graph_leveldb" - "graph_mongo" + cfg "github.com/google/cayley/config" + "github.com/google/cayley/graph/leveldb" + "github.com/google/cayley/graph/mongo" ) func CayleyInit(config *cfg.CayleyConfig, triplePath string) bool { @@ -25,9 +25,9 @@ func CayleyInit(config *cfg.CayleyConfig, triplePath string) bool { dbpath := config.DatabasePath switch config.DatabaseType { case "mongo", "mongodb": - created = graph_mongo.CreateNewMongoGraph(dbpath, config.DatabaseOptions) + created = mongo.CreateNewMongoGraph(dbpath, config.DatabaseOptions) case "leveldb": - created = graph_leveldb.CreateNewLevelDB(dbpath) + created = leveldb.CreateNewLevelDB(dbpath) case "mem": return true } diff --git a/src/cayley_cmd/cayley-load.go b/load.go similarity index 88% rename from src/cayley_cmd/cayley-load.go rename to load.go index 19b242c..0d00e6d 100644 --- a/src/cayley_cmd/cayley-load.go +++ b/load.go @@ -15,19 +15,21 @@ package cayley import ( - cfg "cayley_config" - "github.com/barakmich/glog" - "graph" - "graph_mongo" - "nquads" "os" + + "github.com/barakmich/glog" + + cfg "github.com/google/cayley/config" + "github.com/google/cayley/graph" + "github.com/google/cayley/graph/mongo" + "github.com/google/cayley/nquads" ) func CayleyLoad(ts graph.TripleStore, config *cfg.CayleyConfig, triplePath string, firstTime bool) { switch config.DatabaseType { case "mongo", "mongodb": if firstTime { - loadMongo(ts.(*graph_mongo.MongoTripleStore), triplePath) + loadMongo(ts.(*mongo.MongoTripleStore), triplePath) } else { LoadTriplesFromFileInto(ts, triplePath, config.LoadSize) } @@ -41,7 +43,7 @@ func CayleyLoad(ts graph.TripleStore, config *cfg.CayleyConfig, triplePath strin } -func loadMongo(ts *graph_mongo.MongoTripleStore, path string) { +func loadMongo(ts *mongo.MongoTripleStore, path string) { tChan := make(chan *graph.Triple) go ReadTriplesFromFile(tChan, path) ts.BulkLoad(tChan) diff --git a/make.sh b/make.sh deleted file mode 100755 index 395a277..0000000 --- a/make.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Cayley Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -cd "`dirname '$0'`" -SCRIPTPATH="`pwd`" -cd - > /dev/null - -export GOPATH=$SCRIPTPATH -export GOBIN= - -function deps { -echo "Fetching dependencies to $SCRIPTPATH..." -printf " (00/15)\r" - go get -u -t github.com/smartystreets/goconvey -printf "# (01/15)\r" - go get -u github.com/badgerodon/peg -printf "## (02/15)\r" - go get -u github.com/barakmich/glog -printf "#### (03/15)\r" - go get -u github.com/julienschmidt/httprouter -printf "##### (04/15)\r" - go get -u github.com/petar/GoLLRB/llrb -printf "###### (05/15)\r" - go get -u github.com/robertkrimen/otto -printf "####### (06/15)\r" - go get -u github.com/stretchrcom/testify -printf "######## (07/15)\r" - go get -u github.com/syndtr/goleveldb/leveldb -printf "######### (08/15)\r" - go get -u github.com/syndtr/goleveldb/leveldb/cache -printf "########## (09/15)\r" - go get -u github.com/syndtr/goleveldb/leveldb/iterator -printf "########### (10/15)\r" - go get -u github.com/syndtr/goleveldb/leveldb/opt -printf "############ (11/15)\r" - go get -u github.com/syndtr/goleveldb/leveldb/util -printf "############# (12/15)\r" - go get -u labix.org/v2/mgo -printf "############## (13/15)\r" - go get -u labix.org/v2/mgo/bson -printf "############### (14/15)\r" - go get -u github.com/russross/blackfriday -printf "################ (15/15)\r" -printf "\n" -} - -function build { - go build cayley -} - -$1 diff --git a/src/mql/mql-build-iterator.go b/mql/mql-build-iterator.go similarity index 99% rename from src/mql/mql-build-iterator.go rename to mql/mql-build-iterator.go index 8f7d8d1..6273696 100644 --- a/src/mql/mql-build-iterator.go +++ b/mql/mql-build-iterator.go @@ -17,10 +17,11 @@ package mql import ( "errors" "fmt" - "graph" "log" "math" "strings" + + "github.com/google/cayley/graph" ) func (m *MqlQuery) buildFixed(s string) graph.Iterator { diff --git a/src/mql/mql-fill.go b/mql/mql-fill.go similarity index 98% rename from src/mql/mql-fill.go rename to mql/mql-fill.go index 72fb2b4..26de32a 100644 --- a/src/mql/mql-fill.go +++ b/mql/mql-fill.go @@ -15,8 +15,9 @@ package mql import ( - "graph" "sort" + + "github.com/google/cayley/graph" ) func (m *MqlQuery) treeifyResult(tags map[string]graph.TSVal) map[MqlResultPath]string { diff --git a/src/mql/mql-functional_test.go b/mql/mql-functional_test.go similarity index 98% rename from src/mql/mql-functional_test.go rename to mql/mql-functional_test.go index 396e75c..97c2eac 100644 --- a/src/mql/mql-functional_test.go +++ b/mql/mql-functional_test.go @@ -16,9 +16,11 @@ package mql import ( "encoding/json" - . "github.com/smartystreets/goconvey/convey" - "graph_memstore" "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/google/cayley/graph/memstore" ) // +---+ +---+ @@ -34,7 +36,7 @@ import ( // func buildTripleStore() *MqlSession { - ts := graph_memstore.MakeTestingMemstore() + ts := memstore.MakeTestingMemstore() return NewMqlSession(ts) } diff --git a/src/mql/mql-query.go b/mql/mql-query.go similarity index 98% rename from src/mql/mql-query.go rename to mql/mql-query.go index a56ad5f..66d8179 100644 --- a/src/mql/mql-query.go +++ b/mql/mql-query.go @@ -16,8 +16,9 @@ package mql import ( "fmt" - "graph" "strings" + + "github.com/google/cayley/graph" ) type MqlPath string diff --git a/src/mql/mql-session.go b/mql/mql-session.go similarity index 98% rename from src/mql/mql-session.go rename to mql/mql-session.go index e08e75e..96d88bf 100644 --- a/src/mql/mql-session.go +++ b/mql/mql-session.go @@ -17,9 +17,11 @@ package mql import ( "encoding/json" "fmt" - "github.com/barakmich/glog" - "graph" "sort" + + "github.com/barakmich/glog" + + "github.com/google/cayley/graph" ) type MqlSession struct { diff --git a/src/nquads/nquads.go b/nquads/nquads.go similarity index 99% rename from src/nquads/nquads.go rename to nquads/nquads.go index 63ec89c..f4032b3 100644 --- a/src/nquads/nquads.go +++ b/nquads/nquads.go @@ -16,10 +16,12 @@ package nquads import ( "bufio" - "github.com/barakmich/glog" - "graph" "io" "strings" + + "github.com/barakmich/glog" + + "github.com/google/cayley/graph" ) func isWhitespace(s uint8) bool { diff --git a/src/nquads/nquads_test.go b/nquads/nquads_test.go similarity index 99% rename from src/nquads/nquads_test.go rename to nquads/nquads_test.go index 47e02a5..f5b61ee 100644 --- a/src/nquads/nquads_test.go +++ b/nquads/nquads_test.go @@ -15,9 +15,11 @@ package nquads import ( - . "github.com/smartystreets/goconvey/convey" - "graph" "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/google/cayley/graph" ) func TestParsingNTriples(t *testing.T) { diff --git a/src/cayley_cmd/cayley-open.go b/open.go similarity index 72% rename from src/cayley_cmd/cayley-open.go rename to open.go index c262571..442fa9a 100644 --- a/src/cayley_cmd/cayley-open.go +++ b/open.go @@ -15,24 +15,24 @@ package cayley import ( - "graph" - - cfg "cayley_config" "github.com/barakmich/glog" - "graph_leveldb" - "graph_memstore" - "graph_mongo" + + cfg "github.com/google/cayley/config" + "github.com/google/cayley/graph" + "github.com/google/cayley/graph/leveldb" + "github.com/google/cayley/graph/memstore" + "github.com/google/cayley/graph/mongo" ) func OpenTSFromConfig(config *cfg.CayleyConfig) graph.TripleStore { glog.Infof("Opening database \"%s\" at %s", config.DatabaseType, config.DatabasePath) switch config.DatabaseType { case "mongo", "mongodb": - return graph_mongo.NewMongoTripleStore(config.DatabasePath, config.DatabaseOptions) + return mongo.NewMongoTripleStore(config.DatabasePath, config.DatabaseOptions) case "leveldb": - return graph_leveldb.NewDefaultLevelDBTripleStore(config.DatabasePath, config.DatabaseOptions) + return leveldb.NewDefaultLevelDBTripleStore(config.DatabasePath, config.DatabaseOptions) case "mem": - ts := graph_memstore.NewMemTripleStore() + ts := memstore.NewMemTripleStore() CayleyLoad(ts, config, config.DatabasePath, true) return ts } diff --git a/src/cayley_cmd/cayley-repl.go b/repl.go similarity index 93% rename from src/cayley_cmd/cayley-repl.go rename to repl.go index 9c4757a..08a4d0e 100644 --- a/src/cayley_cmd/cayley-repl.go +++ b/repl.go @@ -16,17 +16,18 @@ package cayley import ( "bufio" - cfg "cayley_config" "fmt" - "graph" - sexp "graph_sexp" - "gremlin" "io" - "mql" - "nquads" "os" "strings" "time" + + cfg "github.com/google/cayley/config" + "github.com/google/cayley/graph" + "github.com/google/cayley/graph/sexp" + "github.com/google/cayley/gremlin" + "github.com/google/cayley/mql" + "github.com/google/cayley/nquads" ) func trace(s string) (string, time.Time) { diff --git a/src/cayley/main.go b/src/cayley/main.go deleted file mode 100644 index c2354d9..0000000 --- a/src/cayley/main.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - cayley "cayley_cmd" - cfg "cayley_config" - cayley_http "cayley_http" - "flag" - "fmt" - "github.com/barakmich/glog" - "graph" - "os" - "runtime" -) - -var tripleFile = flag.String("triples", "", "Triple File to load before going to REPL.") -var cpuprofile = flag.String("prof", "", "Output profiling file.") -var queryLanguage = flag.String("query_lang", "gremlin", "Use this parser as the query language.") -var configFile = flag.String("config", "", "Path to an explicit configuration file.") - -func Usage() { - fmt.Println("Cayley is a graph store and graph query layer.\n") - fmt.Println("Usage:") - fmt.Println(" cayley COMMAND [flags]\n") - fmt.Println("Commands:") - fmt.Println(" init\tCreate an empty database.") - fmt.Println(" load\tBulk-load a triple file into the database.") - fmt.Println(" http\tServe an HTTP endpoint on the given host and port.") - fmt.Println(" repl\tDrop into a REPL of the given query language.") - fmt.Println("\nFlags:") - flag.Parse() - flag.PrintDefaults() -} - -func main() { - // No command? It's time for usage. - if len(os.Args) == 1 { - Usage() - os.Exit(1) - } - cmd := os.Args[1] - newargs := make([]string, 0) - newargs = append(newargs, os.Args[0]) - newargs = append(newargs, os.Args[2:]...) - os.Args = newargs - flag.Parse() - var ts graph.TripleStore - config := cfg.ParseConfigFromFlagsAndFile(*configFile) - if os.Getenv("GOMAXPROCS") == "" { - runtime.GOMAXPROCS(runtime.NumCPU()) - glog.Infoln("Setting GOMAXPROCS to", runtime.NumCPU()) - } else { - glog.Infoln("GOMAXPROCS currently", os.Getenv("GOMAXPROCS"), " -- not adjusting") - } - switch cmd { - case "init": - cayley.CayleyInit(config, *tripleFile) - case "load": - ts = cayley.OpenTSFromConfig(config) - cayley.CayleyLoad(ts, config, *tripleFile, false) - ts.Close() - case "repl": - ts = cayley.OpenTSFromConfig(config) - cayley.CayleyRepl(ts, *queryLanguage, config) - ts.Close() - case "http": - ts = cayley.OpenTSFromConfig(config) - cayley_http.CayleyHTTP(ts, config) - ts.Close() - default: - fmt.Println("No command", cmd) - flag.Usage() - } -} diff --git a/src/graph_leveldb/leveldb-all-iterator.go b/src/graph_leveldb/leveldb-all-iterator.go deleted file mode 100644 index 78f9bdc..0000000 --- a/src/graph_leveldb/leveldb-all-iterator.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph_leveldb - -import ( - "bytes" - "fmt" - leveldb_it "github.com/syndtr/goleveldb/leveldb/iterator" - leveldb_opt "github.com/syndtr/goleveldb/leveldb/opt" - "graph" - "strings" -) - -type LevelDBAllIterator struct { - graph.BaseIterator - prefix []byte - dir string - open bool - it leveldb_it.Iterator - ts *LevelDBTripleStore - ro *leveldb_opt.ReadOptions -} - -func NewLevelDBAllIterator(prefix, dir string, ts *LevelDBTripleStore) *LevelDBAllIterator { - var it LevelDBAllIterator - graph.BaseIteratorInit(&it.BaseIterator) - it.ro = &leveldb_opt.ReadOptions{} - it.ro.DontFillCache = true - it.it = ts.db.NewIterator(nil, it.ro) - it.prefix = []byte(prefix) - it.dir = dir - it.open = true - it.ts = ts - it.it.Seek(it.prefix) - if !it.it.Valid() { - it.open = false - it.it.Release() - } - return &it -} - -func (a *LevelDBAllIterator) Reset() { - if !a.open { - a.it = a.ts.db.NewIterator(nil, a.ro) - a.open = true - } - a.it.Seek(a.prefix) - if !a.it.Valid() { - a.open = false - a.it.Release() - } -} - -func (a *LevelDBAllIterator) Clone() graph.Iterator { - out := NewLevelDBAllIterator(string(a.prefix), a.dir, a.ts) - out.CopyTagsFrom(a) - return out -} - -func (a *LevelDBAllIterator) Next() (graph.TSVal, bool) { - if !a.open { - a.Last = nil - return nil, false - } - var out []byte - out = make([]byte, len(a.it.Key())) - copy(out, a.it.Key()) - a.it.Next() - if !a.it.Valid() { - a.Close() - } - if !bytes.HasPrefix(out, a.prefix) { - a.Close() - return nil, false - } - a.Last = out - return out, true -} - -func (a *LevelDBAllIterator) Check(v graph.TSVal) bool { - a.Last = v - return true -} - -func (lit *LevelDBAllIterator) Close() { - if lit.open { - lit.it.Release() - lit.open = false - } -} - -func (a *LevelDBAllIterator) Size() (int64, bool) { - size, err := a.ts.GetApproximateSizeForPrefix(a.prefix) - if err == nil { - return size, false - } - // INT64_MAX - return int64(^uint64(0) >> 1), false -} - -func (lit *LevelDBAllIterator) DebugString(indent int) string { - size, _ := lit.Size() - return fmt.Sprintf("%s(%s tags: %v leveldb size:%d %s %p)", strings.Repeat(" ", indent), lit.Type(), lit.Tags(), size, lit.dir, lit) -} - -func (lit *LevelDBAllIterator) Type() string { return "all" } -func (lit *LevelDBAllIterator) Sorted() bool { return false } - -func (lit *LevelDBAllIterator) Optimize() (graph.Iterator, bool) { - return lit, false -} - -func (lit *LevelDBAllIterator) GetStats() *graph.IteratorStats { - s, _ := lit.Size() - return &graph.IteratorStats{ - CheckCost: 1, - NextCost: 2, - Size: s, - } -} diff --git a/src/graph_leveldb/leveldb-iterator.go b/src/graph_leveldb/leveldb-iterator.go deleted file mode 100644 index 0e611f1..0000000 --- a/src/graph_leveldb/leveldb-iterator.go +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph_leveldb - -import ( - "bytes" - _ "encoding/binary" - "fmt" - leveldb_it "github.com/syndtr/goleveldb/leveldb/iterator" - leveldb_opt "github.com/syndtr/goleveldb/leveldb/opt" - "graph" - "strings" -) - -type LevelDBIterator struct { - graph.BaseIterator - nextPrefix []byte - checkId []byte - dir string - open bool - it leveldb_it.Iterator - ts *LevelDBTripleStore - ro *leveldb_opt.ReadOptions - originalPrefix string -} - -func NewLevelDBIterator(prefix, dir string, value graph.TSVal, ts *LevelDBTripleStore) *LevelDBIterator { - var it LevelDBIterator - graph.BaseIteratorInit(&it.BaseIterator) - it.checkId = value.([]byte) - it.dir = dir - it.originalPrefix = prefix - it.nextPrefix = make([]byte, 0, 2+ts.hasher.Size()) - it.nextPrefix = append(it.nextPrefix, []byte(prefix)...) - it.nextPrefix = append(it.nextPrefix, []byte(it.checkId[1:])...) - it.ro = &leveldb_opt.ReadOptions{} - it.ro.DontFillCache = true - it.it = ts.db.NewIterator(nil, it.ro) - it.open = true - it.ts = ts - ok := it.it.Seek(it.nextPrefix) - if !ok { - it.open = false - it.it.Release() - } - return &it -} - -func (lit *LevelDBIterator) Reset() { - if !lit.open { - lit.it = lit.ts.db.NewIterator(nil, lit.ro) - lit.open = true - } - ok := lit.it.Seek(lit.nextPrefix) - if !ok { - lit.open = false - lit.it.Release() - } -} - -func (lit *LevelDBIterator) Clone() graph.Iterator { - out := NewLevelDBIterator(lit.originalPrefix, lit.dir, lit.checkId, lit.ts) - out.CopyTagsFrom(lit) - return out -} - -func (lit *LevelDBIterator) Close() { - if lit.open { - lit.it.Release() - lit.open = false - } -} - -func (lit *LevelDBIterator) Next() (graph.TSVal, bool) { - if lit.it == nil { - lit.Last = nil - return nil, false - } - if !lit.open { - lit.Last = nil - return nil, false - } - if !lit.it.Valid() { - lit.Last = nil - lit.Close() - return nil, false - } - if bytes.HasPrefix(lit.it.Key(), lit.nextPrefix) { - out := make([]byte, len(lit.it.Key())) - copy(out, lit.it.Key()) - lit.Last = out - ok := lit.it.Next() - if !ok { - lit.Close() - } - return out, true - } - lit.Close() - lit.Last = nil - return nil, false -} - -func GetPositionFromPrefix(prefix []byte, dir string, ts *LevelDBTripleStore) int { - if bytes.Equal(prefix, []byte("sp")) { - switch dir { - case "s": - return 2 - case "p": - return ts.hasher.Size() + 2 - case "o": - return 2*ts.hasher.Size() + 2 - case "c": - return -1 - } - } - if bytes.Equal(prefix, []byte("po")) { - switch dir { - case "s": - return 2*ts.hasher.Size() + 2 - case "p": - return 2 - case "o": - return ts.hasher.Size() + 2 - case "c": - return -1 - } - } - if bytes.Equal(prefix, []byte("os")) { - switch dir { - case "s": - return ts.hasher.Size() + 2 - case "p": - return 2*ts.hasher.Size() + 2 - case "o": - return 2 - case "c": - return -1 - } - } - if bytes.Equal(prefix, []byte("cp")) { - switch dir { - case "s": - return 2*ts.hasher.Size() + 2 - case "p": - return ts.hasher.Size() + 2 - case "o": - return 3*ts.hasher.Size() + 2 - case "c": - return 2 - } - } - panic("Notreached") -} - -func (lit *LevelDBIterator) Check(v graph.TSVal) bool { - val := v.([]byte) - if val[0] == 'z' { - return false - } - offset := GetPositionFromPrefix(val[0:2], lit.dir, lit.ts) - if offset != -1 { - if bytes.HasPrefix(val[offset:], lit.checkId[1:]) { - return true - } - } else { - nameForDir := lit.ts.GetTriple(v).Get(lit.dir) - hashForDir := lit.ts.GetIdFor(nameForDir).([]byte) - if bytes.Equal(hashForDir, lit.checkId) { - return true - } - } - return false -} - -func (lit *LevelDBIterator) Size() (int64, bool) { - return lit.ts.GetSizeFor(lit.checkId), true -} - -func (lit *LevelDBIterator) DebugString(indent int) string { - size, _ := lit.Size() - return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), lit.Type(), lit.GetUid(), lit.Tags(), lit.dir, size, lit.ts.GetNameFor(lit.checkId)) -} - -func (lit *LevelDBIterator) Type() string { return "leveldb" } -func (lit *LevelDBIterator) Sorted() bool { return false } - -func (lit *LevelDBIterator) Optimize() (graph.Iterator, bool) { - return lit, false -} - -func (lit *LevelDBIterator) GetStats() *graph.IteratorStats { - s, _ := lit.Size() - return &graph.IteratorStats{ - CheckCost: 1, - NextCost: 2, - Size: s, - } -} diff --git a/src/graph_leveldb/leveldb-triplestore-iterator-optimize.go b/src/graph_leveldb/leveldb-triplestore-iterator-optimize.go deleted file mode 100644 index 99ae611..0000000 --- a/src/graph_leveldb/leveldb-triplestore-iterator-optimize.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph_leveldb - -import ( - "graph" -) - -func (ts *LevelDBTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { - switch it.Type() { - case "linksto": - return ts.optimizeLinksTo(it.(*graph.LinksToIterator)) - - } - return it, false -} - -func (ts *LevelDBTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) { - l := it.GetSubIterators() - if l.Len() != 1 { - return it, false - } - primaryIt := l.Front().Value.(graph.Iterator) - if primaryIt.Type() == "fixed" { - size, _ := primaryIt.Size() - if size == 1 { - val, ok := primaryIt.Next() - if !ok { - panic("Sizes lie") - } - newIt := ts.GetTripleIterator(it.Direction(), val) - newIt.CopyTagsFrom(it) - for _, tag := range primaryIt.Tags() { - newIt.AddFixedTag(tag, val) - } - it.Close() - return newIt, true - } - } - return it, false -} diff --git a/src/graph_leveldb/leveldb-triplestore.go b/src/graph_leveldb/leveldb-triplestore.go deleted file mode 100644 index e459bfd..0000000 --- a/src/graph_leveldb/leveldb-triplestore.go +++ /dev/null @@ -1,427 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph_leveldb - -import ( - "bytes" - "crypto/sha1" - "encoding/binary" - "encoding/json" - "fmt" - "github.com/barakmich/glog" - "github.com/syndtr/goleveldb/leveldb" - leveldb_cache "github.com/syndtr/goleveldb/leveldb/cache" - leveldb_opt "github.com/syndtr/goleveldb/leveldb/opt" - leveldb_util "github.com/syndtr/goleveldb/leveldb/util" - "graph" - "hash" -) - -const DefaultCacheSize = 2 -const DefaultWriteBufferSize = 20 - -type LevelDBTripleStore struct { - dbOpts *leveldb_opt.Options - db *leveldb.DB - path string - open bool - size int64 - hasher hash.Hash - writeopts *leveldb_opt.WriteOptions - readopts *leveldb_opt.ReadOptions -} - -func CreateNewLevelDB(path string) bool { - opts := &leveldb_opt.Options{} - db, err := leveldb.OpenFile(path, opts) - if err != nil { - glog.Errorln("Error: couldn't create database", err) - return false - } - defer db.Close() - ts := &LevelDBTripleStore{} - ts.db = db - ts.writeopts = &leveldb_opt.WriteOptions{ - Sync: true, - } - ts.Close() - return true -} - -func NewDefaultLevelDBTripleStore(path string, options graph.OptionsDict) *LevelDBTripleStore { - var ts LevelDBTripleStore - ts.path = path - cache_size := DefaultCacheSize - if val, ok := options.GetIntKey("cache_size_mb"); ok { - cache_size = val - } - ts.dbOpts = &leveldb_opt.Options{ - BlockCache: leveldb_cache.NewLRUCache(cache_size * leveldb_opt.MiB), - } - ts.dbOpts.ErrorIfMissing = true - - write_buffer_mb := DefaultWriteBufferSize - if val, ok := options.GetIntKey("write_buffer_mb"); ok { - write_buffer_mb = val - } - ts.dbOpts.WriteBuffer = write_buffer_mb * leveldb_opt.MiB - ts.hasher = sha1.New() - ts.writeopts = &leveldb_opt.WriteOptions{ - Sync: false, - } - ts.readopts = &leveldb_opt.ReadOptions{} - db, err := leveldb.OpenFile(ts.path, ts.dbOpts) - if err != nil { - panic("Error, couldn't open! " + err.Error()) - } - ts.db = db - glog.Infoln(ts.GetStats()) - ts.getSize() - return &ts -} - -func (ts *LevelDBTripleStore) GetStats() string { - out := "" - stats, err := ts.db.GetProperty("leveldb.stats") - if err == nil { - out += fmt.Sprintln("Stats: ", stats) - } - out += fmt.Sprintln("Size: ", ts.size) - return out -} - -func (ts *LevelDBTripleStore) Size() int64 { - return ts.size -} - -func (ts *LevelDBTripleStore) createKeyFor(dir1, dir2, dir3 string, triple *graph.Triple) []byte { - key := make([]byte, 0, 2+(ts.hasher.Size()*3)) - key = append(key, []byte(dir1+dir2)...) - key = append(key, ts.convertStringToByteHash(triple.Get(dir1))...) - key = append(key, ts.convertStringToByteHash(triple.Get(dir2))...) - key = append(key, ts.convertStringToByteHash(triple.Get(dir3))...) - return key -} - -func (ts *LevelDBTripleStore) createProvKeyFor(dir1, dir2, dir3 string, triple *graph.Triple) []byte { - key := make([]byte, 0, 2+(ts.hasher.Size()*4)) - key = append(key, []byte("c"+dir1)...) - key = append(key, ts.convertStringToByteHash(triple.Get("c"))...) - key = append(key, ts.convertStringToByteHash(triple.Get(dir1))...) - key = append(key, ts.convertStringToByteHash(triple.Get(dir2))...) - key = append(key, ts.convertStringToByteHash(triple.Get(dir3))...) - return key -} - -func (ts *LevelDBTripleStore) createValueKeyFor(s string) []byte { - key := make([]byte, 0, 1+ts.hasher.Size()) - key = append(key, []byte("z")...) - key = append(key, ts.convertStringToByteHash(s)...) - return key -} - -func (ts *LevelDBTripleStore) AddTriple(t *graph.Triple) { - batch := &leveldb.Batch{} - ts.buildWrite(batch, t) - err := ts.db.Write(batch, ts.writeopts) - if err != nil { - glog.Errorf("Couldn't write to DB for triple %s", t.ToString()) - return - } - ts.size++ -} - -func (ts *LevelDBTripleStore) RemoveTriple(t *graph.Triple) { - _, err := ts.db.Get(ts.createKeyFor("s", "p", "o", t), ts.readopts) - if err != nil && err != leveldb.ErrNotFound { - glog.Errorf("Couldn't access DB to confirm deletion") - return - } - if err == leveldb.ErrNotFound { - // No such triple in the database, forget about it. - return - } - batch := &leveldb.Batch{} - batch.Delete(ts.createKeyFor("s", "p", "o", t)) - batch.Delete(ts.createKeyFor("o", "s", "p", t)) - batch.Delete(ts.createKeyFor("p", "o", "s", t)) - ts.UpdateValueKeyBy(t.Get("s"), -1, batch) - ts.UpdateValueKeyBy(t.Get("p"), -1, batch) - ts.UpdateValueKeyBy(t.Get("o"), -1, batch) - if t.Get("c") != "" { - batch.Delete(ts.createProvKeyFor("p", "s", "o", t)) - ts.UpdateValueKeyBy(t.Get("c"), -1, batch) - } - err = ts.db.Write(batch, nil) - if err != nil { - glog.Errorf("Couldn't delete triple %s", t.ToString()) - return - } - ts.size-- -} - -func (ts *LevelDBTripleStore) buildTripleWrite(batch *leveldb.Batch, t *graph.Triple) { - bytes, err := json.Marshal(*t) - if err != nil { - glog.Errorf("Couldn't write to buffer for triple %s\n %s\n", t.ToString(), err) - return - } - batch.Put(ts.createKeyFor("s", "p", "o", t), bytes) - batch.Put(ts.createKeyFor("o", "s", "p", t), bytes) - batch.Put(ts.createKeyFor("p", "o", "s", t), bytes) - if t.Get("c") != "" { - batch.Put(ts.createProvKeyFor("p", "s", "o", t), bytes) - } -} - -func (ts *LevelDBTripleStore) buildWrite(batch *leveldb.Batch, t *graph.Triple) { - ts.buildTripleWrite(batch, t) - ts.UpdateValueKeyBy(t.Get("s"), 1, nil) - ts.UpdateValueKeyBy(t.Get("p"), 1, nil) - ts.UpdateValueKeyBy(t.Get("o"), 1, nil) - if t.Get("c") != "" { - ts.UpdateValueKeyBy(t.Get("c"), 1, nil) - } -} - -type ValueData struct { - Name string - Size int64 -} - -func (ts *LevelDBTripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.Batch) { - value := &ValueData{name, int64(amount)} - key := ts.createValueKeyFor(name) - b, err := ts.db.Get(key, ts.readopts) - - // Error getting the node from the database. - if err != nil && err != leveldb.ErrNotFound { - glog.Errorf("Error reading Value %s from the DB\n", name) - return - } - - // Node exists in the database -- unmarshal and update. - if b != nil && err != leveldb.ErrNotFound { - err = json.Unmarshal(b, value) - if err != nil { - glog.Errorln("Error: couldn't reconstruct value ", err) - return - } - value.Size += int64(amount) - } - - // Are we deleting something? - if amount < 0 { - if value.Size <= 0 { - if batch == nil { - ts.db.Delete(key, ts.writeopts) - } else { - batch.Delete(key) - } - return - } - } - - // Repackage and rewrite. - bytes, err := json.Marshal(&value) - if err != nil { - glog.Errorf("Couldn't write to buffer for value %s\n %s", name, err) - return - } - if batch == nil { - ts.db.Put(key, bytes, ts.writeopts) - } else { - batch.Put(key, bytes) - } -} - -func (ts *LevelDBTripleStore) AddTripleSet(t_s []*graph.Triple) { - batch := &leveldb.Batch{} - newTs := len(t_s) - resizeMap := make(map[string]int) - for _, t := range t_s { - ts.buildTripleWrite(batch, t) - resizeMap[t.Sub]++ - resizeMap[t.Pred]++ - resizeMap[t.Obj]++ - if t.Provenance != "" { - resizeMap[t.Provenance]++ - } - } - for k, v := range resizeMap { - ts.UpdateValueKeyBy(k, v, batch) - } - err := ts.db.Write(batch, ts.writeopts) - if err != nil { - glog.Errorf("Couldn't write to DB for tripleset") - return - } - ts.size += int64(newTs) -} - -func (ldbts *LevelDBTripleStore) Close() { - buf := new(bytes.Buffer) - err := binary.Write(buf, binary.LittleEndian, ldbts.size) - if err == nil { - werr := ldbts.db.Put([]byte("__size"), buf.Bytes(), ldbts.writeopts) - if werr != nil { - glog.Errorf("Couldn't write size before closing!") - } - } else { - glog.Errorf("Couldn't convert size before closing!") - } - ldbts.db.Close() - ldbts.open = false -} - -func (ts *LevelDBTripleStore) GetTriple(k graph.TSVal) *graph.Triple { - var triple graph.Triple - b, err := ts.db.Get(k.([]byte), ts.readopts) - if err != nil && err != leveldb.ErrNotFound { - glog.Errorln("Error: couldn't get triple from DB") - return &graph.Triple{} - } - if err == leveldb.ErrNotFound { - // No harm, no foul. - return &graph.Triple{} - } - err = json.Unmarshal(b, &triple) - if err != nil { - glog.Errorln("Error: couldn't reconstruct triple") - return &graph.Triple{} - } - return &triple -} - -func (ts *LevelDBTripleStore) convertStringToByteHash(s string) []byte { - ts.hasher.Reset() - key := make([]byte, 0, ts.hasher.Size()) - ts.hasher.Write([]byte(s)) - key = ts.hasher.Sum(key) - return key -} - -func (ts *LevelDBTripleStore) GetIdFor(s string) graph.TSVal { - return ts.createValueKeyFor(s) -} - -func (ts *LevelDBTripleStore) getValueData(value_key []byte) ValueData { - var out ValueData - if glog.V(3) { - glog.V(3).Infof("%s %v\n", string(value_key[0]), value_key) - } - b, err := ts.db.Get(value_key, ts.readopts) - if err != nil && err != leveldb.ErrNotFound { - glog.Errorln("Error: couldn't get value from DB") - return out - } - if b != nil && err != leveldb.ErrNotFound { - err = json.Unmarshal(b, &out) - if err != nil { - glog.Errorln("Error: couldn't reconstruct value") - return ValueData{} - } - } - return out -} - -func (ts *LevelDBTripleStore) GetNameFor(k graph.TSVal) string { - if k == nil { - glog.V(2).Infoln("k was nil") - return "" - } - return ts.getValueData(k.([]byte)).Name -} - -func (ts *LevelDBTripleStore) GetSizeFor(k graph.TSVal) int64 { - if k == nil { - return 0 - } - return int64(ts.getValueData(k.([]byte)).Size) -} - -func (ts *LevelDBTripleStore) getSize() { - var size int64 - b, err := ts.db.Get([]byte("__size"), ts.readopts) - if err != nil && err != leveldb.ErrNotFound { - panic("Couldn't read size " + err.Error()) - } - if err == leveldb.ErrNotFound { - // Must be a new database. Cool - ts.size = 0 - return - } - buf := bytes.NewBuffer(b) - err = binary.Read(buf, binary.LittleEndian, &size) - if err != nil { - glog.Errorln("Error: couldn't parse size") - } - ts.size = size -} - -func (ts *LevelDBTripleStore) GetApproximateSizeForPrefix(pre []byte) (int64, error) { - limit := make([]byte, len(pre)) - copy(limit, pre) - end := len(limit) - 1 - limit[end]++ - ranges := make([]leveldb_util.Range, 1) - ranges[0].Start = pre - ranges[0].Limit = limit - sizes, err := ts.db.GetApproximateSizes(ranges) - if err == nil { - return (int64(sizes[0]) >> 6) + 1, nil - } - return 0, nil -} - -func (ts *LevelDBTripleStore) GetTripleIterator(dir string, val graph.TSVal) graph.Iterator { - switch dir { - case "s": - return NewLevelDBIterator("sp", "s", val, ts) - case "p": - return NewLevelDBIterator("po", "p", val, ts) - case "o": - return NewLevelDBIterator("os", "o", val, ts) - case "c": - return NewLevelDBIterator("cp", "c", val, ts) - } - panic("Notreached " + dir) -} - -func (ts *LevelDBTripleStore) GetNodesAllIterator() graph.Iterator { - return NewLevelDBAllIterator("z", "v", ts) -} - -func (ts *LevelDBTripleStore) GetTriplesAllIterator() graph.Iterator { - return NewLevelDBAllIterator("po", "p", ts) -} - -func (ts *LevelDBTripleStore) GetTripleDirection(val graph.TSVal, direction string) graph.TSVal { - v := val.([]uint8) - offset := GetPositionFromPrefix(v[0:2], direction, ts) - if offset != -1 { - return append([]byte("z"), v[offset:offset+ts.hasher.Size()]...) - } else { - return ts.GetTriple(val).Get(direction) - } -} - -func compareBytes(a, b graph.TSVal) bool { - return bytes.Equal(a.([]uint8), b.([]uint8)) -} - -func (ts *LevelDBTripleStore) MakeFixed() *graph.FixedIterator { - return graph.NewFixedIteratorWithCompare(compareBytes) -} diff --git a/src/graph_leveldb/leveldb_test.go b/src/graph_leveldb/leveldb_test.go deleted file mode 100644 index 4b17827..0000000 --- a/src/graph_leveldb/leveldb_test.go +++ /dev/null @@ -1,433 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph_leveldb - -import ( - . "github.com/smartystreets/goconvey/convey" - "graph" - "io/ioutil" - "os" - "sort" - "testing" -) - -func makeTripleSet() []*graph.Triple { - tripleSet := []*graph.Triple{ - graph.MakeTriple("A", "follows", "B", ""), - graph.MakeTriple("C", "follows", "B", ""), - graph.MakeTriple("C", "follows", "D", ""), - graph.MakeTriple("D", "follows", "B", ""), - graph.MakeTriple("B", "follows", "F", ""), - graph.MakeTriple("F", "follows", "G", ""), - graph.MakeTriple("D", "follows", "G", ""), - graph.MakeTriple("E", "follows", "F", ""), - graph.MakeTriple("B", "status", "cool", "status_graph"), - graph.MakeTriple("D", "status", "cool", "status_graph"), - graph.MakeTriple("G", "status", "cool", "status_graph"), - } - return tripleSet -} - -func extractTripleFromIterator(ts graph.TripleStore, it graph.Iterator) []string { - var output []string - for { - val, ok := it.Next() - if !ok { - break - } - output = append(output, ts.GetTriple(val).ToString()) - } - return output -} - -func extractValuesFromIterator(ts graph.TripleStore, it graph.Iterator) []string { - var output []string - for { - val, ok := it.Next() - if !ok { - break - } - output = append(output, ts.GetNameFor(val)) - } - return output -} - -func TestCreateDatabase(t *testing.T) { - - Convey("Given a database path", t, func() { - tmpDir, err := ioutil.TempDir(os.TempDir(), "cayley_test") - t.Log(tmpDir) - if err != nil { - t.Fatal("Cannot use ioutil.", err) - } - - Convey("Creates a database", func() { - ok := CreateNewLevelDB(tmpDir) - So(ok, ShouldBeTrue) - Convey("And has good defaults for a new database", func() { - ts := NewDefaultLevelDBTripleStore(tmpDir, nil) - So(ts, ShouldNotBeNil) - So(ts.Size(), ShouldEqual, 0) - ts.Close() - }) - }) - - Convey("Fails if it cannot create the database", func() { - ok := CreateNewLevelDB("/dev/null/some terrible path") - So(ok, ShouldBeFalse) - So(func() { NewDefaultLevelDBTripleStore("/dev/null/some terrible path", nil) }, ShouldPanic) - }) - - Reset(func() { - os.RemoveAll(tmpDir) - }) - - }) - -} - -func TestLoadDatabase(t *testing.T) { - var ts *LevelDBTripleStore - - Convey("Given a created database path", t, func() { - tmpDir, _ := ioutil.TempDir(os.TempDir(), "cayley_test") - t.Log(tmpDir) - ok := CreateNewLevelDB(tmpDir) - So(ok, ShouldBeTrue) - ts = NewDefaultLevelDBTripleStore(tmpDir, nil) - - Convey("Can load a single triple", func() { - ts.AddTriple(graph.MakeTriple("Something", "points_to", "Something Else", "context")) - So(ts.GetNameFor(ts.GetIdFor("Something")), ShouldEqual, "Something") - So(ts.Size(), ShouldEqual, 1) - }) - - Convey("Can load many triples", func() { - - ts.AddTripleSet(makeTripleSet()) - So(ts.Size(), ShouldEqual, 11) - So(ts.GetSizeFor(ts.GetIdFor("B")), ShouldEqual, 5) - - Convey("Can delete triples", func() { - ts.RemoveTriple(graph.MakeTriple("A", "follows", "B", "")) - So(ts.Size(), ShouldEqual, 10) - So(ts.GetSizeFor(ts.GetIdFor("B")), ShouldEqual, 4) - }) - }) - - Reset(func() { - ts.Close() - os.RemoveAll(tmpDir) - }) - - }) - -} - -func TestAllIterator(t *testing.T) { - var ts *LevelDBTripleStore - - Convey("Given a prepared database", t, func() { - tmpDir, _ := ioutil.TempDir(os.TempDir(), "cayley_test") - t.Log(tmpDir) - defer os.RemoveAll(tmpDir) - ok := CreateNewLevelDB(tmpDir) - So(ok, ShouldBeTrue) - ts = NewDefaultLevelDBTripleStore(tmpDir, nil) - ts.AddTripleSet(makeTripleSet()) - var it graph.Iterator - - Convey("Can create an all iterator for nodes", func() { - it = ts.GetNodesAllIterator() - So(it, ShouldNotBeNil) - - Convey("Has basics", func() { - size, accurate := it.Size() - So(size, ShouldBeBetween, 0, 20) - So(accurate, ShouldBeFalse) - So(it.Type(), ShouldEqual, "all") - re_it, ok := it.Optimize() - So(ok, ShouldBeFalse) - So(re_it, ShouldPointTo, it) - }) - - Convey("Iterates all nodes", func() { - expected := []string{ - "A", - "B", - "C", - "D", - "E", - "F", - "G", - "follows", - "status", - "cool", - "status_graph", - } - sort.Strings(expected) - actual := extractValuesFromIterator(ts, it) - sort.Strings(actual) - So(actual, ShouldResemble, expected) - it.Reset() - actual = extractValuesFromIterator(ts, it) - sort.Strings(actual) - So(actual, ShouldResemble, expected) - - }) - - Convey("Contains a couple nodes", func() { - So(it.Check(ts.GetIdFor("A")), ShouldBeTrue) - So(it.Check(ts.GetIdFor("cool")), ShouldBeTrue) - //So(it.Check(ts.GetIdFor("baller")), ShouldBeFalse) - }) - - Reset(func() { - it.Reset() - }) - }) - - Convey("Can create an all iterator for edges", func() { - it := ts.GetTriplesAllIterator() - So(it, ShouldNotBeNil) - Convey("Has basics", func() { - size, accurate := it.Size() - So(size, ShouldBeBetween, 0, 20) - So(accurate, ShouldBeFalse) - So(it.Type(), ShouldEqual, "all") - re_it, ok := it.Optimize() - So(ok, ShouldBeFalse) - So(re_it, ShouldPointTo, it) - }) - - Convey("Iterates an edge", func() { - edge_val, _ := it.Next() - triple := ts.GetTriple(edge_val) - set := makeTripleSet() - var string_set []string - for _, t := range set { - string_set = append(string_set, t.ToString()) - } - So(triple.ToString(), ShouldBeIn, string_set) - }) - - Reset(func() { - ts.Close() - }) - }) - }) - -} - -func TestSetIterator(t *testing.T) { - var ts *LevelDBTripleStore - var tmpDir string - - Convey("Given a prepared database", t, func() { - tmpDir, _ = ioutil.TempDir(os.TempDir(), "cayley_test") - t.Log(tmpDir) - defer os.RemoveAll(tmpDir) - ok := CreateNewLevelDB(tmpDir) - So(ok, ShouldBeTrue) - ts = NewDefaultLevelDBTripleStore(tmpDir, nil) - ts.AddTripleSet(makeTripleSet()) - var it graph.Iterator - - Convey("Can create a subject iterator", func() { - it = ts.GetTripleIterator("s", ts.GetIdFor("C")) - - Convey("Containing the right things", func() { - expected := []string{ - graph.MakeTriple("C", "follows", "B", "").ToString(), - graph.MakeTriple("C", "follows", "D", "").ToString(), - } - actual := extractTripleFromIterator(ts, it) - sort.Strings(actual) - sort.Strings(expected) - So(actual, ShouldResemble, expected) - }) - - Convey("And checkable", func() { - and := graph.NewAndIterator() - and.AddSubIterator(ts.GetTriplesAllIterator()) - and.AddSubIterator(it) - - expected := []string{ - graph.MakeTriple("C", "follows", "B", "").ToString(), - graph.MakeTriple("C", "follows", "D", "").ToString(), - } - actual := extractTripleFromIterator(ts, and) - sort.Strings(actual) - sort.Strings(expected) - So(actual, ShouldResemble, expected) - }) - Reset(func() { - it.Reset() - }) - - }) - - Convey("Can create an object iterator", func() { - it = ts.GetTripleIterator("o", ts.GetIdFor("F")) - - Convey("Containing the right things", func() { - expected := []string{ - graph.MakeTriple("B", "follows", "F", "").ToString(), - graph.MakeTriple("E", "follows", "F", "").ToString(), - } - actual := extractTripleFromIterator(ts, it) - sort.Strings(actual) - sort.Strings(expected) - So(actual, ShouldResemble, expected) - }) - - Convey("Mutually and-checkable", func() { - and := graph.NewAndIterator() - and.AddSubIterator(ts.GetTripleIterator("s", ts.GetIdFor("B"))) - and.AddSubIterator(it) - - expected := []string{ - graph.MakeTriple("B", "follows", "F", "").ToString(), - } - actual := extractTripleFromIterator(ts, and) - sort.Strings(actual) - sort.Strings(expected) - So(actual, ShouldResemble, expected) - }) - - }) - - Convey("Can create a predicate iterator", func() { - it = ts.GetTripleIterator("p", ts.GetIdFor("status")) - - Convey("Containing the right things", func() { - expected := []string{ - graph.MakeTriple("B", "status", "cool", "status_graph").ToString(), - graph.MakeTriple("D", "status", "cool", "status_graph").ToString(), - graph.MakeTriple("G", "status", "cool", "status_graph").ToString(), - } - actual := extractTripleFromIterator(ts, it) - sort.Strings(actual) - sort.Strings(expected) - So(actual, ShouldResemble, expected) - }) - - }) - - Convey("Can create a provenance iterator", func() { - it = ts.GetTripleIterator("c", ts.GetIdFor("status_graph")) - - Convey("Containing the right things", func() { - expected := []string{ - graph.MakeTriple("B", "status", "cool", "status_graph").ToString(), - graph.MakeTriple("D", "status", "cool", "status_graph").ToString(), - graph.MakeTriple("G", "status", "cool", "status_graph").ToString(), - } - actual := extractTripleFromIterator(ts, it) - sort.Strings(actual) - sort.Strings(expected) - So(actual, ShouldResemble, expected) - }) - - Convey("Can be cross-checked", func() { - and := graph.NewAndIterator() - // Order is important - and.AddSubIterator(ts.GetTripleIterator("s", ts.GetIdFor("B"))) - and.AddSubIterator(it) - - expected := []string{ - graph.MakeTriple("B", "status", "cool", "status_graph").ToString(), - } - actual := extractTripleFromIterator(ts, and) - So(actual, ShouldResemble, expected) - }) - - Convey("Can check against other iterators", func() { - and := graph.NewAndIterator() - // Order is important - and.AddSubIterator(it) - and.AddSubIterator(ts.GetTripleIterator("s", ts.GetIdFor("B"))) - - expected := []string{ - graph.MakeTriple("B", "status", "cool", "status_graph").ToString(), - } - actual := extractTripleFromIterator(ts, and) - So(actual, ShouldResemble, expected) - }) - Reset(func() { - it.Reset() - }) - - }) - - Reset(func() { - ts.Close() - }) - - }) - -} - -func TestOptimize(t *testing.T) { - var ts *LevelDBTripleStore - var lto graph.Iterator - var tmpDir string - - Convey("Given a prepared database", t, func() { - tmpDir, _ = ioutil.TempDir(os.TempDir(), "cayley_test") - t.Log(tmpDir) - defer os.RemoveAll(tmpDir) - ok := CreateNewLevelDB(tmpDir) - So(ok, ShouldBeTrue) - ts = NewDefaultLevelDBTripleStore(tmpDir, nil) - ts.AddTripleSet(makeTripleSet()) - - Convey("With an linksto-fixed pair", func() { - fixed := ts.MakeFixed() - fixed.AddValue(ts.GetIdFor("F")) - fixed.AddTag("internal") - lto = graph.NewLinksToIterator(ts, fixed, "o") - - Convey("Creates an appropriate iterator", func() { - oldIt := lto.Clone() - newIt, ok := lto.Optimize() - So(ok, ShouldBeTrue) - So(newIt.Type(), ShouldEqual, "leveldb") - - Convey("Containing the right things", func() { - afterOp := extractTripleFromIterator(ts, newIt) - beforeOp := extractTripleFromIterator(ts, oldIt) - sort.Strings(afterOp) - sort.Strings(beforeOp) - So(afterOp, ShouldResemble, beforeOp) - }) - - Convey("With the correct tags", func() { - oldIt.Next() - newIt.Next() - oldResults := make(map[string]graph.TSVal) - oldIt.TagResults(&oldResults) - newResults := make(map[string]graph.TSVal) - oldIt.TagResults(&newResults) - So(newResults, ShouldResemble, oldResults) - }) - - }) - - }) - - }) - -}