diff --git a/TODO.md b/TODO.md index 1c84516..c781c8f 100644 --- a/TODO.md +++ b/TODO.md @@ -68,7 +68,7 @@ The necessary component to make mid-query limit work. Acts as a limit on Next(), ### Postgres Backend It'd be nice to run on SQL as well. It's a big why not? #### Generalist layout - Notionally, this is a simple quad table with a number of indicies. Iterators and iterator optimization (ie, rewriting SQL queries) is the 'fun' part + Notionally, this is a simple quad table with a number of indices. Iterators and iterator optimization (ie, rewriting SQL queries) is the 'fun' part #### "Short Schema" Layout? This one is the crazy one. Suppose a world where we actually use the table schema for predicates, and update the table schema as we go along. Yes, it sucks when you add a new predicate (and the cell values are unclear) but for small worlds (or, "short schemas") it may (or may not) be interesting. diff --git a/docs/Configuration.md b/docs/Configuration.md index e1ac677..470c53e 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -122,7 +122,7 @@ Optionally disable syncing to disk per transaction. Nosync being true means much * Type: String * Default: "cayley" -The name of the database within MongoDB to connect to. Manages its own collections and indicies therein. +The name of the database within MongoDB to connect to. Manages its own collections and indices therein. ## Per-Replication Options diff --git a/docs/GremlinAPI.md b/docs/GremlinAPI.md index 144b425..2c03c0e 100644 --- a/docs/GremlinAPI.md +++ b/docs/GremlinAPI.md @@ -16,7 +16,7 @@ Arguments: Returns: Query object -Starts a query path at the given vertex/verticies. No ids means "all vertices". +Starts a query path at the given vertex/vertices. No ids means "all vertices". ####**`graph.Morphism()`** @@ -98,7 +98,7 @@ g.V("D").Out() // Finds all things D points at on the status linkage. // Result is B G and cool_person g.V("D").Out(["follows", "status"]) -// Finds all things D points at on the status linkage, given from a seperate query path. +// Finds all things D points at on the status linkage, given from a separate query path. // Result is {"id": cool_person, "pred": "status"} g.V("D").Out(g.V("status"), "pred") ``` @@ -327,7 +327,7 @@ Starts as if at the g.M() and follows through the morphism path. Example: ```javascript: friendOfFriend = g.Morphism().Out("follows").Out("follows") -// Returns the followed people of who C follows -- a simplistic "friend of my frind" +// Returns the followed people of who C follows -- a simplistic "friend of my friend" // and whether or not they have a "cool" status. Potential for recommending followers abounds. // Returns B and G g.V("C").Follow(friendOfFriend).Has("status", "cool_person") diff --git a/graph/bolt/bolt_test.go b/graph/bolt/bolt_test.go new file mode 100644 index 0000000..986550b --- /dev/null +++ b/graph/bolt/bolt_test.go @@ -0,0 +1,467 @@ +// Copyright 2015 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bolt + +import ( + "fmt" + "io/ioutil" + "os" + "reflect" + "sort" + "testing" + + "github.com/google/cayley/graph" + "github.com/google/cayley/graph/iterator" + "github.com/google/cayley/quad" + "github.com/google/cayley/writer" +) + +func makeQuadSet() []quad.Quad { + quadSet := []quad.Quad{ + {"A", "follows", "B", ""}, + {"C", "follows", "B", ""}, + {"C", "follows", "D", ""}, + {"D", "follows", "B", ""}, + {"B", "follows", "F", ""}, + {"F", "follows", "G", ""}, + {"D", "follows", "G", ""}, + {"E", "follows", "F", ""}, + {"B", "status", "cool", "status_graph"}, + {"D", "status", "cool", "status_graph"}, + {"G", "status", "cool", "status_graph"}, + } + return quadSet +} + +func iteratedQuads(qs graph.QuadStore, it graph.Iterator) []quad.Quad { + var res ordered + for graph.Next(it) { + res = append(res, qs.Quad(it.Result())) + } + sort.Sort(res) + return res +} + +type ordered []quad.Quad + +func (o ordered) Len() int { return len(o) } +func (o ordered) Less(i, j int) bool { + switch { + case o[i].Subject < o[j].Subject, + + o[i].Subject == o[j].Subject && + o[i].Predicate < o[j].Predicate, + + o[i].Subject == o[j].Subject && + o[i].Predicate == o[j].Predicate && + o[i].Object < o[j].Object, + + o[i].Subject == o[j].Subject && + o[i].Predicate == o[j].Predicate && + o[i].Object == o[j].Object && + o[i].Label < o[j].Label: + + return true + + default: + return false + } +} +func (o ordered) Swap(i, j int) { o[i], o[j] = o[j], o[i] } + +func iteratedNames(qs graph.QuadStore, it graph.Iterator) []string { + var res []string + for graph.Next(it) { + res = append(res, qs.NameOf(it.Result())) + } + sort.Strings(res) + return res +} + +func TestCreateDatabase(t *testing.T) { + tmpFile, err := ioutil.TempFile(os.TempDir(), "cayley_test") + if err != nil { + t.Fatalf("Could not create working directory: %v", err) + } + t.Log(tmpFile) + + err = createNewBolt(tmpFile.Name(), nil) + if err != nil { + t.Fatal("Failed to create LevelDB database.") + } + + qs, err := newQuadStore(tmpFile.Name(), nil) + if qs == nil || err != nil { + t.Error("Failed to create leveldb QuadStore.") + } + if s := qs.Size(); s != 0 { + t.Errorf("Unexpected size, got:%d expected:0", s) + } + qs.Close() + + err = createNewBolt("/dev/null/some terrible path", nil) + if err == nil { + t.Errorf("Created LevelDB database for bad path.") + } + + os.RemoveAll(tmpFile.Name()) +} + +func TestLoadDatabase(t *testing.T) { + tmpFile, err := ioutil.TempFile(os.TempDir(), "cayley_test") + if err != nil { + t.Fatalf("Could not create working directory: %v", err) + } + defer os.RemoveAll(tmpFile.Name()) + t.Log(tmpFile.Name()) + + err = createNewBolt(tmpFile.Name(), nil) + if err != nil { + t.Fatal("Failed to create Bolt database.", err) + } + + qs, err := newQuadStore(tmpFile.Name(), nil) + if qs == nil || err != nil { + t.Error("Failed to create Bolt QuadStore.") + } + + w, _ := writer.NewSingleReplication(qs, nil) + w.AddQuad(quad.Quad{ + Subject: "Something", + Predicate: "points_to", + Object: "Something Else", + Label: "context", + }) + for _, pq := range []string{"Something", "points_to", "Something Else", "context"} { + if got := qs.NameOf(qs.ValueOf(pq)); got != pq { + t.Errorf("Failed to roundtrip %q, got:%q expect:%q", pq, got, pq) + } + } + if s := qs.Size(); s != 1 { + t.Errorf("Unexpected quadstore size, got:%d expect:1", s) + } + qs.Close() + os.RemoveAll(tmpFile.Name()) + + err = createNewBolt(tmpFile.Name(), nil) + if err != nil { + t.Fatal("Failed to create Bolt database.", err) + } + qs, err = newQuadStore(tmpFile.Name(), nil) + if qs == nil || err != nil { + t.Error("Failed to create Bolt QuadStore.") + } + w, _ = writer.NewSingleReplication(qs, nil) + + ts2, didConvert := qs.(*QuadStore) + if !didConvert { + t.Errorf("Could not convert from generic to LevelDB QuadStore") + } + + //Test horizon + horizon := qs.Horizon() + if horizon.Int() != 0 { + t.Errorf("Unexpected horizon value, got:%d expect:0", horizon.Int()) + } + + w.AddQuadSet(makeQuadSet()) + if s := qs.Size(); s != 11 { + t.Errorf("Unexpected quadstore size, got:%d expect:11", s) + } + if s := ts2.SizeOf(qs.ValueOf("B")); s != 5 { + t.Errorf("Unexpected quadstore size, got:%d expect:5", s) + } + horizon = qs.Horizon() + if horizon.Int() != 11 { + t.Errorf("Unexpected horizon value, got:%d expect:11", horizon.Int()) + } + + w.RemoveQuad(quad.Quad{ + Subject: "A", + Predicate: "follows", + Object: "B", + Label: "", + }) + if s := qs.Size(); s != 10 { + t.Errorf("Unexpected quadstore size after RemoveQuad, got:%d expect:10", s) + } + if s := ts2.SizeOf(qs.ValueOf("B")); s != 4 { + t.Errorf("Unexpected quadstore size, got:%d expect:4", s) + } + + qs.Close() +} + +func TestIterator(t *testing.T) { + tmpFile, err := ioutil.TempFile(os.TempDir(), "cayley_test") + if err != nil { + t.Fatalf("Could not create working directory: %v", err) + } + defer os.RemoveAll(tmpFile.Name()) + t.Log(tmpFile.Name()) + + err = createNewBolt(tmpFile.Name(), nil) + if err != nil { + t.Fatal("Failed to create LevelDB database.") + } + + qs, err := newQuadStore(tmpFile.Name(), nil) + if qs == nil || err != nil { + t.Error("Failed to create leveldb QuadStore.") + } + + w, _ := writer.NewSingleReplication(qs, nil) + w.AddQuadSet(makeQuadSet()) + var it graph.Iterator + + it = qs.NodesAllIterator() + if it == nil { + t.Fatal("Got nil iterator.") + } + + size, _ := it.Size() + if size <= 0 || size >= 20 { + t.Errorf("Unexpected size, got:%d expect:(0, 20)", size) + } + if typ := it.Type(); typ != graph.All { + t.Errorf("Unexpected iterator type, got:%v expect:%v", typ, graph.All) + } + optIt, changed := it.Optimize() + if changed || optIt != it { + t.Errorf("Optimize unexpectedly changed iterator.") + } + + expect := []string{ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "follows", + "status", + "cool", + "status_graph", + } + sort.Strings(expect) + for i := 0; i < 2; i++ { + got := iteratedNames(qs, it) + sort.Strings(got) + if !reflect.DeepEqual(got, expect) { + t.Errorf("Unexpected iterated result on repeat %d, got:%v expect:%v", i, got, expect) + } + it.Reset() + } + + for _, pq := range expect { + if !it.Contains(qs.ValueOf(pq)) { + t.Errorf("Failed to find and check %q correctly", pq) + } + } + // FIXME(kortschak) Why does this fail? + /* + for _, pq := range []string{"baller"} { + if it.Contains(qs.ValueOf(pq)) { + t.Errorf("Failed to check %q correctly", pq) + } + } + */ + it.Reset() + + it = qs.QuadsAllIterator() + graph.Next(it) + fmt.Printf("%#v\n", it.Result()) + q := qs.Quad(it.Result()) + fmt.Println(q) + set := makeQuadSet() + var ok bool + for _, e := range set { + if e.String() == q.String() { + ok = true + break + } + } + if !ok { + t.Errorf("Failed to find %q during iteration, got:%q", q, set) + } + + qs.Close() +} + +func TestSetIterator(t *testing.T) { + + tmpFile, _ := ioutil.TempFile(os.TempDir(), "cayley_test") + t.Log(tmpFile.Name()) + defer os.RemoveAll(tmpFile.Name()) + err := createNewBolt(tmpFile.Name(), nil) + if err != nil { + t.Fatalf("Failed to create working directory") + } + + qs, err := newQuadStore(tmpFile.Name(), nil) + if qs == nil || err != nil { + t.Error("Failed to create leveldb QuadStore.") + } + defer qs.Close() + + w, _ := writer.NewSingleReplication(qs, nil) + w.AddQuadSet(makeQuadSet()) + + expect := []quad.Quad{ + {"C", "follows", "B", ""}, + {"C", "follows", "D", ""}, + } + sort.Sort(ordered(expect)) + + // Subject iterator. + it := qs.QuadIterator(quad.Subject, qs.ValueOf("C")) + + if got := iteratedQuads(qs, it); !reflect.DeepEqual(got, expect) { + t.Errorf("Failed to get expected results, got:%v expect:%v", got, expect) + } + it.Reset() + + and := iterator.NewAnd() + and.AddSubIterator(qs.QuadsAllIterator()) + and.AddSubIterator(it) + + if got := iteratedQuads(qs, and); !reflect.DeepEqual(got, expect) { + t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect) + } + + // Object iterator. + it = qs.QuadIterator(quad.Object, qs.ValueOf("F")) + + expect = []quad.Quad{ + {"B", "follows", "F", ""}, + {"E", "follows", "F", ""}, + } + sort.Sort(ordered(expect)) + if got := iteratedQuads(qs, it); !reflect.DeepEqual(got, expect) { + t.Errorf("Failed to get expected results, got:%v expect:%v", got, expect) + } + + and = iterator.NewAnd() + and.AddSubIterator(qs.QuadIterator(quad.Subject, qs.ValueOf("B"))) + and.AddSubIterator(it) + + expect = []quad.Quad{ + {"B", "follows", "F", ""}, + } + if got := iteratedQuads(qs, and); !reflect.DeepEqual(got, expect) { + t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect) + } + + // Predicate iterator. + it = qs.QuadIterator(quad.Predicate, qs.ValueOf("status")) + + expect = []quad.Quad{ + {"B", "status", "cool", "status_graph"}, + {"D", "status", "cool", "status_graph"}, + {"G", "status", "cool", "status_graph"}, + } + sort.Sort(ordered(expect)) + if got := iteratedQuads(qs, it); !reflect.DeepEqual(got, expect) { + t.Errorf("Failed to get expected results from predicate iterator, got:%v expect:%v", got, expect) + } + + // Label iterator. + it = qs.QuadIterator(quad.Label, qs.ValueOf("status_graph")) + + expect = []quad.Quad{ + {"B", "status", "cool", "status_graph"}, + {"D", "status", "cool", "status_graph"}, + {"G", "status", "cool", "status_graph"}, + } + sort.Sort(ordered(expect)) + if got := iteratedQuads(qs, it); !reflect.DeepEqual(got, expect) { + t.Errorf("Failed to get expected results from predicate iterator, got:%v expect:%v", got, expect) + } + it.Reset() + + // Order is important + and = iterator.NewAnd() + and.AddSubIterator(qs.QuadIterator(quad.Subject, qs.ValueOf("B"))) + and.AddSubIterator(it) + + expect = []quad.Quad{ + {"B", "status", "cool", "status_graph"}, + } + if got := iteratedQuads(qs, and); !reflect.DeepEqual(got, expect) { + t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect) + } + it.Reset() + + // Order is important + and = iterator.NewAnd() + and.AddSubIterator(it) + and.AddSubIterator(qs.QuadIterator(quad.Subject, qs.ValueOf("B"))) + + expect = []quad.Quad{ + {"B", "status", "cool", "status_graph"}, + } + if got := iteratedQuads(qs, and); !reflect.DeepEqual(got, expect) { + t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect) + } +} + +func TestOptimize(t *testing.T) { + tmpFile, _ := ioutil.TempFile(os.TempDir(), "cayley_test") + t.Log(tmpFile.Name()) + defer os.RemoveAll(tmpFile.Name()) + err := createNewBolt(tmpFile.Name(), nil) + if err != nil { + t.Fatalf("Failed to create working directory") + } + qs, err := newQuadStore(tmpFile.Name(), nil) + if qs == nil || err != nil { + t.Error("Failed to create leveldb QuadStore.") + } + + w, _ := writer.NewSingleReplication(qs, nil) + w.AddQuadSet(makeQuadSet()) + + // With an linksto-fixed pair + fixed := qs.FixedIterator() + fixed.Add(qs.ValueOf("F")) + fixed.Tagger().Add("internal") + lto := iterator.NewLinksTo(qs, fixed, quad.Object) + + oldIt := lto.Clone() + newIt, ok := lto.Optimize() + if !ok { + t.Errorf("Failed to optimize iterator") + } + if newIt.Type() != Type() { + t.Errorf("Optimized iterator type does not match original, got:%v expect:%v", newIt.Type(), Type()) + } + + newQuads := iteratedQuads(qs, newIt) + oldQuads := iteratedQuads(qs, oldIt) + if !reflect.DeepEqual(newQuads, oldQuads) { + t.Errorf("Optimized iteration does not match original") + } + + graph.Next(oldIt) + oldResults := make(map[string]graph.Value) + oldIt.TagResults(oldResults) + graph.Next(newIt) + newResults := make(map[string]graph.Value) + newIt.TagResults(newResults) + if !reflect.DeepEqual(newResults, oldResults) { + t.Errorf("Discordant tag results, new:%v old:%v", newResults, oldResults) + } +} diff --git a/graph/bolt/quadstore.go b/graph/bolt/quadstore.go index e593b17..5b0e2e2 100644 --- a/graph/bolt/quadstore.go +++ b/graph/bolt/quadstore.go @@ -372,7 +372,7 @@ func (qs *QuadStore) Close() { } func (qs *QuadStore) Quad(k graph.Value) quad.Quad { - var q quad.Quad + var d graph.Delta tok := k.(*Token) err := qs.db.View(func(tx *bolt.Tx) error { b := tx.Bucket(tok.bucket) @@ -394,13 +394,13 @@ func (qs *QuadStore) Quad(k graph.Value) quad.Quad { // No harm, no foul. return nil } - return json.Unmarshal(data, &q) + return json.Unmarshal(data, &d) }) if err != nil { glog.Error("Error getting quad: ", err) return quad.Quad{} } - return q + return d.Quad } func (qs *QuadStore) ValueOf(s string) graph.Value { diff --git a/graph/primarykey.go b/graph/primarykey.go index 6b4c6a8..1df1c51 100644 --- a/graph/primarykey.go +++ b/graph/primarykey.go @@ -102,7 +102,7 @@ func (p *PrimaryKey) String() string { return "" } -func (p *PrimaryKey) MarshalJSON() ([]byte, error) { +func (p PrimaryKey) MarshalJSON() ([]byte, error) { switch p.keyType { case none: return nil, errors.New("Cannot marshal PrimaryKey with KeyType of 'none'")