From d3bc8c173609e892ba6c2a11e06dc4409562b0ba Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Tue, 26 Aug 2014 11:19:15 -0700 Subject: [PATCH 01/15] First implementations of loop and not operator for Cayley Gremlin. --- graph/iterator.go | 2 + graph/iterator/entrypoint_iterator.go | 91 ++++++++++++++ graph/iterator/loop_iterator.go | 218 ++++++++++++++++++++++++++++++++++ graph/iterator/not_iterator.go | 138 +++++++++++++++++++++ query/gremlin/build_iterator.go | 73 ++++++++++++ query/gremlin/traversals.go | 2 + 6 files changed, 524 insertions(+) create mode 100644 graph/iterator/entrypoint_iterator.go create mode 100644 graph/iterator/loop_iterator.go create mode 100644 graph/iterator/not_iterator.go diff --git a/graph/iterator.go b/graph/iterator.go index 83f1999..cba3100 100644 --- a/graph/iterator.go +++ b/graph/iterator.go @@ -206,6 +206,7 @@ const ( Not Optional Materialize + Loop ) var ( @@ -227,6 +228,7 @@ var ( "not", "optional", "materialize", + "loop", } ) diff --git a/graph/iterator/entrypoint_iterator.go b/graph/iterator/entrypoint_iterator.go new file mode 100644 index 0000000..d7185f0 --- /dev/null +++ b/graph/iterator/entrypoint_iterator.go @@ -0,0 +1,91 @@ +package iterator + +import ( + "fmt" + + "github.com/google/cayley/graph" +) + +type EntryPoint struct { + uid uint64 + primaryIt graph.Iterator +} + +func NewEntryPoint(it graph.Iterator) *EntryPoint { + return &EntryPoint{ + uid: NextUID(), + primaryIt: it, + } +} + +func (it *EntryPoint) UID() uint64 { + return it.uid +} + +func (it *EntryPoint) Tagger() *graph.Tagger { + return nil +} + +func (it *EntryPoint) TagResults(dst map[string]graph.Value) { + it.primaryIt.TagResults(dst) +} + +func (it *EntryPoint) Contains(val graph.Value) bool { + return it.primaryIt.Contains(val) +} + +func (it *EntryPoint) Clone() graph.Iterator { + return NewEntryPoint(it.primaryIt) +} + +func (it *EntryPoint) Type() graph.Type { return it.primaryIt.Type() } + +func (it *EntryPoint) Reset() { + it.primaryIt.Reset() +} + +func (it *EntryPoint) SetIterator(iterator graph.Iterator) { + it.primaryIt = iterator + it.primaryIt.Reset() +} + +func (it *EntryPoint) Next() bool { + return graph.Next(it.primaryIt) +} + +func (it *EntryPoint) Result() graph.Value { + return it.primaryIt.Result() +} + +func (it *EntryPoint) NextPath() bool { + return it.primaryIt.NextPath() +} + +func (it *EntryPoint) Stats() graph.IteratorStats { + return it.primaryIt.Stats() +} + +func (it *EntryPoint) Size() (int64, bool) { + return it.Stats().Size, false +} + +func (it *EntryPoint) Optimize() (graph.Iterator, bool) { + return it, false +} + +func (it *EntryPoint) SubIterators() []graph.Iterator { + return []graph.Iterator{it.primaryIt} +} + +func (it *EntryPoint) DebugString(indent int) string { + return fmt.Sprintf("todo") +} + +func (it *EntryPoint) Close() { + it.primaryIt.Close() +} + +// DEPRECATED +func (it *EntryPoint) ResultTree() *graph.ResultTree { + return graph.NewResultTree(it.Result()) +} diff --git a/graph/iterator/loop_iterator.go b/graph/iterator/loop_iterator.go new file mode 100644 index 0000000..86e0718 --- /dev/null +++ b/graph/iterator/loop_iterator.go @@ -0,0 +1,218 @@ +package iterator + +import ( + "fmt" + "strings" + + "github.com/google/cayley/graph" +) + +type Loop struct { + uid uint64 + tags graph.Tagger + ts graph.TripleStore + baseIt graph.Iterator + loopIt graph.Iterator + loopEntryIt *EntryPoint + filterIt graph.Iterator + filterEntryIt *EntryPoint + result graph.Value + runstats graph.IteratorStats + prevValuesIt graph.FixedIterator + loops int + bounded bool + loopsCompleted int + finished bool +} + +func NewLoop(ts graph.TripleStore, baseIt, loopIt, filterIt graph.Iterator, loopEntryIt, filterEntryIt *EntryPoint, loops int, bounded bool) *Loop { + return &Loop{ + uid: NextUID(), + ts: ts, + baseIt: baseIt, + loopEntryIt: loopEntryIt, + loopIt: loopIt, + filterEntryIt: filterEntryIt, + filterIt: filterIt, + prevValuesIt: ts.FixedIterator(), + loops: loops, + bounded: bounded, + loopsCompleted: 0, + finished: false, + } +} + +func (it *Loop) UID() uint64 { + return it.uid +} + +func (it *Loop) Tagger() *graph.Tagger { + return &it.tags +} + +func (it *Loop) TagResults(dst map[string]graph.Value) { + for _, tag := range it.tags.Tags() { + dst[tag] = it.Result() + } + + for tag, value := range it.tags.Fixed() { + dst[tag] = value + } + + it.baseIt.TagResults(dst) + it.loopIt.TagResults(dst) +} + +func (it *Loop) Contains(val graph.Value) bool { + graph.ContainsLogIn(it, val) + if it.loopIt.Contains(val) { + return graph.ContainsLogOut(it, val, true) + } + return graph.ContainsLogOut(it, val, false) +} + +func (it *Loop) Clone() graph.Iterator { + out := NewLoop(it.ts, it.baseIt, it.loopIt, it.filterIt, it.loopEntryIt, it.filterEntryIt, it.loops, it.bounded) + out.tags.CopyFrom(it) + return out +} + +func (it *Loop) Type() graph.Type { return graph.Loop } + +func (it *Loop) Reset() { + // Reset the iterators + it.baseIt.Reset() + it.loopEntryIt.SetIterator(it.baseIt) + it.loopIt.Reset() + it.prevValuesIt.Close() + it.prevValuesIt = it.ts.FixedIterator() + + // Reset the state + it.loopsCompleted = 0 + it.finished = false + it.result = nil +} + +func (it *Loop) advanceLoop() { + // Set the loop iterator to feed from the previous iteration results + it.loopEntryIt.SetIterator(it.prevValuesIt) + + // Reset the loop iterator - will also clean the values in the underlying fixed iterator. + it.loopIt.Reset() + + it.filterIt.Reset() + + // Increment the completed loops count + it.loopsCompleted += 1 + + // Mark the loop as finished - no more results can be expected. + // Either the number of loops has been executed, or there are + // no more expandable nodes. + if size, _ := it.prevValuesIt.Size(); (it.bounded && it.loopsCompleted >= it.loops) || size == 0 { + it.finished = true + } + + // Clean the set of values seen in the previous loop + it.prevValuesIt = it.ts.FixedIterator() +} + +func (it *Loop) checkFilter(value graph.Value) bool { + fixed := it.ts.FixedIterator() + fixed.Add(value) + + it.filterEntryIt.SetIterator(fixed) + it.filterIt.Reset() + + //fmt.Println("Before add value") + //it.filterEntryIt.Add(value) + //fmt.Println("After add value") + + //fmt.Println("Before next") + answer := graph.Next(it.filterIt) + //fmt.Println("After next") + return answer +} + +func (it *Loop) Next() bool { + graph.NextLogIn(it) + it.runstats.Next += 1 + + return it.next() +} + +func (it *Loop) next() bool { + // Check if the loop has any more results + if it.finished { + return graph.NextLogOut(it, nil, false) + } + + for i := 0; ; i++ { + if found := graph.Next(it.loopIt); !found { + // A value has not been found, try looping again + it.advanceLoop() + return it.next() + } + + if it.checkFilter(it.loopIt.Result()) { + // A value has been found. + it.result = it.loopIt.Result() + it.prevValuesIt.Add(it.result) + it.runstats.ContainsNext += 1 + + return graph.NextLogOut(it, it.result, true) + } + } +} + +func (it *Loop) Result() graph.Value { + return it.result +} + +func (it *Loop) NextPath() bool { + return it.loopIt.NextPath() +} + +func (it *Loop) Stats() graph.IteratorStats { + subitStats := it.loopIt.Stats() + // TODO(barakmich): These should really come from the triplestore itself + fanoutFactor := int64(20) + checkConstant := int64(1) + nextConstant := int64(2) + return graph.IteratorStats{ + NextCost: nextConstant + subitStats.NextCost, + ContainsCost: checkConstant + subitStats.ContainsCost, + Size: fanoutFactor * subitStats.Size, + Next: it.runstats.Next, + Contains: it.runstats.Contains, + ContainsNext: it.runstats.ContainsNext, + } +} + +func (it *Loop) Size() (int64, bool) { + return it.Stats().Size, false +} + +// TODO +func (it *Loop) Optimize() (graph.Iterator, bool) { + return it, false +} + +func (it *Loop) SubIterators() []graph.Iterator { + return []graph.Iterator{it.baseIt, it.loopIt} +} + +func (it *Loop) DebugString(indent int) string { + return fmt.Sprintf("%s(%s %d \n%s)", + strings.Repeat(" ", indent), + it.Type(), it.UID(), it.baseIt.DebugString(indent+4)) +} + +func (it *Loop) Close() { + it.baseIt.Close() + it.loopIt.Close() +} + +// DEPRECATED +func (it *Loop) ResultTree() *graph.ResultTree { + return graph.NewResultTree(it.Result()) +} diff --git a/graph/iterator/not_iterator.go b/graph/iterator/not_iterator.go new file mode 100644 index 0000000..8b2d0c9 --- /dev/null +++ b/graph/iterator/not_iterator.go @@ -0,0 +1,138 @@ +package iterator + +import "github.com/google/cayley/graph" + +type Not struct { + uid uint64 + tags graph.Tagger + primaryIt graph.Iterator + forbiddenIt graph.Iterator + result graph.Value + runstats graph.IteratorStats +} + +func NewNot(primaryIt, forbiddenIt graph.Iterator) *Not { + return &Not{ + uid: NextUID(), + primaryIt: primaryIt, + forbiddenIt: forbiddenIt, + } +} + +func (it *Not) UID() uint64 { + return it.uid +} + +func (it *Not) Reset() { + it.result = nil + it.primaryIt.Reset() + it.forbiddenIt.Reset() +} + +func (it *Not) Tagger() *graph.Tagger { + return &it.tags +} + +func (it *Not) TagResults(dst map[string]graph.Value) { + for _, tag := range it.tags.Tags() { + dst[tag] = it.Result() + } + + for tag, value := range it.tags.Fixed() { + dst[tag] = value + } + + if it.primaryIt != nil { + it.primaryIt.TagResults(dst) + } + + // todo +} + +func (it *Not) Clone() graph.Iterator { + not := NewNot(it.primaryIt.Clone(), it.forbiddenIt.Clone()) + not.tags.CopyFrom(it) + return not +} + +func (it *Not) SubIterators() []graph.Iterator { + return []graph.Iterator{it.primaryIt, it.forbiddenIt} +} + +func (it *Not) ResultTree() *graph.ResultTree { + tree := graph.NewResultTree(it.Result()) + tree.AddSubtree(it.primaryIt.ResultTree()) + tree.AddSubtree(it.forbiddenIt.ResultTree()) + return tree +} + +func (it *Not) DebugString(indent int) string { + return "todo" +} + +func (it *Not) Next() bool { + graph.NextLogIn(it) + it.runstats.Next += 1 + for graph.Next(it.primaryIt) { + if curr := it.primaryIt.Result(); !it.forbiddenIt.Contains(curr) { + it.result = curr + it.runstats.ContainsNext += 1 + return graph.NextLogOut(it, curr, true) + } + } + return graph.NextLogOut(it, nil, false) +} + +func (it *Not) Result() graph.Value { + return it.result +} + +func (it *Not) Contains(val graph.Value) bool { + graph.ContainsLogIn(it, val) + it.runstats.Contains += 1 + + mainGood := it.primaryIt.Contains(val) + if mainGood { + mainGood = !it.forbiddenIt.Contains(val) + } + return graph.ContainsLogOut(it, val, mainGood) +} + +func (it *Not) NextPath() bool { + if it.primaryIt.NextPath() { + return true + } + return it.forbiddenIt.NextPath() +} + +func (it *Not) Close() { + it.primaryIt.Close() + it.forbiddenIt.Close() +} + +func (it *Not) Type() graph.Type { return graph.Not } + +func (it *Not) Optimize() (graph.Iterator, bool) { + //it.forbiddenIt = NewMaterialize(it.forbiddenIt) + return it, false +} + +func (it *Not) Stats() graph.IteratorStats { + subitStats := it.primaryIt.Stats() + // TODO(barakmich): These should really come from the triplestore itself + fanoutFactor := int64(20) + checkConstant := int64(1) + nextConstant := int64(2) + return graph.IteratorStats{ + NextCost: nextConstant + subitStats.NextCost, + ContainsCost: checkConstant + subitStats.ContainsCost, + Size: fanoutFactor * subitStats.Size, + Next: it.runstats.Next, + Contains: it.runstats.Contains, + ContainsNext: it.runstats.ContainsNext, + } +} + +func (it *Not) Size() (int64, bool) { + return it.Stats().Size, false +} diff --git a/query/gremlin/build_iterator.go b/query/gremlin/build_iterator.go index 02196f4..85ad3a5 100644 --- a/query/gremlin/build_iterator.go +++ b/query/gremlin/build_iterator.go @@ -298,6 +298,79 @@ func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph. it = buildIteratorTreeHelper(arg.Object(), ts, subIt) case "in": it = buildInOutIterator(obj, ts, subIt, true) + case "not": + // Not is implemented as the difference between the primary iterator + // and the iterator chain of (primaryIt, follow, followR). + // Build the follow iterator + arg, _ := obj.Get("_gremlin_values") + firstArg, _ := arg.Object().Get("0") + if isVertexChain(firstArg.Object()) { + return iterator.NewNull() + } + + // Build the followR iterator + revArg, _ := obj.Get("_gremlin_followr") + if isVertexChain(revArg.Object()) { + return iterator.NewNull() + } + + followIt := buildIteratorTreeHelper(firstArg.Object(), ts, subIt) + forbiddenIt := buildIteratorTreeHelper(revArg.Object(), ts, followIt) + + it = iterator.NewNot(subIt, forbiddenIt) + case "loop": + arg, _ := obj.Get("_gremlin_values") + firstArg, _ := arg.Object().Get("0") + secondArg, _ := arg.Object().Get("1") + thirdArg, _ := arg.Object().Get("2") + + // Parse the loop iterating sequence + // Check if the first argument is a vertex chain + if isVertexChain(firstArg.Object()) { + return iterator.NewNull() + } + + loopEntryIt := iterator.NewEntryPoint(subIt) + loopIt := buildIteratorTreeHelper(firstArg.Object(), ts, loopEntryIt) + + // Parse the number of loops to execute + noLoops := 0 + bounded := false + if secondArg.IsNumber() { + if no, err := secondArg.ToInteger(); err == nil { + noLoops = int(no) + bounded = true + } else { + return iterator.NewNull() + } + } else if secondArg.IsBoolean() { + if boolVal, err := secondArg.ToBoolean(); err == nil && boolVal { + bounded = false + } else { + return iterator.NewNull() + } + } else { + thirdArg = secondArg + } + + // If the number of loops is negative, the loop is unbounded + if noLoops <= 0 { + bounded = false + } else { + bounded = true + } + + filterEntryIt := iterator.NewEntryPoint(nil) + var filterIt graph.Iterator + if thirdArg.IsNull() || thirdArg.IsUndefined() { + filterIt = filterEntryIt + } else if isVertexChain(thirdArg.Object()) { + return iterator.NewNull() + } else { + filterIt = buildIteratorTreeHelper(thirdArg.Object(), ts, filterEntryIt) + } + + it = iterator.NewLoop(ts, subIt, loopIt, filterIt, loopEntryIt, filterEntryIt, noLoops, bounded) } return it } diff --git a/query/gremlin/traversals.go b/query/gremlin/traversals.go index 2a2250b..6f752dc 100644 --- a/query/gremlin/traversals.go +++ b/query/gremlin/traversals.go @@ -38,6 +38,8 @@ func (wk *worker) embedTraversals(env *otto.Otto, obj *otto.Object) { obj.Set("Has", wk.gremlinFunc("has", obj, env)) obj.Set("Save", wk.gremlinFunc("save", obj, env)) obj.Set("SaveR", wk.gremlinFunc("saver", obj, env)) + obj.Set("Loop", gremlinFunc("loop", obj, env, ses)) + obj.Set("Not", gremlinFollowR("not", obj, env, ses)) } func (wk *worker) gremlinFunc(kind string, prev *otto.Object, env *otto.Otto) func(otto.FunctionCall) otto.Value { From d9b67c8335ffe68c25cae0f5af603f62f9317ab5 Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Tue, 26 Aug 2014 11:59:14 -0700 Subject: [PATCH 02/15] Added comments for the not operator. --- graph/iterator/not_iterator.go | 10 ++++++++-- query/gremlin/build_iterator.go | 9 ++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/graph/iterator/not_iterator.go b/graph/iterator/not_iterator.go index 8b2d0c9..2c6182f 100644 --- a/graph/iterator/not_iterator.go +++ b/graph/iterator/not_iterator.go @@ -2,6 +2,8 @@ package iterator import "github.com/google/cayley/graph" +// Not iterator acts like a set difference between the primary iterator +// and the forbidden iterator. type Not struct { uid uint64 tags graph.Tagger @@ -33,6 +35,7 @@ func (it *Not) Tagger() *graph.Tagger { return &it.tags } +// TODO func (it *Not) TagResults(dst map[string]graph.Value) { for _, tag := range it.tags.Tags() { dst[tag] = it.Result() @@ -45,8 +48,6 @@ func (it *Not) TagResults(dst map[string]graph.Value) { if it.primaryIt != nil { it.primaryIt.TagResults(dst) } - - // todo } func (it *Not) Clone() graph.Iterator { @@ -73,7 +74,10 @@ func (it *Not) DebugString(indent int) string { func (it *Not) Next() bool { graph.NextLogIn(it) it.runstats.Next += 1 + for graph.Next(it.primaryIt) { + // Consider only the elements from the primary set which are not + // contained in the forbidden set. if curr := it.primaryIt.Result(); !it.forbiddenIt.Contains(curr) { it.result = curr it.runstats.ContainsNext += 1 @@ -112,11 +116,13 @@ func (it *Not) Close() { func (it *Not) Type() graph.Type { return graph.Not } +// TODO func (it *Not) Optimize() (graph.Iterator, bool) { //it.forbiddenIt = NewMaterialize(it.forbiddenIt) return it, false } +// TODO func (it *Not) Stats() graph.IteratorStats { subitStats := it.primaryIt.Stats() // TODO(barakmich): These should really come from the triplestore itself diff --git a/query/gremlin/build_iterator.go b/query/gremlin/build_iterator.go index 85ad3a5..f7e69c7 100644 --- a/query/gremlin/build_iterator.go +++ b/query/gremlin/build_iterator.go @@ -300,21 +300,24 @@ func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph. it = buildInOutIterator(obj, ts, subIt, true) case "not": // Not is implemented as the difference between the primary iterator - // and the iterator chain of (primaryIt, follow, followR). - // Build the follow iterator + // and the iterator chain composed of (primaryIt->Follow->FollowR). + + // Arguments for follow iterator arg, _ := obj.Get("_gremlin_values") firstArg, _ := arg.Object().Get("0") if isVertexChain(firstArg.Object()) { return iterator.NewNull() } - // Build the followR iterator + // Arguments for followR iterator revArg, _ := obj.Get("_gremlin_followr") if isVertexChain(revArg.Object()) { return iterator.NewNull() } + // Build the primaryIt->Follow iterator followIt := buildIteratorTreeHelper(firstArg.Object(), ts, subIt) + // Build the primaryIt->Follow->FollowR iterator forbiddenIt := buildIteratorTreeHelper(revArg.Object(), ts, followIt) it = iterator.NewNot(subIt, forbiddenIt) From a0318aa7b24eadf027a20aeaa2385b2bc6628e80 Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Tue, 26 Aug 2014 12:26:54 -0700 Subject: [PATCH 03/15] Added comments for the loop iterator. --- graph/iterator/loop_iterator.go | 29 ++++++++++++++++++++++------- query/gremlin/build_iterator.go | 11 ++++++++--- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/graph/iterator/loop_iterator.go b/graph/iterator/loop_iterator.go index 86e0718..8a796fa 100644 --- a/graph/iterator/loop_iterator.go +++ b/graph/iterator/loop_iterator.go @@ -7,6 +7,14 @@ import ( "github.com/google/cayley/graph" ) +// Loop implements a loop operator. Composed of the following members: +// - baseIt - the iterator composing the query preceding the loop. +// - loopIt - an iterator implementing the loop morph query; has an EntryPoint iterator as starting point. +// - loopEntryIt - the starting point for the loop iterator; knowing this allows us to plug and change the source iterator +// - filterIt - an iterator implementing the filtering part of the loop; has an EntryPoint iterator as starting point +// - filterEntryIt - the starting point for the filter iterator; allows to plug and change the source iterator +// - prevValuesIt - the results obtained for each loop iteration will be stored in this iterator; +// this allows us to use this iterator as source for the next loop type Loop struct { uid uint64 tags graph.Tagger @@ -50,6 +58,7 @@ func (it *Loop) Tagger() *graph.Tagger { return &it.tags } +// TODO func (it *Loop) TagResults(dst map[string]graph.Value) { for _, tag := range it.tags.Tags() { dst[tag] = it.Result() @@ -63,6 +72,7 @@ func (it *Loop) TagResults(dst map[string]graph.Value) { it.loopIt.TagResults(dst) } +// TODO func (it *Loop) Contains(val graph.Value) bool { graph.ContainsLogIn(it, val) if it.loopIt.Contains(val) { @@ -71,6 +81,7 @@ func (it *Loop) Contains(val graph.Value) bool { return graph.ContainsLogOut(it, val, false) } +// TODO func (it *Loop) Clone() graph.Iterator { out := NewLoop(it.ts, it.baseIt, it.loopIt, it.filterIt, it.loopEntryIt, it.filterEntryIt, it.loops, it.bounded) out.tags.CopyFrom(it) @@ -116,20 +127,18 @@ func (it *Loop) advanceLoop() { it.prevValuesIt = it.ts.FixedIterator() } +// checkFilter checks whether a value is expandable using the filter iterator. func (it *Loop) checkFilter(value graph.Value) bool { + // Create a fixed iterator containing the value fixed := it.ts.FixedIterator() fixed.Add(value) + // Set is as the source for the filter iterator. it.filterEntryIt.SetIterator(fixed) it.filterIt.Reset() - //fmt.Println("Before add value") - //it.filterEntryIt.Add(value) - //fmt.Println("After add value") - - //fmt.Println("Before next") + // Check if the filter has a next value. answer := graph.Next(it.filterIt) - //fmt.Println("After next") return answer } @@ -148,11 +157,12 @@ func (it *Loop) next() bool { for i := 0; ; i++ { if found := graph.Next(it.loopIt); !found { - // A value has not been found, try looping again + // A value has not been found, try a new loop iteration. it.advanceLoop() return it.next() } + // For a found value, we must check it passes the filter. if it.checkFilter(it.loopIt.Result()) { // A value has been found. it.result = it.loopIt.Result() @@ -168,10 +178,12 @@ func (it *Loop) Result() graph.Value { return it.result } +// TODO func (it *Loop) NextPath() bool { return it.loopIt.NextPath() } +// TODO func (it *Loop) Stats() graph.IteratorStats { subitStats := it.loopIt.Stats() // TODO(barakmich): These should really come from the triplestore itself @@ -197,10 +209,12 @@ func (it *Loop) Optimize() (graph.Iterator, bool) { return it, false } +// TODO func (it *Loop) SubIterators() []graph.Iterator { return []graph.Iterator{it.baseIt, it.loopIt} } +// TODO func (it *Loop) DebugString(indent int) string { return fmt.Sprintf("%s(%s %d \n%s)", strings.Repeat(" ", indent), @@ -210,6 +224,7 @@ func (it *Loop) DebugString(indent int) string { func (it *Loop) Close() { it.baseIt.Close() it.loopIt.Close() + it.filterIt.Close() } // DEPRECATED diff --git a/query/gremlin/build_iterator.go b/query/gremlin/build_iterator.go index f7e69c7..60e53d7 100644 --- a/query/gremlin/build_iterator.go +++ b/query/gremlin/build_iterator.go @@ -328,15 +328,17 @@ func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph. thirdArg, _ := arg.Object().Get("2") // Parse the loop iterating sequence - // Check if the first argument is a vertex chain if isVertexChain(firstArg.Object()) { return iterator.NewNull() } + // Create the loop iterator: first, create an entry point iterator. loopEntryIt := iterator.NewEntryPoint(subIt) + // Then create a loop iterator on top of the entry point. loopIt := buildIteratorTreeHelper(firstArg.Object(), ts, loopEntryIt) - // Parse the number of loops to execute + // Parse the number of loops to execute. + // bounded=false means it will loop until no more results are produced. noLoops := 0 bounded := false if secondArg.IsNumber() { @@ -356,20 +358,23 @@ func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph. thirdArg = secondArg } - // If the number of loops is negative, the loop is unbounded + // If the number of loops is le 0, the loop is unbounded if noLoops <= 0 { bounded = false } else { bounded = true } + // Create the filter iterator filterEntryIt := iterator.NewEntryPoint(nil) var filterIt graph.Iterator if thirdArg.IsNull() || thirdArg.IsUndefined() { + // There is no filter morphism, use the entry point as a filter. filterIt = filterEntryIt } else if isVertexChain(thirdArg.Object()) { return iterator.NewNull() } else { + // There is a filter morphism, create the filter iterator based on the entry point. filterIt = buildIteratorTreeHelper(thirdArg.Object(), ts, filterEntryIt) } From 5d4e22498da75523322e1b40c532062b0faaf596 Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Wed, 27 Aug 2014 13:42:56 -0700 Subject: [PATCH 04/15] Converted the Not operator to a complement operator in terms of functionality. --- graph/iterator/not_iterator.go | 51 +++++++++++++++++++++-------------------- query/gremlin/build_iterator.go | 28 ++++++---------------- query/gremlin/traversals.go | 4 ++-- 3 files changed, 35 insertions(+), 48 deletions(-) diff --git a/graph/iterator/not_iterator.go b/graph/iterator/not_iterator.go index 2c6182f..0b35406 100644 --- a/graph/iterator/not_iterator.go +++ b/graph/iterator/not_iterator.go @@ -5,19 +5,21 @@ import "github.com/google/cayley/graph" // Not iterator acts like a set difference between the primary iterator // and the forbidden iterator. type Not struct { - uid uint64 - tags graph.Tagger - primaryIt graph.Iterator - forbiddenIt graph.Iterator - result graph.Value - runstats graph.IteratorStats + uid uint64 + tags graph.Tagger + ts graph.TripleStore + primaryIt graph.Iterator + allIt graph.Iterator + result graph.Value + runstats graph.IteratorStats } -func NewNot(primaryIt, forbiddenIt graph.Iterator) *Not { +func NewNot(ts graph.TripleStore, primaryIt graph.Iterator) *Not { return &Not{ - uid: NextUID(), - primaryIt: primaryIt, - forbiddenIt: forbiddenIt, + uid: NextUID(), + ts: ts, + allIt: ts.NodesAllIterator(), + primaryIt: primaryIt, } } @@ -28,14 +30,13 @@ func (it *Not) UID() uint64 { func (it *Not) Reset() { it.result = nil it.primaryIt.Reset() - it.forbiddenIt.Reset() + it.allIt = it.ts.NodesAllIterator() } func (it *Not) Tagger() *graph.Tagger { return &it.tags } -// TODO func (it *Not) TagResults(dst map[string]graph.Value) { for _, tag := range it.tags.Tags() { dst[tag] = it.Result() @@ -51,19 +52,19 @@ func (it *Not) TagResults(dst map[string]graph.Value) { } func (it *Not) Clone() graph.Iterator { - not := NewNot(it.primaryIt.Clone(), it.forbiddenIt.Clone()) + not := NewNot(it.ts, it.primaryIt.Clone()) not.tags.CopyFrom(it) return not } func (it *Not) SubIterators() []graph.Iterator { - return []graph.Iterator{it.primaryIt, it.forbiddenIt} + return []graph.Iterator{it.primaryIt, it.allIt} } func (it *Not) ResultTree() *graph.ResultTree { tree := graph.NewResultTree(it.Result()) tree.AddSubtree(it.primaryIt.ResultTree()) - tree.AddSubtree(it.forbiddenIt.ResultTree()) + tree.AddSubtree(it.allIt.ResultTree()) return tree } @@ -76,9 +77,7 @@ func (it *Not) Next() bool { it.runstats.Next += 1 for graph.Next(it.primaryIt) { - // Consider only the elements from the primary set which are not - // contained in the forbidden set. - if curr := it.primaryIt.Result(); !it.forbiddenIt.Contains(curr) { + if curr := it.allIt.Result(); !it.primaryIt.Contains(curr) { it.result = curr it.runstats.ContainsNext += 1 return graph.NextLogOut(it, curr, true) @@ -95,28 +94,30 @@ func (it *Not) Contains(val graph.Value) bool { graph.ContainsLogIn(it, val) it.runstats.Contains += 1 - mainGood := it.primaryIt.Contains(val) - if mainGood { - mainGood = !it.forbiddenIt.Contains(val) + if it.primaryIt.Contains(val) { + return graph.ContainsLogOut(it, val, false) } - return graph.ContainsLogOut(it, val, mainGood) + + // TODO - figure out if this really needs to be checked or it's safe to return true directly + return graph.ContainsLogOut(it, val, it.allIt.Contains(val)) } +// TODO func (it *Not) NextPath() bool { if it.primaryIt.NextPath() { return true } - return it.forbiddenIt.NextPath() + return false } func (it *Not) Close() { it.primaryIt.Close() - it.forbiddenIt.Close() + it.allIt.Close() } func (it *Not) Type() graph.Type { return graph.Not } -// TODO +// TODO - call optimize for the primaryIt and allIt? func (it *Not) Optimize() (graph.Iterator, bool) { //it.forbiddenIt = NewMaterialize(it.forbiddenIt) return it, false diff --git a/query/gremlin/build_iterator.go b/query/gremlin/build_iterator.go index 60e53d7..11fd453 100644 --- a/query/gremlin/build_iterator.go +++ b/query/gremlin/build_iterator.go @@ -299,28 +299,14 @@ func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph. case "in": it = buildInOutIterator(obj, ts, subIt, true) case "not": - // Not is implemented as the difference between the primary iterator - // and the iterator chain composed of (primaryIt->Follow->FollowR). + // arg, _ := obj.Get("_gremlin_values") + // firstArg, _ := arg.Object().Get("0") + // if !isVertexChain(firstArg.Object()) { + // return iterator.NewNull() + // } + // forbiddenIt := buildIteratorTree(firstArg.Object(), ts) - // Arguments for follow iterator - arg, _ := obj.Get("_gremlin_values") - firstArg, _ := arg.Object().Get("0") - if isVertexChain(firstArg.Object()) { - return iterator.NewNull() - } - - // Arguments for followR iterator - revArg, _ := obj.Get("_gremlin_followr") - if isVertexChain(revArg.Object()) { - return iterator.NewNull() - } - - // Build the primaryIt->Follow iterator - followIt := buildIteratorTreeHelper(firstArg.Object(), ts, subIt) - // Build the primaryIt->Follow->FollowR iterator - forbiddenIt := buildIteratorTreeHelper(revArg.Object(), ts, followIt) - - it = iterator.NewNot(subIt, forbiddenIt) + it = iterator.NewNot(ts, subIt) case "loop": arg, _ := obj.Get("_gremlin_values") firstArg, _ := arg.Object().Get("0") diff --git a/query/gremlin/traversals.go b/query/gremlin/traversals.go index 6f752dc..83d3853 100644 --- a/query/gremlin/traversals.go +++ b/query/gremlin/traversals.go @@ -38,8 +38,8 @@ func (wk *worker) embedTraversals(env *otto.Otto, obj *otto.Object) { obj.Set("Has", wk.gremlinFunc("has", obj, env)) obj.Set("Save", wk.gremlinFunc("save", obj, env)) obj.Set("SaveR", wk.gremlinFunc("saver", obj, env)) - obj.Set("Loop", gremlinFunc("loop", obj, env, ses)) - obj.Set("Not", gremlinFollowR("not", obj, env, ses)) + obj.Set("Loop", wk.gremlinFunc("loop", obj, env)) + obj.Set("Not", wk.gremlinFunc("not", obj, env)) } func (wk *worker) gremlinFunc(kind string, prev *otto.Object, env *otto.Otto) func(otto.FunctionCall) otto.Value { From def70ca12e784547feb03cf671a061b10345c814 Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Wed, 27 Aug 2014 14:48:30 -0700 Subject: [PATCH 05/15] Exposed Except (alias Difference) operator in Gremlin API. --- query/gremlin/build_iterator.go | 19 ++++++++++++------- query/gremlin/traversals.go | 3 ++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/query/gremlin/build_iterator.go b/query/gremlin/build_iterator.go index 11fd453..5d285de 100644 --- a/query/gremlin/build_iterator.go +++ b/query/gremlin/build_iterator.go @@ -299,14 +299,19 @@ func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph. case "in": it = buildInOutIterator(obj, ts, subIt, true) case "not": - // arg, _ := obj.Get("_gremlin_values") - // firstArg, _ := arg.Object().Get("0") - // if !isVertexChain(firstArg.Object()) { - // return iterator.NewNull() - // } - // forbiddenIt := buildIteratorTree(firstArg.Object(), ts) + arg, _ := obj.Get("_gremlin_values") + firstArg, _ := arg.Object().Get("0") + if !isVertexChain(firstArg.Object()) { + return iterator.NewNull() + } - it = iterator.NewNot(ts, subIt) + toComplementIt := buildIteratorTree(firstArg.Object(), ts) + notIt := iterator.NewNot(ts, toComplementIt) + + and := iterator.NewAnd() + and.AddSubIterator(subIt) + and.AddSubIterator(notIt) + it = and case "loop": arg, _ := obj.Get("_gremlin_values") firstArg, _ := arg.Object().Get("0") diff --git a/query/gremlin/traversals.go b/query/gremlin/traversals.go index 83d3853..e22a5c6 100644 --- a/query/gremlin/traversals.go +++ b/query/gremlin/traversals.go @@ -39,7 +39,8 @@ func (wk *worker) embedTraversals(env *otto.Otto, obj *otto.Object) { obj.Set("Save", wk.gremlinFunc("save", obj, env)) obj.Set("SaveR", wk.gremlinFunc("saver", obj, env)) obj.Set("Loop", wk.gremlinFunc("loop", obj, env)) - obj.Set("Not", wk.gremlinFunc("not", obj, env)) + obj.Set("Except", wk.gremlinFunc("not", obj, env)) + obj.Set("Difference", wk.gremlinFunc("not", obj, env)) } func (wk *worker) gremlinFunc(kind string, prev *otto.Object, env *otto.Otto) func(otto.FunctionCall) otto.Value { From a5fd1905d0885c359ab26f8faa96015f6f815a2c Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Thu, 28 Aug 2014 16:02:21 -0700 Subject: [PATCH 06/15] Improvements to the Not iterator: - Pass the all iterator as an argument - Fixed next method (call all.Next() ). --- graph/iterator/not_iterator.go | 69 +++++++++++++++++++++++++++++------------ query/gremlin/build_iterator.go | 3 +- 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/graph/iterator/not_iterator.go b/graph/iterator/not_iterator.go index 0b35406..f6e7eec 100644 --- a/graph/iterator/not_iterator.go +++ b/graph/iterator/not_iterator.go @@ -1,25 +1,28 @@ package iterator -import "github.com/google/cayley/graph" +import ( + "fmt" + "strings" -// Not iterator acts like a set difference between the primary iterator -// and the forbidden iterator. + "github.com/google/cayley/graph" +) + +// Not iterator acts like a complement for the primary iterator. +// It will return all the vertices which are not part of the primary iterator. type Not struct { uid uint64 tags graph.Tagger - ts graph.TripleStore primaryIt graph.Iterator allIt graph.Iterator result graph.Value runstats graph.IteratorStats } -func NewNot(ts graph.TripleStore, primaryIt graph.Iterator) *Not { +func NewNot(primaryIt, allIt graph.Iterator) *Not { return &Not{ uid: NextUID(), - ts: ts, - allIt: ts.NodesAllIterator(), primaryIt: primaryIt, + allIt: allIt, } } @@ -27,10 +30,11 @@ func (it *Not) UID() uint64 { return it.uid } +// Reset resets the internal iterators and the iterator itself. func (it *Not) Reset() { it.result = nil it.primaryIt.Reset() - it.allIt = it.ts.NodesAllIterator() + it.allIt.Reset() } func (it *Not) Tagger() *graph.Tagger { @@ -52,15 +56,19 @@ func (it *Not) TagResults(dst map[string]graph.Value) { } func (it *Not) Clone() graph.Iterator { - not := NewNot(it.ts, it.primaryIt.Clone()) + not := NewNot(it.primaryIt.Clone(), it.allIt.Clone()) not.tags.CopyFrom(it) return not } +// SubIterators returns a slice of the sub iterators. +// The first iterator is the primary iterator, for which the complement +// is generated. func (it *Not) SubIterators() []graph.Iterator { return []graph.Iterator{it.primaryIt, it.allIt} } +// DEPRECATED func (it *Not) ResultTree() *graph.ResultTree { tree := graph.NewResultTree(it.Result()) tree.AddSubtree(it.primaryIt.ResultTree()) @@ -68,15 +76,33 @@ func (it *Not) ResultTree() *graph.ResultTree { return tree } +// DebugString prints information about the iterator. func (it *Not) DebugString(indent int) string { - return "todo" + var tags string + for _, k := range it.tags.Tags() { + tags += fmt.Sprintf("%s;", k) + } + + spaces := strings.Repeat(" ", indent+2) + return fmt.Sprintf("%s(%s %d\n%stags:%v\n%sprimary_it:\n%s\n)", + strings.Repeat(" ", indent), + it.Type(), + it.UID(), + spaces, + it.tags.Tags(), + spaces, + it.primaryIt.DebugString(indent+4), + ) } +// Next advances the Not iterator. It returns whether there is another valid +// new value. It fetches the next value of the all iterator which is not +// contained by the primary iterator. func (it *Not) Next() bool { graph.NextLogIn(it) it.runstats.Next += 1 - for graph.Next(it.primaryIt) { + for graph.Next(it.allIt) { if curr := it.allIt.Result(); !it.primaryIt.Contains(curr) { it.result = curr it.runstats.ContainsNext += 1 @@ -90,6 +116,9 @@ func (it *Not) Result() graph.Value { return it.result } +// Contains checks whether the passed value is part of the primary iterator's +// complement. For a valid value, it updates the Result returned by the iterator +// to the value itself. func (it *Not) Contains(val graph.Value) bool { graph.ContainsLogIn(it, val) it.runstats.Contains += 1 @@ -98,15 +127,13 @@ func (it *Not) Contains(val graph.Value) bool { return graph.ContainsLogOut(it, val, false) } - // TODO - figure out if this really needs to be checked or it's safe to return true directly - return graph.ContainsLogOut(it, val, it.allIt.Contains(val)) + it.result = val + return graph.ContainsLogOut(it, val, true) } -// TODO +// NextPath checks whether there is another path. Not applicable, hence it will +// return false. func (it *Not) NextPath() bool { - if it.primaryIt.NextPath() { - return true - } return false } @@ -117,13 +144,15 @@ func (it *Not) Close() { func (it *Not) Type() graph.Type { return graph.Not } -// TODO - call optimize for the primaryIt and allIt? func (it *Not) Optimize() (graph.Iterator, bool) { - //it.forbiddenIt = NewMaterialize(it.forbiddenIt) + // TODO - consider wrapping the primaryIt with a MaterializeIt + if optimizedPrimaryIt, optimized := it.primaryIt.Optimize(); optimized { + it.primaryIt = optimizedPrimaryIt + return it, true + } return it, false } -// TODO func (it *Not) Stats() graph.IteratorStats { subitStats := it.primaryIt.Stats() // TODO(barakmich): These should really come from the triplestore itself diff --git a/query/gremlin/build_iterator.go b/query/gremlin/build_iterator.go index 5d285de..a855637 100644 --- a/query/gremlin/build_iterator.go +++ b/query/gremlin/build_iterator.go @@ -305,8 +305,9 @@ func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph. return iterator.NewNull() } + allIt := ts.NodesAllIterator() toComplementIt := buildIteratorTree(firstArg.Object(), ts) - notIt := iterator.NewNot(ts, toComplementIt) + notIt := iterator.NewNot(toComplementIt, allIt) and := iterator.NewAnd() and.AddSubIterator(subIt) From 305815e663dd7e6359ce78ea6a563b06d4ed4317 Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Thu, 28 Aug 2014 16:23:15 -0700 Subject: [PATCH 07/15] Fixed Stats method for Not iterator. Added unit tests for Not iterator. --- graph/iterator/not_iterator.go | 13 +++++------ graph/iterator/not_iterator_test.go | 44 +++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 graph/iterator/not_iterator_test.go diff --git a/graph/iterator/not_iterator.go b/graph/iterator/not_iterator.go index f6e7eec..32d8957 100644 --- a/graph/iterator/not_iterator.go +++ b/graph/iterator/not_iterator.go @@ -154,15 +154,12 @@ func (it *Not) Optimize() (graph.Iterator, bool) { } func (it *Not) Stats() graph.IteratorStats { - subitStats := it.primaryIt.Stats() - // TODO(barakmich): These should really come from the triplestore itself - fanoutFactor := int64(20) - checkConstant := int64(1) - nextConstant := int64(2) + primaryStats := it.primaryIt.Stats() + allStats := it.allIt.Stats() return graph.IteratorStats{ - NextCost: nextConstant + subitStats.NextCost, - ContainsCost: checkConstant + subitStats.ContainsCost, - Size: fanoutFactor * subitStats.Size, + NextCost: allStats.NextCost + primaryStats.ContainsCost, + ContainsCost: primaryStats.ContainsCost, + Size: allStats.Size - primaryStats.Size, Next: it.runstats.Next, Contains: it.runstats.Contains, ContainsNext: it.runstats.ContainsNext, diff --git a/graph/iterator/not_iterator_test.go b/graph/iterator/not_iterator_test.go new file mode 100644 index 0000000..3414ca0 --- /dev/null +++ b/graph/iterator/not_iterator_test.go @@ -0,0 +1,44 @@ +package iterator + +import ( + "reflect" + "testing" +) + +func TestNotIteratorBasics(t *testing.T) { + allIt := newFixed() + allIt.Add(1) + allIt.Add(2) + allIt.Add(3) + allIt.Add(4) + + toComplementIt := newFixed() + toComplementIt.Add(2) + toComplementIt.Add(4) + + not := NewNot(toComplementIt, allIt) + + if v, _ := not.Size(); v != 2 { + t.Errorf("Unexpected iterator size: got:%d, expected: %d", v, 2) + } + + expect := []int{1, 3} + for i := 0; i < 2; i++ { + if got := iterated(not); !reflect.DeepEqual(got, expect) { + t.Errorf("Failed to iterate Not correctly on repeat %d: got:%v expected:%v", i, got, expect) + } + not.Reset() + } + + for _, v := range []int{1, 3} { + if !not.Contains(v) { + t.Errorf("Failed to correctly check %d as true", v) + } + } + + for _, v := range []int{2, 4} { + if not.Contains(v) { + t.Errorf("Failed to correctly check %d as false", v) + } + } +} From b97e6782ff75c0cd3fbcd8625adedb03cc3886c1 Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Thu, 28 Aug 2014 17:52:58 -0700 Subject: [PATCH 08/15] Changed identifier "not" to "except" for iterator builder. --- query/gremlin/build_iterator.go | 2 +- query/gremlin/traversals.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/query/gremlin/build_iterator.go b/query/gremlin/build_iterator.go index a855637..8f718c6 100644 --- a/query/gremlin/build_iterator.go +++ b/query/gremlin/build_iterator.go @@ -298,7 +298,7 @@ func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph. it = buildIteratorTreeHelper(arg.Object(), ts, subIt) case "in": it = buildInOutIterator(obj, ts, subIt, true) - case "not": + case "except": arg, _ := obj.Get("_gremlin_values") firstArg, _ := arg.Object().Get("0") if !isVertexChain(firstArg.Object()) { diff --git a/query/gremlin/traversals.go b/query/gremlin/traversals.go index e22a5c6..8177d80 100644 --- a/query/gremlin/traversals.go +++ b/query/gremlin/traversals.go @@ -39,8 +39,8 @@ func (wk *worker) embedTraversals(env *otto.Otto, obj *otto.Object) { obj.Set("Save", wk.gremlinFunc("save", obj, env)) obj.Set("SaveR", wk.gremlinFunc("saver", obj, env)) obj.Set("Loop", wk.gremlinFunc("loop", obj, env)) - obj.Set("Except", wk.gremlinFunc("not", obj, env)) - obj.Set("Difference", wk.gremlinFunc("not", obj, env)) + obj.Set("Except", wk.gremlinFunc("except", obj, env)) + obj.Set("Difference", wk.gremlinFunc("except", obj, env)) } func (wk *worker) gremlinFunc(kind string, prev *otto.Object, env *otto.Otto) func(otto.FunctionCall) otto.Value { From 55d235bd55b7ef112fb495e2aebf98217078f78a Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Thu, 28 Aug 2014 18:56:06 -0700 Subject: [PATCH 09/15] Added Gremlin tests for the Except operator. --- graph/iterator/not_iterator.go | 6 +++--- query/gremlin/gremlin_test.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/graph/iterator/not_iterator.go b/graph/iterator/not_iterator.go index 32d8957..28d6e84 100644 --- a/graph/iterator/not_iterator.go +++ b/graph/iterator/not_iterator.go @@ -146,11 +146,11 @@ func (it *Not) Type() graph.Type { return graph.Not } func (it *Not) Optimize() (graph.Iterator, bool) { // TODO - consider wrapping the primaryIt with a MaterializeIt - if optimizedPrimaryIt, optimized := it.primaryIt.Optimize(); optimized { + optimizedPrimaryIt, optimized := it.primaryIt.Optimize() + if optimized { it.primaryIt = optimizedPrimaryIt - return it, true } - return it, false + return it, optimized } func (it *Not) Stats() graph.IteratorStats { diff --git a/query/gremlin/gremlin_test.go b/query/gremlin/gremlin_test.go index 6d9f0cf..f2eec35 100644 --- a/query/gremlin/gremlin_test.go +++ b/query/gremlin/gremlin_test.go @@ -120,6 +120,20 @@ var testQueries = []struct { tag: "acd", expect: []string{"D"}, }, + { + message: "use Except to filter out a single vertex", + query: ` + g.V("A", "B").Except(g.V("A")).All() + `, + expect: []string{"B"}, + }, + { + message: "use chained Except", + query: ` + g.V("A", "B", "C").Except(g.V("B")).Except(g.V("C")).All() + `, + expect: []string{"A"}, + }, // Morphism tests. { From 20bd08d5babb43cb12eaac7412e5d8bdb65b05ea Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Fri, 5 Sep 2014 16:07:34 -0700 Subject: [PATCH 10/15] Fixed Optimize method for not iterator. --- graph/iterator/not_iterator.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/graph/iterator/not_iterator.go b/graph/iterator/not_iterator.go index 28d6e84..08a2542 100644 --- a/graph/iterator/not_iterator.go +++ b/graph/iterator/not_iterator.go @@ -84,7 +84,7 @@ func (it *Not) DebugString(indent int) string { } spaces := strings.Repeat(" ", indent+2) - return fmt.Sprintf("%s(%s %d\n%stags:%v\n%sprimary_it:\n%s\n)", + return fmt.Sprintf("%s(%s %d\n%stags:%v\n%sprimary_it:\n%s\n%sall_it:\n%s\n%s)", strings.Repeat(" ", indent), it.Type(), it.UID(), @@ -92,6 +92,9 @@ func (it *Not) DebugString(indent int) string { it.tags.Tags(), spaces, it.primaryIt.DebugString(indent+4), + spaces, + it.allIt.DebugString(indent+4), + strings.Repeat(" ", indent), ) } @@ -150,7 +153,7 @@ func (it *Not) Optimize() (graph.Iterator, bool) { if optimized { it.primaryIt = optimizedPrimaryIt } - return it, optimized + return it, false } func (it *Not) Stats() graph.IteratorStats { From a742ebcd467d65e919d8ae5206e6f8b417da7225 Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Mon, 8 Sep 2014 17:09:42 -0700 Subject: [PATCH 11/15] Cleaned up code for pull request for an Except operator. --- graph/iterator/entrypoint_iterator.go | 91 ------------- graph/iterator/loop_iterator.go | 233 ---------------------------------- query/gremlin/traversals.go | 1 - 3 files changed, 325 deletions(-) delete mode 100644 graph/iterator/entrypoint_iterator.go delete mode 100644 graph/iterator/loop_iterator.go diff --git a/graph/iterator/entrypoint_iterator.go b/graph/iterator/entrypoint_iterator.go deleted file mode 100644 index d7185f0..0000000 --- a/graph/iterator/entrypoint_iterator.go +++ /dev/null @@ -1,91 +0,0 @@ -package iterator - -import ( - "fmt" - - "github.com/google/cayley/graph" -) - -type EntryPoint struct { - uid uint64 - primaryIt graph.Iterator -} - -func NewEntryPoint(it graph.Iterator) *EntryPoint { - return &EntryPoint{ - uid: NextUID(), - primaryIt: it, - } -} - -func (it *EntryPoint) UID() uint64 { - return it.uid -} - -func (it *EntryPoint) Tagger() *graph.Tagger { - return nil -} - -func (it *EntryPoint) TagResults(dst map[string]graph.Value) { - it.primaryIt.TagResults(dst) -} - -func (it *EntryPoint) Contains(val graph.Value) bool { - return it.primaryIt.Contains(val) -} - -func (it *EntryPoint) Clone() graph.Iterator { - return NewEntryPoint(it.primaryIt) -} - -func (it *EntryPoint) Type() graph.Type { return it.primaryIt.Type() } - -func (it *EntryPoint) Reset() { - it.primaryIt.Reset() -} - -func (it *EntryPoint) SetIterator(iterator graph.Iterator) { - it.primaryIt = iterator - it.primaryIt.Reset() -} - -func (it *EntryPoint) Next() bool { - return graph.Next(it.primaryIt) -} - -func (it *EntryPoint) Result() graph.Value { - return it.primaryIt.Result() -} - -func (it *EntryPoint) NextPath() bool { - return it.primaryIt.NextPath() -} - -func (it *EntryPoint) Stats() graph.IteratorStats { - return it.primaryIt.Stats() -} - -func (it *EntryPoint) Size() (int64, bool) { - return it.Stats().Size, false -} - -func (it *EntryPoint) Optimize() (graph.Iterator, bool) { - return it, false -} - -func (it *EntryPoint) SubIterators() []graph.Iterator { - return []graph.Iterator{it.primaryIt} -} - -func (it *EntryPoint) DebugString(indent int) string { - return fmt.Sprintf("todo") -} - -func (it *EntryPoint) Close() { - it.primaryIt.Close() -} - -// DEPRECATED -func (it *EntryPoint) ResultTree() *graph.ResultTree { - return graph.NewResultTree(it.Result()) -} diff --git a/graph/iterator/loop_iterator.go b/graph/iterator/loop_iterator.go deleted file mode 100644 index 8a796fa..0000000 --- a/graph/iterator/loop_iterator.go +++ /dev/null @@ -1,233 +0,0 @@ -package iterator - -import ( - "fmt" - "strings" - - "github.com/google/cayley/graph" -) - -// Loop implements a loop operator. Composed of the following members: -// - baseIt - the iterator composing the query preceding the loop. -// - loopIt - an iterator implementing the loop morph query; has an EntryPoint iterator as starting point. -// - loopEntryIt - the starting point for the loop iterator; knowing this allows us to plug and change the source iterator -// - filterIt - an iterator implementing the filtering part of the loop; has an EntryPoint iterator as starting point -// - filterEntryIt - the starting point for the filter iterator; allows to plug and change the source iterator -// - prevValuesIt - the results obtained for each loop iteration will be stored in this iterator; -// this allows us to use this iterator as source for the next loop -type Loop struct { - uid uint64 - tags graph.Tagger - ts graph.TripleStore - baseIt graph.Iterator - loopIt graph.Iterator - loopEntryIt *EntryPoint - filterIt graph.Iterator - filterEntryIt *EntryPoint - result graph.Value - runstats graph.IteratorStats - prevValuesIt graph.FixedIterator - loops int - bounded bool - loopsCompleted int - finished bool -} - -func NewLoop(ts graph.TripleStore, baseIt, loopIt, filterIt graph.Iterator, loopEntryIt, filterEntryIt *EntryPoint, loops int, bounded bool) *Loop { - return &Loop{ - uid: NextUID(), - ts: ts, - baseIt: baseIt, - loopEntryIt: loopEntryIt, - loopIt: loopIt, - filterEntryIt: filterEntryIt, - filterIt: filterIt, - prevValuesIt: ts.FixedIterator(), - loops: loops, - bounded: bounded, - loopsCompleted: 0, - finished: false, - } -} - -func (it *Loop) UID() uint64 { - return it.uid -} - -func (it *Loop) Tagger() *graph.Tagger { - return &it.tags -} - -// TODO -func (it *Loop) TagResults(dst map[string]graph.Value) { - for _, tag := range it.tags.Tags() { - dst[tag] = it.Result() - } - - for tag, value := range it.tags.Fixed() { - dst[tag] = value - } - - it.baseIt.TagResults(dst) - it.loopIt.TagResults(dst) -} - -// TODO -func (it *Loop) Contains(val graph.Value) bool { - graph.ContainsLogIn(it, val) - if it.loopIt.Contains(val) { - return graph.ContainsLogOut(it, val, true) - } - return graph.ContainsLogOut(it, val, false) -} - -// TODO -func (it *Loop) Clone() graph.Iterator { - out := NewLoop(it.ts, it.baseIt, it.loopIt, it.filterIt, it.loopEntryIt, it.filterEntryIt, it.loops, it.bounded) - out.tags.CopyFrom(it) - return out -} - -func (it *Loop) Type() graph.Type { return graph.Loop } - -func (it *Loop) Reset() { - // Reset the iterators - it.baseIt.Reset() - it.loopEntryIt.SetIterator(it.baseIt) - it.loopIt.Reset() - it.prevValuesIt.Close() - it.prevValuesIt = it.ts.FixedIterator() - - // Reset the state - it.loopsCompleted = 0 - it.finished = false - it.result = nil -} - -func (it *Loop) advanceLoop() { - // Set the loop iterator to feed from the previous iteration results - it.loopEntryIt.SetIterator(it.prevValuesIt) - - // Reset the loop iterator - will also clean the values in the underlying fixed iterator. - it.loopIt.Reset() - - it.filterIt.Reset() - - // Increment the completed loops count - it.loopsCompleted += 1 - - // Mark the loop as finished - no more results can be expected. - // Either the number of loops has been executed, or there are - // no more expandable nodes. - if size, _ := it.prevValuesIt.Size(); (it.bounded && it.loopsCompleted >= it.loops) || size == 0 { - it.finished = true - } - - // Clean the set of values seen in the previous loop - it.prevValuesIt = it.ts.FixedIterator() -} - -// checkFilter checks whether a value is expandable using the filter iterator. -func (it *Loop) checkFilter(value graph.Value) bool { - // Create a fixed iterator containing the value - fixed := it.ts.FixedIterator() - fixed.Add(value) - - // Set is as the source for the filter iterator. - it.filterEntryIt.SetIterator(fixed) - it.filterIt.Reset() - - // Check if the filter has a next value. - answer := graph.Next(it.filterIt) - return answer -} - -func (it *Loop) Next() bool { - graph.NextLogIn(it) - it.runstats.Next += 1 - - return it.next() -} - -func (it *Loop) next() bool { - // Check if the loop has any more results - if it.finished { - return graph.NextLogOut(it, nil, false) - } - - for i := 0; ; i++ { - if found := graph.Next(it.loopIt); !found { - // A value has not been found, try a new loop iteration. - it.advanceLoop() - return it.next() - } - - // For a found value, we must check it passes the filter. - if it.checkFilter(it.loopIt.Result()) { - // A value has been found. - it.result = it.loopIt.Result() - it.prevValuesIt.Add(it.result) - it.runstats.ContainsNext += 1 - - return graph.NextLogOut(it, it.result, true) - } - } -} - -func (it *Loop) Result() graph.Value { - return it.result -} - -// TODO -func (it *Loop) NextPath() bool { - return it.loopIt.NextPath() -} - -// TODO -func (it *Loop) Stats() graph.IteratorStats { - subitStats := it.loopIt.Stats() - // TODO(barakmich): These should really come from the triplestore itself - fanoutFactor := int64(20) - checkConstant := int64(1) - nextConstant := int64(2) - return graph.IteratorStats{ - NextCost: nextConstant + subitStats.NextCost, - ContainsCost: checkConstant + subitStats.ContainsCost, - Size: fanoutFactor * subitStats.Size, - Next: it.runstats.Next, - Contains: it.runstats.Contains, - ContainsNext: it.runstats.ContainsNext, - } -} - -func (it *Loop) Size() (int64, bool) { - return it.Stats().Size, false -} - -// TODO -func (it *Loop) Optimize() (graph.Iterator, bool) { - return it, false -} - -// TODO -func (it *Loop) SubIterators() []graph.Iterator { - return []graph.Iterator{it.baseIt, it.loopIt} -} - -// TODO -func (it *Loop) DebugString(indent int) string { - return fmt.Sprintf("%s(%s %d \n%s)", - strings.Repeat(" ", indent), - it.Type(), it.UID(), it.baseIt.DebugString(indent+4)) -} - -func (it *Loop) Close() { - it.baseIt.Close() - it.loopIt.Close() - it.filterIt.Close() -} - -// DEPRECATED -func (it *Loop) ResultTree() *graph.ResultTree { - return graph.NewResultTree(it.Result()) -} diff --git a/query/gremlin/traversals.go b/query/gremlin/traversals.go index 8177d80..1576fc1 100644 --- a/query/gremlin/traversals.go +++ b/query/gremlin/traversals.go @@ -38,7 +38,6 @@ func (wk *worker) embedTraversals(env *otto.Otto, obj *otto.Object) { obj.Set("Has", wk.gremlinFunc("has", obj, env)) obj.Set("Save", wk.gremlinFunc("save", obj, env)) obj.Set("SaveR", wk.gremlinFunc("saver", obj, env)) - obj.Set("Loop", wk.gremlinFunc("loop", obj, env)) obj.Set("Except", wk.gremlinFunc("except", obj, env)) obj.Set("Difference", wk.gremlinFunc("except", obj, env)) } From 985161325e5b2e7b400d870d381f3bfa3fafa2b3 Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Mon, 8 Sep 2014 17:13:05 -0700 Subject: [PATCH 12/15] Changed triplestore to quadstore in the Except operator build_iterator code. --- query/gremlin/build_iterator.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/query/gremlin/build_iterator.go b/query/gremlin/build_iterator.go index a814924..d95a5e8 100644 --- a/query/gremlin/build_iterator.go +++ b/query/gremlin/build_iterator.go @@ -297,7 +297,7 @@ func buildIteratorTreeHelper(obj *otto.Object, qs graph.QuadStore, base graph.It } it = buildIteratorTreeHelper(arg.Object(), qs, subIt) case "in": - it = buildInOutIterator(obj, ts, subIt, true) + it = buildInOutIterator(obj, qs, subIt, true) case "except": arg, _ := obj.Get("_gremlin_values") firstArg, _ := arg.Object().Get("0") @@ -305,8 +305,8 @@ func buildIteratorTreeHelper(obj *otto.Object, qs graph.QuadStore, base graph.It return iterator.NewNull() } - allIt := ts.NodesAllIterator() - toComplementIt := buildIteratorTree(firstArg.Object(), ts) + allIt := qs.NodesAllIterator() + toComplementIt := buildIteratorTree(firstArg.Object(), qs) notIt := iterator.NewNot(toComplementIt, allIt) and := iterator.NewAnd() From 1d0540af179dd78b270f6e0bf87fbc03bc0a488b Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Mon, 8 Sep 2014 17:17:46 -0700 Subject: [PATCH 13/15] Removed "loop" iterator type. --- graph/iterator.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/graph/iterator.go b/graph/iterator.go index aeaa9b3..9fe2d12 100644 --- a/graph/iterator.go +++ b/graph/iterator.go @@ -206,7 +206,6 @@ const ( Not Optional Materialize - Loop ) var ( @@ -228,7 +227,6 @@ var ( "not", "optional", "materialize", - "loop", } ) From 741f9cceecd52c283ced81a5391de3fd177da323 Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Mon, 8 Sep 2014 17:33:45 -0700 Subject: [PATCH 14/15] Fixed not iterator test. --- graph/iterator/not_iterator_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graph/iterator/not_iterator_test.go b/graph/iterator/not_iterator_test.go index 3414ca0..ac63b7c 100644 --- a/graph/iterator/not_iterator_test.go +++ b/graph/iterator/not_iterator_test.go @@ -6,13 +6,13 @@ import ( ) func TestNotIteratorBasics(t *testing.T) { - allIt := newFixed() + allIt := NewFixed(Identity) allIt.Add(1) allIt.Add(2) allIt.Add(3) allIt.Add(4) - toComplementIt := newFixed() + toComplementIt := NewFixed(Identity) toComplementIt.Add(2) toComplementIt.Add(4) From 52be8fb21b6a911e1ed430943665705a33b418cc Mon Sep 17 00:00:00 2001 From: Matei Chiperi Date: Thu, 2 Oct 2014 16:34:17 -0700 Subject: [PATCH 15/15] Updated Except operator to the new operator interface. --- graph/iterator/not_iterator.go | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/graph/iterator/not_iterator.go b/graph/iterator/not_iterator.go index 08a2542..db394f8 100644 --- a/graph/iterator/not_iterator.go +++ b/graph/iterator/not_iterator.go @@ -1,9 +1,6 @@ package iterator import ( - "fmt" - "strings" - "github.com/google/cayley/graph" ) @@ -76,28 +73,6 @@ func (it *Not) ResultTree() *graph.ResultTree { return tree } -// DebugString prints information about the iterator. -func (it *Not) DebugString(indent int) string { - var tags string - for _, k := range it.tags.Tags() { - tags += fmt.Sprintf("%s;", k) - } - - spaces := strings.Repeat(" ", indent+2) - return fmt.Sprintf("%s(%s %d\n%stags:%v\n%sprimary_it:\n%s\n%sall_it:\n%s\n%s)", - strings.Repeat(" ", indent), - it.Type(), - it.UID(), - spaces, - it.tags.Tags(), - spaces, - it.primaryIt.DebugString(indent+4), - spaces, - it.allIt.DebugString(indent+4), - strings.Repeat(" ", indent), - ) -} - // Next advances the Not iterator. It returns whether there is another valid // new value. It fetches the next value of the all iterator which is not // contained by the primary iterator. @@ -172,3 +147,17 @@ func (it *Not) Stats() graph.IteratorStats { func (it *Not) Size() (int64, bool) { return it.Stats().Size, false } + +func (it *Not) Describe() graph.Description { + subIts := []graph.Description{ + it.primaryIt.Describe(), + it.allIt.Describe(), + } + + return graph.Description{ + UID: it.UID(), + Type: it.Type(), + Tags: it.tags.Tags(), + Iterators: subIts, + } +}