Reduce graph.Iterator method names
This commit is contained in:
parent
e4fb5d2bb8
commit
8576f66d20
21 changed files with 115 additions and 98 deletions
|
|
@ -35,12 +35,14 @@ type Iterator interface {
|
||||||
AddFixedTag(string, TSVal)
|
AddFixedTag(string, TSVal)
|
||||||
FixedTags() map[string]TSVal
|
FixedTags() map[string]TSVal
|
||||||
CopyTagsFrom(Iterator)
|
CopyTagsFrom(Iterator)
|
||||||
|
|
||||||
// Fills a tag-to-result-value map.
|
// Fills a tag-to-result-value map.
|
||||||
TagResults(*map[string]TSVal)
|
TagResults(*map[string]TSVal)
|
||||||
// Returns the current result.
|
// Returns the current result.
|
||||||
LastResult() TSVal
|
LastResult() TSVal
|
||||||
|
|
||||||
// DEPRECATED -- Fills a ResultTree struct with Result().
|
// DEPRECATED -- Fills a ResultTree struct with Result().
|
||||||
GetResultTree() *ResultTree
|
ResultTree() *ResultTree
|
||||||
|
|
||||||
// These methods are the heart and soul of the iterator, as they constitute
|
// These methods are the heart and soul of the iterator, as they constitute
|
||||||
// the iteration interface.
|
// the iteration interface.
|
||||||
|
|
@ -57,14 +59,23 @@ type Iterator interface {
|
||||||
// Next() advances the iterator and returns the next valid result. Returns
|
// Next() advances the iterator and returns the next valid result. Returns
|
||||||
// (<value>, true) or (nil, false)
|
// (<value>, true) or (nil, false)
|
||||||
Next() (TSVal, bool)
|
Next() (TSVal, bool)
|
||||||
|
|
||||||
// NextResult() advances iterators that may have more than one valid result,
|
// NextResult() advances iterators that may have more than one valid result,
|
||||||
// from the bottom up.
|
// from the bottom up.
|
||||||
NextResult() bool
|
NextResult() bool
|
||||||
|
|
||||||
|
// Return whether this iterator is reliably nextable. Most iterators are.
|
||||||
|
// However, some iterators, like "not" are, by definition, the whole database
|
||||||
|
// except themselves. Next() on these is unproductive, if impossible.
|
||||||
|
CanNext() bool
|
||||||
|
|
||||||
// Check(), given a value, returns whether or not that value is within the set
|
// Check(), given a value, returns whether or not that value is within the set
|
||||||
// held by this iterator.
|
// held by this iterator.
|
||||||
Check(TSVal) bool
|
Check(TSVal) bool
|
||||||
|
|
||||||
// Start iteration from the beginning
|
// Start iteration from the beginning
|
||||||
Reset()
|
Reset()
|
||||||
|
|
||||||
// Create a new iterator just like this one
|
// Create a new iterator just like this one
|
||||||
Clone() Iterator
|
Clone() Iterator
|
||||||
// These methods relate to choosing the right iterator, or optimizing an
|
// These methods relate to choosing the right iterator, or optimizing an
|
||||||
|
|
@ -74,31 +85,32 @@ type Iterator interface {
|
||||||
// this iterator, as well as the size. Roughly, it will take NextCost * Size
|
// this iterator, as well as the size. Roughly, it will take NextCost * Size
|
||||||
// "cost units" to get everything out of the iterator. This is a wibbly-wobbly
|
// "cost units" to get everything out of the iterator. This is a wibbly-wobbly
|
||||||
// thing, and not exact, but a useful heuristic.
|
// thing, and not exact, but a useful heuristic.
|
||||||
GetStats() *IteratorStats
|
|
||||||
|
Stats() IteratorStats
|
||||||
// Helpful accessor for the number of things in the iterator. The first return
|
// Helpful accessor for the number of things in the iterator. The first return
|
||||||
// value is the size, and the second return value is whether that number is exact,
|
// value is the size, and the second return value is whether that number is exact,
|
||||||
// or a conservative estimate.
|
// or a conservative estimate.
|
||||||
Size() (int64, bool)
|
Size() (int64, bool)
|
||||||
|
|
||||||
// Returns a string relating to what the function of the iterator is. By
|
// Returns a string relating to what the function of the iterator is. By
|
||||||
// knowing the names of the iterators, we can devise optimization strategies.
|
// knowing the names of the iterators, we can devise optimization strategies.
|
||||||
Type() string
|
Type() string
|
||||||
|
|
||||||
// Optimizes an iterator. Can replace the iterator, or merely move things
|
// Optimizes an iterator. Can replace the iterator, or merely move things
|
||||||
// around internally. if it chooses to replace it with a better iterator,
|
// around internally. if it chooses to replace it with a better iterator,
|
||||||
// returns (the new iterator, true), if not, it returns (self, false).
|
// returns (the new iterator, true), if not, it returns (self, false).
|
||||||
Optimize() (Iterator, bool)
|
Optimize() (Iterator, bool)
|
||||||
|
|
||||||
// Return a slice of the subiterators for this iterator.
|
// Return a slice of the subiterators for this iterator.
|
||||||
GetSubIterators() []Iterator
|
SubIterators() []Iterator
|
||||||
|
|
||||||
// Return a string representation of the iterator, indented by the given amount.
|
// Return a string representation of the iterator, indented by the given amount.
|
||||||
DebugString(int) string
|
DebugString(int) string
|
||||||
// Return whether this iterator is relaiably nextable. Most iterators are.
|
|
||||||
// However, some iterators, like "not" are, by definition, the whole database
|
|
||||||
// except themselves. Next() on these is unproductive, if impossible.
|
|
||||||
Nextable() bool
|
|
||||||
// Close the iterator and do internal cleanup.
|
// Close the iterator and do internal cleanup.
|
||||||
Close()
|
Close()
|
||||||
GetUid() int
|
|
||||||
|
UID() uintptr
|
||||||
}
|
}
|
||||||
|
|
||||||
type FixedIterator interface {
|
type FixedIterator interface {
|
||||||
|
|
@ -116,16 +128,16 @@ type IteratorStats struct {
|
||||||
// well as what they return. Highly useful for tracing the execution path of a query.
|
// well as what they return. Highly useful for tracing the execution path of a query.
|
||||||
func CheckLogIn(it Iterator, val TSVal) {
|
func CheckLogIn(it Iterator, val TSVal) {
|
||||||
if glog.V(4) {
|
if glog.V(4) {
|
||||||
glog.V(4).Infof("%s %d CHECK %d", strings.ToUpper(it.Type()), it.GetUid(), val)
|
glog.V(4).Infof("%s %d CHECK %d", strings.ToUpper(it.Type()), it.UID(), val)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func CheckLogOut(it Iterator, val TSVal, good bool) bool {
|
func CheckLogOut(it Iterator, val TSVal, good bool) bool {
|
||||||
if glog.V(4) {
|
if glog.V(4) {
|
||||||
if good {
|
if good {
|
||||||
glog.V(4).Infof("%s %d CHECK %d GOOD", strings.ToUpper(it.Type()), it.GetUid(), val)
|
glog.V(4).Infof("%s %d CHECK %d GOOD", strings.ToUpper(it.Type()), it.UID(), val)
|
||||||
} else {
|
} else {
|
||||||
glog.V(4).Infof("%s %d CHECK %d BAD", strings.ToUpper(it.Type()), it.GetUid(), val)
|
glog.V(4).Infof("%s %d CHECK %d BAD", strings.ToUpper(it.Type()), it.UID(), val)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return good
|
return good
|
||||||
|
|
@ -133,16 +145,16 @@ func CheckLogOut(it Iterator, val TSVal, good bool) bool {
|
||||||
|
|
||||||
func NextLogIn(it Iterator) {
|
func NextLogIn(it Iterator) {
|
||||||
if glog.V(4) {
|
if glog.V(4) {
|
||||||
glog.V(4).Infof("%s %d NEXT", strings.ToUpper(it.Type()), it.GetUid())
|
glog.V(4).Infof("%s %d NEXT", strings.ToUpper(it.Type()), it.UID())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func NextLogOut(it Iterator, val TSVal, ok bool) (TSVal, bool) {
|
func NextLogOut(it Iterator, val TSVal, ok bool) (TSVal, bool) {
|
||||||
if glog.V(4) {
|
if glog.V(4) {
|
||||||
if ok {
|
if ok {
|
||||||
glog.V(4).Infof("%s %d NEXT IS %d", strings.ToUpper(it.Type()), it.GetUid(), val)
|
glog.V(4).Infof("%s %d NEXT IS %d", strings.ToUpper(it.Type()), it.UID(), val)
|
||||||
} else {
|
} else {
|
||||||
glog.V(4).Infof("%s %d NEXT DONE", strings.ToUpper(it.Type()), it.GetUid())
|
glog.V(4).Infof("%s %d NEXT DONE", strings.ToUpper(it.Type()), it.UID())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return val, ok
|
return val, ok
|
||||||
|
|
|
||||||
|
|
@ -108,9 +108,9 @@ func (it *Int64) Optimize() (graph.Iterator, bool) { return it, false }
|
||||||
|
|
||||||
// Stats for an Int64 are simple. Super cheap to do any operation,
|
// Stats for an Int64 are simple. Super cheap to do any operation,
|
||||||
// and as big as the range.
|
// and as big as the range.
|
||||||
func (it *Int64) GetStats() *graph.IteratorStats {
|
func (it *Int64) Stats() graph.IteratorStats {
|
||||||
s, _ := it.Size()
|
s, _ := it.Size()
|
||||||
return &graph.IteratorStats{
|
return graph.IteratorStats{
|
||||||
CheckCost: 1,
|
CheckCost: 1,
|
||||||
NextCost: 1,
|
NextCost: 1,
|
||||||
Size: s,
|
Size: s,
|
||||||
|
|
|
||||||
|
|
@ -64,7 +64,7 @@ func (it *And) Clone() graph.Iterator {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns a slice of the subiterators, in order (primary iterator first).
|
// Returns a slice of the subiterators, in order (primary iterator first).
|
||||||
func (it *And) GetSubIterators() []graph.Iterator {
|
func (it *And) SubIterators() []graph.Iterator {
|
||||||
iters := make([]graph.Iterator, len(it.internalIterators)+1)
|
iters := make([]graph.Iterator, len(it.internalIterators)+1)
|
||||||
iters[0] = it.primaryIt
|
iters[0] = it.primaryIt
|
||||||
copy(iters[1:], it.internalIterators)
|
copy(iters[1:], it.internalIterators)
|
||||||
|
|
@ -84,11 +84,11 @@ func (it *And) TagResults(out *map[string]graph.TSVal) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// DEPRECATED Returns the ResultTree for this iterator, recurses to it's subiterators.
|
// DEPRECATED Returns the ResultTree for this iterator, recurses to it's subiterators.
|
||||||
func (it *And) GetResultTree() *graph.ResultTree {
|
func (it *And) ResultTree() *graph.ResultTree {
|
||||||
tree := graph.NewResultTree(it.LastResult())
|
tree := graph.NewResultTree(it.LastResult())
|
||||||
tree.AddSubtree(it.primaryIt.GetResultTree())
|
tree.AddSubtree(it.primaryIt.ResultTree())
|
||||||
for _, sub := range it.internalIterators {
|
for _, sub := range it.internalIterators {
|
||||||
tree.AddSubtree(sub.GetResultTree())
|
tree.AddSubtree(sub.ResultTree())
|
||||||
}
|
}
|
||||||
return tree
|
return tree
|
||||||
}
|
}
|
||||||
|
|
@ -109,13 +109,14 @@ func (it *And) DebugString(indent int) string {
|
||||||
return fmt.Sprintf("%s(%s %d\n%stags:%s\n%sprimary_it:\n%s\n%sother_its:\n%s)",
|
return fmt.Sprintf("%s(%s %d\n%stags:%s\n%sprimary_it:\n%s\n%sother_its:\n%s)",
|
||||||
strings.Repeat(" ", indent),
|
strings.Repeat(" ", indent),
|
||||||
it.Type(),
|
it.Type(),
|
||||||
it.GetUid(),
|
it.UID(),
|
||||||
spaces,
|
spaces,
|
||||||
tags,
|
tags,
|
||||||
spaces,
|
spaces,
|
||||||
it.primaryIt.DebugString(indent+4),
|
it.primaryIt.DebugString(indent+4),
|
||||||
spaces,
|
spaces,
|
||||||
total)
|
total,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add a subiterator to this And iterator.
|
// Add a subiterator to this And iterator.
|
||||||
|
|
|
||||||
|
|
@ -42,7 +42,7 @@ import (
|
||||||
func (it *And) Optimize() (graph.Iterator, bool) {
|
func (it *And) Optimize() (graph.Iterator, bool) {
|
||||||
// First, let's get the slice of iterators, in order (first one is Next()ed,
|
// First, let's get the slice of iterators, in order (first one is Next()ed,
|
||||||
// the rest are Check()ed)
|
// the rest are Check()ed)
|
||||||
old := it.GetSubIterators()
|
old := it.SubIterators()
|
||||||
|
|
||||||
// And call Optimize() on our subtree, replacing each one in the order we
|
// And call Optimize() on our subtree, replacing each one in the order we
|
||||||
// found them. it_list is the newly optimized versions of these, and changed
|
// found them. it_list is the newly optimized versions of these, and changed
|
||||||
|
|
@ -145,20 +145,20 @@ func optimizeOrder(its []graph.Iterator) []graph.Iterator {
|
||||||
// all of it's contents, and to Check() each of those against everyone
|
// all of it's contents, and to Check() each of those against everyone
|
||||||
// else.
|
// else.
|
||||||
for _, it := range its {
|
for _, it := range its {
|
||||||
if !it.Nextable() {
|
if !it.CanNext() {
|
||||||
bad = append(bad, it)
|
bad = append(bad, it)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
rootStats := it.GetStats()
|
rootStats := it.Stats()
|
||||||
cost := rootStats.NextCost
|
cost := rootStats.NextCost
|
||||||
for _, f := range its {
|
for _, f := range its {
|
||||||
if !f.Nextable() {
|
if !f.CanNext() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if f == it {
|
if f == it {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
stats := f.GetStats()
|
stats := f.Stats()
|
||||||
cost += stats.CheckCost
|
cost += stats.CheckCost
|
||||||
}
|
}
|
||||||
cost *= rootStats.Size
|
cost *= rootStats.Size
|
||||||
|
|
@ -177,7 +177,7 @@ func optimizeOrder(its []graph.Iterator) []graph.Iterator {
|
||||||
|
|
||||||
// ... push everyone else after...
|
// ... push everyone else after...
|
||||||
for _, it := range its {
|
for _, it := range its {
|
||||||
if !it.Nextable() {
|
if !it.CanNext() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if it != best {
|
if it != best {
|
||||||
|
|
@ -192,7 +192,7 @@ func optimizeOrder(its []graph.Iterator) []graph.Iterator {
|
||||||
type byCost []graph.Iterator
|
type byCost []graph.Iterator
|
||||||
|
|
||||||
func (c byCost) Len() int { return len(c) }
|
func (c byCost) Len() int { return len(c) }
|
||||||
func (c byCost) Less(i, j int) bool { return c[i].GetStats().CheckCost < c[j].GetStats().CheckCost }
|
func (c byCost) Less(i, j int) bool { return c[i].Stats().CheckCost < c[j].Stats().CheckCost }
|
||||||
func (c byCost) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
|
func (c byCost) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
|
||||||
|
|
||||||
// optimizeCheck(l) creates an alternate check list, containing the same contents
|
// optimizeCheck(l) creates an alternate check list, containing the same contents
|
||||||
|
|
@ -202,7 +202,7 @@ func (it *And) optimizeCheck() {
|
||||||
// TODO(kortschak) Reuse it.checkList if possible.
|
// TODO(kortschak) Reuse it.checkList if possible.
|
||||||
// This involves providing GetSubIterators with a slice to fill.
|
// This involves providing GetSubIterators with a slice to fill.
|
||||||
// Generally this is a worthwhile thing to do in other places as well.
|
// Generally this is a worthwhile thing to do in other places as well.
|
||||||
it.checkList = it.GetSubIterators()
|
it.checkList = it.SubIterators()
|
||||||
sort.Sort(byCost(it.checkList))
|
sort.Sort(byCost(it.checkList))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -212,7 +212,7 @@ func (it *And) optimizeCheck() {
|
||||||
// getSubTags() returns a map of the tags for all the subiterators.
|
// getSubTags() returns a map of the tags for all the subiterators.
|
||||||
func (it *And) getSubTags() map[string]struct{} {
|
func (it *And) getSubTags() map[string]struct{} {
|
||||||
tags := make(map[string]struct{})
|
tags := make(map[string]struct{})
|
||||||
for _, sub := range it.GetSubIterators() {
|
for _, sub := range it.SubIterators() {
|
||||||
for _, tag := range sub.Tags() {
|
for _, tag := range sub.Tags() {
|
||||||
tags[tag] = struct{}{}
|
tags[tag] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
@ -292,23 +292,23 @@ func hasOneUsefulIterator(its []graph.Iterator) graph.Iterator {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// and.GetStats() lives here in and-iterator-optimize.go because it may
|
// and.Stats() lives here in and-iterator-optimize.go because it may
|
||||||
// in the future return different statistics based on how it is optimized.
|
// in the future return different statistics based on how it is optimized.
|
||||||
// For now, however, it's pretty static.
|
// For now, however, it's pretty static.
|
||||||
func (it *And) GetStats() *graph.IteratorStats {
|
func (it *And) Stats() graph.IteratorStats {
|
||||||
primaryStats := it.primaryIt.GetStats()
|
primaryStats := it.primaryIt.Stats()
|
||||||
CheckCost := primaryStats.CheckCost
|
CheckCost := primaryStats.CheckCost
|
||||||
NextCost := primaryStats.NextCost
|
NextCost := primaryStats.NextCost
|
||||||
Size := primaryStats.Size
|
Size := primaryStats.Size
|
||||||
for _, sub := range it.internalIterators {
|
for _, sub := range it.internalIterators {
|
||||||
stats := sub.GetStats()
|
stats := sub.Stats()
|
||||||
NextCost += stats.CheckCost
|
NextCost += stats.CheckCost
|
||||||
CheckCost += stats.CheckCost
|
CheckCost += stats.CheckCost
|
||||||
if Size > stats.Size {
|
if Size > stats.Size {
|
||||||
Size = stats.Size
|
Size = stats.Size
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return &graph.IteratorStats{
|
return graph.IteratorStats{
|
||||||
CheckCost: CheckCost,
|
CheckCost: CheckCost,
|
||||||
NextCost: NextCost,
|
NextCost: NextCost,
|
||||||
Size: Size,
|
Size: Size,
|
||||||
|
|
|
||||||
|
|
@ -79,7 +79,7 @@ func TestReorderWithTag(t *testing.T) {
|
||||||
}
|
}
|
||||||
expectedTags := []string{"good", "slow"}
|
expectedTags := []string{"good", "slow"}
|
||||||
tagsOut := make([]string, 0)
|
tagsOut := make([]string, 0)
|
||||||
for _, sub := range newIt.GetSubIterators() {
|
for _, sub := range newIt.SubIterators() {
|
||||||
for _, x := range sub.Tags() {
|
for _, x := range sub.Tags() {
|
||||||
tagsOut = append(tagsOut, x)
|
tagsOut = append(tagsOut, x)
|
||||||
}
|
}
|
||||||
|
|
@ -98,12 +98,12 @@ func TestAndStatistics(t *testing.T) {
|
||||||
// Make all2 the default iterator
|
// Make all2 the default iterator
|
||||||
a.AddSubIterator(all2)
|
a.AddSubIterator(all2)
|
||||||
a.AddSubIterator(all)
|
a.AddSubIterator(all)
|
||||||
stats1 := a.GetStats()
|
stats1 := a.Stats()
|
||||||
newIt, changed := a.Optimize()
|
newIt, changed := a.Optimize()
|
||||||
if !changed {
|
if !changed {
|
||||||
t.Error("Didn't optimize")
|
t.Error("Didn't optimize")
|
||||||
}
|
}
|
||||||
stats2 := newIt.GetStats()
|
stats2 := newIt.Stats()
|
||||||
if stats2.NextCost > stats1.NextCost {
|
if stats2.NextCost > stats1.NextCost {
|
||||||
t.Error("And didn't optimize. Next cost old ", stats1.NextCost, "and new ", stats2.NextCost)
|
t.Error("And didn't optimize. Next cost old ", stats1.NextCost, "and new ", stats2.NextCost)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -148,8 +148,8 @@ func (it *Fixed) Size() (int64, bool) {
|
||||||
|
|
||||||
// As we right now have to scan the entire list, Next and Check are linear with the
|
// As we right now have to scan the entire list, Next and Check are linear with the
|
||||||
// size. However, a better data structure could remove these limits.
|
// size. However, a better data structure could remove these limits.
|
||||||
func (it *Fixed) GetStats() *graph.IteratorStats {
|
func (it *Fixed) Stats() graph.IteratorStats {
|
||||||
return &graph.IteratorStats{
|
return graph.IteratorStats{
|
||||||
CheckCost: int64(len(it.values)),
|
CheckCost: int64(len(it.values)),
|
||||||
NextCost: int64(len(it.values)),
|
NextCost: int64(len(it.values)),
|
||||||
Size: int64(len(it.values)),
|
Size: int64(len(it.values)),
|
||||||
|
|
|
||||||
|
|
@ -65,7 +65,7 @@ func NewHasA(ts graph.TripleStore, subIt graph.Iterator, d graph.Direction) *Has
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return our sole subiterator.
|
// Return our sole subiterator.
|
||||||
func (it *HasA) GetSubIterators() []graph.Iterator {
|
func (it *HasA) SubIterators() []graph.Iterator {
|
||||||
return []graph.Iterator{it.primaryIt}
|
return []graph.Iterator{it.primaryIt}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -105,9 +105,9 @@ func (it *HasA) TagResults(out *map[string]graph.TSVal) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// DEPRECATED Return results in a ResultTree.
|
// DEPRECATED Return results in a ResultTree.
|
||||||
func (it *HasA) GetResultTree() *graph.ResultTree {
|
func (it *HasA) ResultTree() *graph.ResultTree {
|
||||||
tree := graph.NewResultTree(it.LastResult())
|
tree := graph.NewResultTree(it.LastResult())
|
||||||
tree.AddSubtree(it.primaryIt.GetResultTree())
|
tree.AddSubtree(it.primaryIt.ResultTree())
|
||||||
return tree
|
return tree
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -117,7 +117,7 @@ func (it *HasA) DebugString(indent int) string {
|
||||||
for _, k := range it.Tags() {
|
for _, k := range it.Tags() {
|
||||||
tags += fmt.Sprintf("%s;", k)
|
tags += fmt.Sprintf("%s;", k)
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("%s(%s %d tags:%s direction:%s\n%s)", strings.Repeat(" ", indent), it.Type(), it.GetUid(), tags, it.dir, it.primaryIt.DebugString(indent+4))
|
return fmt.Sprintf("%s(%s %d tags:%s direction:%s\n%s)", strings.Repeat(" ", indent), it.Type(), it.UID(), tags, it.dir, it.primaryIt.DebugString(indent+4))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check a value against our internal iterator. In order to do this, we must first open a new
|
// Check a value against our internal iterator. In order to do this, we must first open a new
|
||||||
|
|
@ -196,15 +196,15 @@ func (it *HasA) Next() (graph.TSVal, bool) {
|
||||||
// one sticks -- potentially expensive, depending on fanout. Size, however, is
|
// one sticks -- potentially expensive, depending on fanout. Size, however, is
|
||||||
// potentially smaller. we know at worst it's the size of the subiterator, but
|
// potentially smaller. we know at worst it's the size of the subiterator, but
|
||||||
// if there are many repeated values, it could be much smaller in totality.
|
// if there are many repeated values, it could be much smaller in totality.
|
||||||
func (it *HasA) GetStats() *graph.IteratorStats {
|
func (it *HasA) Stats() graph.IteratorStats {
|
||||||
subitStats := it.primaryIt.GetStats()
|
subitStats := it.primaryIt.Stats()
|
||||||
// TODO(barakmich): These should really come from the triplestore itself
|
// TODO(barakmich): These should really come from the triplestore itself
|
||||||
// and be optimized.
|
// and be optimized.
|
||||||
faninFactor := int64(1)
|
faninFactor := int64(1)
|
||||||
fanoutFactor := int64(30)
|
fanoutFactor := int64(30)
|
||||||
nextConstant := int64(2)
|
nextConstant := int64(2)
|
||||||
tripleConstant := int64(1)
|
tripleConstant := int64(1)
|
||||||
return &graph.IteratorStats{
|
return graph.IteratorStats{
|
||||||
NextCost: tripleConstant + subitStats.NextCost,
|
NextCost: tripleConstant + subitStats.NextCost,
|
||||||
CheckCost: (fanoutFactor * nextConstant) * subitStats.CheckCost,
|
CheckCost: (fanoutFactor * nextConstant) * subitStats.CheckCost,
|
||||||
Size: faninFactor * subitStats.Size,
|
Size: faninFactor * subitStats.Size,
|
||||||
|
|
|
||||||
|
|
@ -20,13 +20,18 @@ package iterator
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
"github.com/barakmich/glog"
|
"github.com/barakmich/glog"
|
||||||
|
|
||||||
"github.com/google/cayley/graph"
|
"github.com/google/cayley/graph"
|
||||||
)
|
)
|
||||||
|
|
||||||
var iterator_n int = 0
|
var nextIteratorID uintptr
|
||||||
|
|
||||||
|
func nextID() uintptr {
|
||||||
|
return atomic.AddUintptr(&nextIteratorID, 1) - 1
|
||||||
|
}
|
||||||
|
|
||||||
// The Base iterator is the iterator other iterators inherit from to get some
|
// The Base iterator is the iterator other iterators inherit from to get some
|
||||||
// default functionality.
|
// default functionality.
|
||||||
|
|
@ -34,21 +39,20 @@ type Base struct {
|
||||||
Last graph.TSVal
|
Last graph.TSVal
|
||||||
tags []string
|
tags []string
|
||||||
fixedTags map[string]graph.TSVal
|
fixedTags map[string]graph.TSVal
|
||||||
nextable bool
|
canNext bool
|
||||||
uid int
|
uid uintptr
|
||||||
}
|
}
|
||||||
|
|
||||||
// Called by subclases.
|
// Called by subclases.
|
||||||
func BaseInit(it *Base) {
|
func BaseInit(it *Base) {
|
||||||
// Your basic iterator is nextable
|
// Your basic iterator is nextable
|
||||||
it.nextable = true
|
it.canNext = true
|
||||||
it.uid = iterator_n
|
|
||||||
if glog.V(2) {
|
if glog.V(2) {
|
||||||
iterator_n++
|
it.uid = nextID()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (it *Base) GetUid() int {
|
func (it *Base) UID() uintptr {
|
||||||
return it.uid
|
return it.uid
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -99,12 +103,12 @@ func (it *Base) Check(v graph.TSVal) bool {
|
||||||
|
|
||||||
// Base iterators should never appear in a tree if they are, select against
|
// Base iterators should never appear in a tree if they are, select against
|
||||||
// them.
|
// them.
|
||||||
func (it *Base) GetStats() *graph.IteratorStats {
|
func (it *Base) Stats() graph.IteratorStats {
|
||||||
return &graph.IteratorStats{100000, 100000, 100000}
|
return graph.IteratorStats{100000, 100000, 100000}
|
||||||
}
|
}
|
||||||
|
|
||||||
// DEPRECATED
|
// DEPRECATED
|
||||||
func (it *Base) GetResultTree() *graph.ResultTree {
|
func (it *Base) ResultTree() *graph.ResultTree {
|
||||||
tree := graph.NewResultTree(it.LastResult())
|
tree := graph.NewResultTree(it.LastResult())
|
||||||
return tree
|
return tree
|
||||||
}
|
}
|
||||||
|
|
@ -129,12 +133,12 @@ func (it *Base) Size() (int64, bool) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// No subiterators. Only those with subiterators need to do anything here.
|
// No subiterators. Only those with subiterators need to do anything here.
|
||||||
func (it *Base) GetSubIterators() []graph.Iterator {
|
func (it *Base) SubIterators() []graph.Iterator {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Accessor
|
// Accessor
|
||||||
func (it *Base) Nextable() bool { return it.nextable }
|
func (it *Base) CanNext() bool { return it.canNext }
|
||||||
|
|
||||||
// Fill the map based on the tags assigned to this iterator. Default
|
// Fill the map based on the tags assigned to this iterator. Default
|
||||||
// functionality works well for most iterators.
|
// functionality works well for most iterators.
|
||||||
|
|
@ -182,6 +186,6 @@ func (it *Null) DebugString(indent int) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
// A null iterator costs nothing. Use it!
|
// A null iterator costs nothing. Use it!
|
||||||
func (it *Null) GetStats() *graph.IteratorStats {
|
func (it *Null) Stats() graph.IteratorStats {
|
||||||
return &graph.IteratorStats{}
|
return graph.IteratorStats{}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -83,9 +83,9 @@ func (it *LinksTo) TagResults(out *map[string]graph.TSVal) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// DEPRECATED
|
// DEPRECATED
|
||||||
func (it *LinksTo) GetResultTree() *graph.ResultTree {
|
func (it *LinksTo) ResultTree() *graph.ResultTree {
|
||||||
tree := graph.NewResultTree(it.LastResult())
|
tree := graph.NewResultTree(it.LastResult())
|
||||||
tree.AddSubtree(it.primaryIt.GetResultTree())
|
tree.AddSubtree(it.primaryIt.ResultTree())
|
||||||
return tree
|
return tree
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -93,7 +93,7 @@ func (it *LinksTo) GetResultTree() *graph.ResultTree {
|
||||||
func (it *LinksTo) DebugString(indent int) string {
|
func (it *LinksTo) DebugString(indent int) string {
|
||||||
return fmt.Sprintf("%s(%s %d direction:%s\n%s)",
|
return fmt.Sprintf("%s(%s %d direction:%s\n%s)",
|
||||||
strings.Repeat(" ", indent),
|
strings.Repeat(" ", indent),
|
||||||
it.Type(), it.GetUid(), it.dir, it.primaryIt.DebugString(indent+4))
|
it.Type(), it.UID(), it.dir, it.primaryIt.DebugString(indent+4))
|
||||||
}
|
}
|
||||||
|
|
||||||
// If it checks in the right direction for the subiterator, it is a valid link
|
// If it checks in the right direction for the subiterator, it is a valid link
|
||||||
|
|
@ -109,7 +109,7 @@ func (it *LinksTo) Check(val graph.TSVal) bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return a list containing only our subiterator.
|
// Return a list containing only our subiterator.
|
||||||
func (it *LinksTo) GetSubIterators() []graph.Iterator {
|
func (it *LinksTo) SubIterators() []graph.Iterator {
|
||||||
return []graph.Iterator{it.primaryIt}
|
return []graph.Iterator{it.primaryIt}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -169,13 +169,13 @@ func (it *LinksTo) NextResult() bool {
|
||||||
func (it *LinksTo) Type() string { return "linksto" }
|
func (it *LinksTo) Type() string { return "linksto" }
|
||||||
|
|
||||||
// Return a guess as to how big or costly it is to next the iterator.
|
// Return a guess as to how big or costly it is to next the iterator.
|
||||||
func (it *LinksTo) GetStats() *graph.IteratorStats {
|
func (it *LinksTo) Stats() graph.IteratorStats {
|
||||||
subitStats := it.primaryIt.GetStats()
|
subitStats := it.primaryIt.Stats()
|
||||||
// TODO(barakmich): These should really come from the triplestore itself
|
// TODO(barakmich): These should really come from the triplestore itself
|
||||||
fanoutFactor := int64(20)
|
fanoutFactor := int64(20)
|
||||||
checkConstant := int64(1)
|
checkConstant := int64(1)
|
||||||
nextConstant := int64(2)
|
nextConstant := int64(2)
|
||||||
return &graph.IteratorStats{
|
return graph.IteratorStats{
|
||||||
NextCost: nextConstant + subitStats.NextCost,
|
NextCost: nextConstant + subitStats.NextCost,
|
||||||
CheckCost: checkConstant + subitStats.CheckCost,
|
CheckCost: checkConstant + subitStats.CheckCost,
|
||||||
Size: fanoutFactor * subitStats.Size,
|
Size: fanoutFactor * subitStats.Size,
|
||||||
|
|
|
||||||
|
|
@ -47,7 +47,7 @@ type Optional struct {
|
||||||
func NewOptional(it graph.Iterator) *Optional {
|
func NewOptional(it graph.Iterator) *Optional {
|
||||||
var o Optional
|
var o Optional
|
||||||
BaseInit(&o.Base)
|
BaseInit(&o.Base)
|
||||||
o.nextable = false
|
o.canNext = false
|
||||||
o.subIt = it
|
o.subIt = it
|
||||||
return &o
|
return &o
|
||||||
}
|
}
|
||||||
|
|
@ -127,9 +127,9 @@ func (it *Optional) Optimize() (graph.Iterator, bool) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// We're only as expensive as our subiterator. Except, we can't be nexted.
|
// We're only as expensive as our subiterator. Except, we can't be nexted.
|
||||||
func (it *Optional) GetStats() *graph.IteratorStats {
|
func (it *Optional) Stats() graph.IteratorStats {
|
||||||
subStats := it.subIt.GetStats()
|
subStats := it.subIt.Stats()
|
||||||
return &graph.IteratorStats{
|
return graph.IteratorStats{
|
||||||
CheckCost: subStats.CheckCost,
|
CheckCost: subStats.CheckCost,
|
||||||
NextCost: int64(1 << 62),
|
NextCost: int64(1 << 62),
|
||||||
Size: subStats.Size,
|
Size: subStats.Size,
|
||||||
|
|
|
||||||
|
|
@ -77,7 +77,7 @@ func (it *Or) Clone() graph.Iterator {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns a list.List of the subiterators, in order. The returned slice must not be modified.
|
// Returns a list.List of the subiterators, in order. The returned slice must not be modified.
|
||||||
func (it *Or) GetSubIterators() []graph.Iterator {
|
func (it *Or) SubIterators() []graph.Iterator {
|
||||||
return it.internalIterators
|
return it.internalIterators
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -89,10 +89,10 @@ func (it *Or) TagResults(out *map[string]graph.TSVal) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// DEPRECATED Returns the ResultTree for this graph.iterator, recurses to it's subiterators.
|
// DEPRECATED Returns the ResultTree for this graph.iterator, recurses to it's subiterators.
|
||||||
func (it *Or) GetResultTree() *graph.ResultTree {
|
func (it *Or) ResultTree() *graph.ResultTree {
|
||||||
tree := graph.NewResultTree(it.LastResult())
|
tree := graph.NewResultTree(it.LastResult())
|
||||||
for _, sub := range it.internalIterators {
|
for _, sub := range it.internalIterators {
|
||||||
tree.AddSubtree(sub.GetResultTree())
|
tree.AddSubtree(sub.ResultTree())
|
||||||
}
|
}
|
||||||
return tree
|
return tree
|
||||||
}
|
}
|
||||||
|
|
@ -233,7 +233,7 @@ func (it *Or) Close() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (it *Or) Optimize() (graph.Iterator, bool) {
|
func (it *Or) Optimize() (graph.Iterator, bool) {
|
||||||
old := it.GetSubIterators()
|
old := it.SubIterators()
|
||||||
optIts := optimizeSubIterators(old)
|
optIts := optimizeSubIterators(old)
|
||||||
// Close the replaced iterators (they ought to close themselves, but Close()
|
// Close the replaced iterators (they ought to close themselves, but Close()
|
||||||
// is idempotent, so this just protects against any machinations).
|
// is idempotent, so this just protects against any machinations).
|
||||||
|
|
@ -256,12 +256,12 @@ func (it *Or) Optimize() (graph.Iterator, bool) {
|
||||||
return newOr, true
|
return newOr, true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (it *Or) GetStats() *graph.IteratorStats {
|
func (it *Or) Stats() graph.IteratorStats {
|
||||||
CheckCost := int64(0)
|
CheckCost := int64(0)
|
||||||
NextCost := int64(0)
|
NextCost := int64(0)
|
||||||
Size := int64(0)
|
Size := int64(0)
|
||||||
for _, sub := range it.internalIterators {
|
for _, sub := range it.internalIterators {
|
||||||
stats := sub.GetStats()
|
stats := sub.Stats()
|
||||||
NextCost += stats.NextCost
|
NextCost += stats.NextCost
|
||||||
CheckCost += stats.CheckCost
|
CheckCost += stats.CheckCost
|
||||||
if it.isShortCircuiting {
|
if it.isShortCircuiting {
|
||||||
|
|
@ -272,7 +272,7 @@ func (it *Or) GetStats() *graph.IteratorStats {
|
||||||
Size += stats.Size
|
Size += stats.Size
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return &graph.IteratorStats{
|
return graph.IteratorStats{
|
||||||
CheckCost: CheckCost,
|
CheckCost: CheckCost,
|
||||||
NextCost: NextCost,
|
NextCost: NextCost,
|
||||||
Size: Size,
|
Size: Size,
|
||||||
|
|
|
||||||
|
|
@ -116,7 +116,7 @@ func (qs *queryShape) MakeNode(it graph.Iterator) *Node {
|
||||||
|
|
||||||
switch it.Type() {
|
switch it.Type() {
|
||||||
case "and":
|
case "and":
|
||||||
for _, sub := range it.GetSubIterators() {
|
for _, sub := range it.SubIterators() {
|
||||||
qs.nodeId++
|
qs.nodeId++
|
||||||
newNode := qs.MakeNode(sub)
|
newNode := qs.MakeNode(sub)
|
||||||
if sub.Type() != "or" {
|
if sub.Type() != "or" {
|
||||||
|
|
@ -143,7 +143,7 @@ func (qs *queryShape) MakeNode(it graph.Iterator) *Node {
|
||||||
qs.AddNode(newNode)
|
qs.AddNode(newNode)
|
||||||
qs.RemoveHasa()
|
qs.RemoveHasa()
|
||||||
case "or":
|
case "or":
|
||||||
for _, sub := range it.GetSubIterators() {
|
for _, sub := range it.SubIterators() {
|
||||||
qs.nodeId++
|
qs.nodeId++
|
||||||
newNode := qs.MakeNode(sub)
|
newNode := qs.MakeNode(sub)
|
||||||
if sub.Type() == "or" {
|
if sub.Type() == "or" {
|
||||||
|
|
|
||||||
|
|
@ -185,6 +185,6 @@ func (it *Comparison) Optimize() (graph.Iterator, bool) {
|
||||||
|
|
||||||
// We're only as expensive as our subiterator.
|
// We're only as expensive as our subiterator.
|
||||||
// Again, optimized value comparison iterators should do better.
|
// Again, optimized value comparison iterators should do better.
|
||||||
func (it *Comparison) GetStats() *graph.IteratorStats {
|
func (it *Comparison) Stats() graph.IteratorStats {
|
||||||
return it.subIt.GetStats()
|
return it.subIt.Stats()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -125,9 +125,9 @@ func (it *AllIterator) Optimize() (graph.Iterator, bool) {
|
||||||
return it, false
|
return it, false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (it *AllIterator) GetStats() *graph.IteratorStats {
|
func (it *AllIterator) Stats() graph.IteratorStats {
|
||||||
s, _ := it.Size()
|
s, _ := it.Size()
|
||||||
return &graph.IteratorStats{
|
return graph.IteratorStats{
|
||||||
CheckCost: 1,
|
CheckCost: 1,
|
||||||
NextCost: 2,
|
NextCost: 2,
|
||||||
Size: s,
|
Size: s,
|
||||||
|
|
|
||||||
|
|
@ -192,7 +192,7 @@ func (it *Iterator) Size() (int64, bool) {
|
||||||
|
|
||||||
func (it *Iterator) DebugString(indent int) string {
|
func (it *Iterator) DebugString(indent int) string {
|
||||||
size, _ := it.Size()
|
size, _ := it.Size()
|
||||||
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.GetUid(), it.Tags(), it.dir, size, it.ts.GetNameFor(it.checkId))
|
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.UID(), it.Tags(), it.dir, size, it.ts.GetNameFor(it.checkId))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (it *Iterator) Type() string { return "leveldb" }
|
func (it *Iterator) Type() string { return "leveldb" }
|
||||||
|
|
@ -202,9 +202,9 @@ func (it *Iterator) Optimize() (graph.Iterator, bool) {
|
||||||
return it, false
|
return it, false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (it *Iterator) GetStats() *graph.IteratorStats {
|
func (it *Iterator) Stats() graph.IteratorStats {
|
||||||
s, _ := it.Size()
|
s, _ := it.Size()
|
||||||
return &graph.IteratorStats{
|
return graph.IteratorStats{
|
||||||
CheckCost: 1,
|
CheckCost: 1,
|
||||||
NextCost: 2,
|
NextCost: 2,
|
||||||
Size: s,
|
Size: s,
|
||||||
|
|
|
||||||
|
|
@ -29,7 +29,7 @@ func (ts *TripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bool) {
|
func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bool) {
|
||||||
subs := it.GetSubIterators()
|
subs := it.SubIterators()
|
||||||
if len(subs) != 1 {
|
if len(subs) != 1 {
|
||||||
return it, false
|
return it, false
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -111,8 +111,8 @@ func (it *Iterator) Optimize() (graph.Iterator, bool) {
|
||||||
return it, false
|
return it, false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (it *Iterator) GetStats() *graph.IteratorStats {
|
func (it *Iterator) Stats() graph.IteratorStats {
|
||||||
return &graph.IteratorStats{
|
return graph.IteratorStats{
|
||||||
CheckCost: int64(math.Log(float64(it.tree.Len()))) + 1,
|
CheckCost: int64(math.Log(float64(it.tree.Len()))) + 1,
|
||||||
NextCost: 1,
|
NextCost: 1,
|
||||||
Size: int64(it.tree.Len()),
|
Size: int64(it.tree.Len()),
|
||||||
|
|
|
||||||
|
|
@ -29,7 +29,7 @@ func (ts *TripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bool) {
|
func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bool) {
|
||||||
subs := it.GetSubIterators()
|
subs := it.SubIterators()
|
||||||
if len(subs) != 1 {
|
if len(subs) != 1 {
|
||||||
return it, false
|
return it, false
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -171,9 +171,9 @@ func (it *Iterator) DebugString(indent int) string {
|
||||||
return fmt.Sprintf("%s(%s size:%d %s %s)", strings.Repeat(" ", indent), it.Type(), size, it.hash, it.name)
|
return fmt.Sprintf("%s(%s size:%d %s %s)", strings.Repeat(" ", indent), it.Type(), size, it.hash, it.name)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (it *Iterator) GetStats() *graph.IteratorStats {
|
func (it *Iterator) Stats() graph.IteratorStats {
|
||||||
size, _ := it.Size()
|
size, _ := it.Size()
|
||||||
return &graph.IteratorStats{
|
return graph.IteratorStats{
|
||||||
CheckCost: 1,
|
CheckCost: 1,
|
||||||
NextCost: 5,
|
NextCost: 5,
|
||||||
Size: size,
|
Size: size,
|
||||||
|
|
|
||||||
|
|
@ -29,7 +29,7 @@ func (ts *TripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bool) {
|
func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bool) {
|
||||||
subs := it.GetSubIterators()
|
subs := it.SubIterators()
|
||||||
if len(subs) != 1 {
|
if len(subs) != 1 {
|
||||||
return it, false
|
return it, false
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -48,11 +48,11 @@ func StringResultTreeEvaluator(it Iterator) string {
|
||||||
if !ok {
|
if !ok {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
out += it.GetResultTree().String()
|
out += it.ResultTree().String()
|
||||||
out += "\n"
|
out += "\n"
|
||||||
for it.NextResult() == true {
|
for it.NextResult() == true {
|
||||||
out += " "
|
out += " "
|
||||||
out += it.GetResultTree().String()
|
out += it.ResultTree().String()
|
||||||
out += "\n"
|
out += "\n"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue