Merge pull request #101 from barakmich/materializer_iterator

Add Materialize Iterator

cayley_test.go

@@ -348,7 +348,7 @@ func TestQueries(t *testing.T) {
 
 		// TODO(kortschak) Be more rigorous in this result validation.
 		if len(got) != len(test.expect) {
-			t.Errorf("Unexpected number of results, got:%d expect:%d.", len(got), len(test.expect))
+			t.Errorf("Unexpected number of results, got:%d expect:%d on %s.", len(got), len(test.expect), test.message)
 		}
 	}
 }

graph/iterator.go

@@ -153,6 +153,22 @@ func Next(it Iterator) (Value, bool) {
 	return nil, false
 }
 
+// Height is a convienence function to measure the height of an iterator tree.
+func Height(it Iterator, until Type) int {
+	if it.Type() == until {
+		return 1
+	}
+	subs := it.SubIterators()
+	maxDepth := 0
+	for _, sub := range subs {
+		h := Height(sub, until)
+		if h > maxDepth {
+			maxDepth = h
+		}
+	}
+	return maxDepth + 1
+}
+
 // FixedIterator wraps iterators that are modifiable by addition of fixed value sets.
 type FixedIterator interface {
 	Iterator
@@ -180,6 +196,7 @@ const (
 	Fixed
 	Not
 	Optional
+	Materialize
 )
 
 var (
@@ -200,6 +217,7 @@ var (
 		"fixed",
 		"not",
 		"optional",
+		"materialize",
 	}
 )
 

graph/iterator/and_iterator_optimize.go

@@ -70,6 +70,8 @@ func (it *And) Optimize() (graph.Iterator, bool) {
 	// now a permutation of itself, but the contents are unchanged.
 	its = optimizeOrder(its)
 
+	its = materializeIts(its)
+
 	// Okay! At this point we have an optimized order.
 
 	// The easiest thing to do at this point is merely to create a new And iterator
@@ -293,6 +295,21 @@ func hasOneUsefulIterator(its []graph.Iterator) graph.Iterator {
 	return nil
 }
 
+func materializeIts(its []graph.Iterator) []graph.Iterator {
+	var out []graph.Iterator
+	for _, it := range its {
+		stats := it.Stats()
+		if stats.Size*stats.NextCost < stats.ContainsCost {
+			if graph.Height(it, graph.Materialize) > 10 {
+				out = append(out, NewMaterialize(it))
+				continue
+			}
+		}
+		out = append(out, it)
+	}
+	return out
+}
+
 // and.Stats() lives here in and-iterator-optimize.go because it may
 // in the future return different statistics based on how it is optimized.
 // For now, however, it's pretty static.

graph/iterator/materialize_iterator.go

@@ -0,0 +1,261 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iterator
+
+// A simple iterator that, when first called Contains() or Next() upon, materializes the whole subiterator, stores it locally, and responds. Essentially a cache.
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/barakmich/glog"
+
+	"github.com/google/cayley/graph"
+)
+
+var abortMaterializeAt = 1000
+
+type result struct {
+	id   graph.Value
+	tags map[string]graph.Value
+}
+
+type Materialize struct {
+	uid         uint64
+	tags        graph.Tagger
+	containsMap map[graph.Value]int
+	values      [][]result
+	index       int
+	subindex    int
+	subIt       graph.Iterator
+	hasRun      bool
+	aborted     bool
+}
+
+func NewMaterialize(sub graph.Iterator) *Materialize {
+	return &Materialize{
+		uid:         NextUID(),
+		containsMap: make(map[graph.Value]int),
+		subIt:       sub,
+		index:       -1,
+	}
+}
+
+func (it *Materialize) UID() uint64 {
+	return it.uid
+}
+
+func (it *Materialize) Reset() {
+	it.subIt.Reset()
+	it.index = -1
+}
+
+func (it *Materialize) Close() {
+	it.subIt.Close()
+	it.containsMap = nil
+	it.values = nil
+	it.hasRun = false
+}
+
+func (it *Materialize) Tagger() *graph.Tagger {
+	return &it.tags
+}
+
+func (it *Materialize) TagResults(dst map[string]graph.Value) {
+	if !it.hasRun {
+		return
+	}
+	if it.aborted {
+		it.subIt.TagResults(dst)
+		return
+	}
+	if it.Result() == nil {
+		return
+	}
+	for _, tag := range it.tags.Tags() {
+		dst[tag] = it.Result()
+	}
+	for tag, value := range it.values[it.index][it.subindex].tags {
+		dst[tag] = value
+	}
+}
+
+func (it *Materialize) Clone() graph.Iterator {
+	out := NewMaterialize(it.subIt.Clone())
+	out.tags.CopyFrom(it)
+	if it.hasRun {
+		out.hasRun = true
+		out.aborted = it.aborted
+		out.values = it.values
+		out.containsMap = it.containsMap
+	}
+	return out
+}
+
+// Print some information about the iterator.
+func (it *Materialize) DebugString(indent int) string {
+	return fmt.Sprintf("%s(%s tags: %s Size: %d\n%s)",
+		strings.Repeat(" ", indent),
+		it.Type(),
+		it.tags.Tags(),
+		len(it.values),
+		it.subIt.DebugString(indent+4),
+	)
+}
+
+// Register this iterator as a Materialize iterator.
+func (it *Materialize) Type() graph.Type { return graph.Materialize }
+
+// DEPRECATED
+func (it *Materialize) ResultTree() *graph.ResultTree {
+	tree := graph.NewResultTree(it.Result())
+	tree.AddSubtree(it.subIt.ResultTree())
+	return tree
+}
+
+func (it *Materialize) Result() graph.Value {
+	if len(it.values) == 0 {
+		return nil
+	}
+	if it.index == -1 {
+		return nil
+	}
+	if it.index >= len(it.values) {
+		return nil
+	}
+	return it.values[it.index][it.subindex].id
+}
+
+func (it *Materialize) SubIterators() []graph.Iterator {
+	return []graph.Iterator{it.subIt}
+}
+
+func (it *Materialize) Optimize() (graph.Iterator, bool) {
+	newSub, changed := it.subIt.Optimize()
+	if changed {
+		it.subIt = newSub
+		if it.subIt.Type() == graph.Null {
+			return it.subIt, true
+		}
+	}
+	return it, false
+}
+
+// Size is the number of values stored, if we've got them all.
+// Otherwise, guess based on the size of the subiterator.
+func (it *Materialize) Size() (int64, bool) {
+	if it.hasRun {
+		return int64(len(it.values)), true
+	}
+	return it.subIt.Size()
+}
+
+// The entire point of Materialize is to amortize the cost by
+// putting it all up front.
+func (it *Materialize) Stats() graph.IteratorStats {
+	overhead := int64(2)
+	size, _ := it.Size()
+	subitStats := it.subIt.Stats()
+	return graph.IteratorStats{
+		ContainsCost: overhead * subitStats.NextCost,
+		NextCost:     overhead * subitStats.NextCost,
+		Size:         size,
+	}
+}
+
+func (it *Materialize) Next() (graph.Value, bool) {
+	graph.NextLogIn(it)
+	if !it.hasRun {
+		it.materializeSet()
+	}
+	if it.aborted {
+		return graph.Next(it.subIt)
+	}
+
+	it.index++
+	it.subindex = 0
+	if it.index >= len(it.values) {
+		return graph.NextLogOut(it, nil, false)
+	}
+	return graph.NextLogOut(it, it.Result(), true)
+}
+
+func (it *Materialize) Contains(v graph.Value) bool {
+	graph.ContainsLogIn(it, v)
+	if !it.hasRun {
+		it.materializeSet()
+	}
+	if it.aborted {
+		return it.subIt.Contains(v)
+	}
+	if i, ok := it.containsMap[v]; ok {
+		it.index = i
+		it.subindex = 0
+		return graph.ContainsLogOut(it, v, true)
+	}
+	return graph.ContainsLogOut(it, v, false)
+}
+
+func (it *Materialize) NextResult() bool {
+	if !it.hasRun {
+		it.materializeSet()
+	}
+	if it.aborted {
+		return it.subIt.NextResult()
+	}
+
+	it.subindex++
+	if it.subindex >= len(it.values[it.index]) {
+		// Don't go off the end of the world
+		it.subindex--
+		return false
+	}
+	return true
+}
+
+func (it *Materialize) materializeSet() {
+	i := 0
+	for {
+		val, ok := graph.Next(it.subIt)
+		if !ok {
+			break
+		}
+		i += 1
+		if i > abortMaterializeAt {
+			it.aborted = true
+			break
+		}
+		if _, ok := it.containsMap[val]; !ok {
+			it.containsMap[val] = len(it.values)
+			it.values = append(it.values, nil)
+		}
+		index := it.containsMap[val]
+		tags := make(map[string]graph.Value)
+		it.subIt.TagResults(tags)
+		it.values[index] = append(it.values[index], result{id: val, tags: tags})
+		for it.subIt.NextResult() == true {
+			tags := make(map[string]graph.Value)
+			it.subIt.TagResults(tags)
+			it.values[index] = append(it.values[index], result{id: val, tags: tags})
+		}
+	}
+	if it.aborted {
+		it.values = nil
+		it.containsMap = nil
+		it.subIt.Reset()
+	}
+	glog.Infof("Materialization List %d: %#v", it.values)
+	it.hasRun = true
+}

graph/triplestore.go

@@ -28,14 +28,17 @@ import (
 	"github.com/google/cayley/quad"
 )
 
-// Defines an opaque "triple store value" type. However the backend wishes to
-// implement it, a Value is merely a token to a triple or a node that the backing
-// store itself understands, and the base iterators pass around.
+// Value defines an opaque "triple store value" type. However the backend wishes
+// to implement it, a Value is merely a token to a triple or a node that the
+// backing store itself understands, and the base iterators pass around.
 //
 // For example, in a very traditional, graphd-style graph, these are int64s
 // (guids of the primitives). In a very direct sort of graph, these could be
 // pointers to structs, or merely triples, or whatever works best for the
 // backing store.
+//
+// These must be comparable, ie, not arrays or maps, as they may be used as keys
+// for maps.
 type Value interface{}
 
 type TripleStore interface {