barak/cayley
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'master' into exceptop

Browse source
This commit is contained in:
Matei Chiperi 2014-10-02 16:08:51 -07:00
commit f86bddd50a
24 changed files with 262 additions and 169 deletions
Download patch file Download diff file Expand all files Collapse all files

11
graph/bolt/all_iterator.go
View file

@ -17,7 +17,6 @@ package bolt
import (
"bytes"
"fmt"
"strings"
"github.com/barakmich/glog"
"github.com/boltdb/bolt"
@ -179,9 +178,15 @@ func (it *AllIterator) Size() (int64, bool) {
return it.qs.size, true
}
func (it *AllIterator) DebugString(indent int) string {
func (it *AllIterator) Describe() graph.Description {
size, _ := it.Size()
return fmt.Sprintf("%s(%s tags: %v bolt size:%d %s %p)", strings.Repeat(" ", indent), it.Type(), it.tags.Tags(), size, it.dir, it)
return graph.Description{
UID: it.UID(),
Type: it.Type(),
Tags: it.tags.Tags(),
Size: size,
Direction: it.dir,
}
}
func (it *AllIterator) Type() graph.Type { return graph.All }

22
graph/bolt/iterator.go
View file

@ -19,7 +19,6 @@ import (
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/barakmich/glog"
"github.com/boltdb/bolt"
@ -272,7 +271,7 @@ func (it *Iterator) Contains(v graph.Value) bool {
return false
}
offset := PositionOf(val, it.dir, it.qs)
if bytes.HasPrefix(val.key[offset:], it.checkID) {
if len(val.key) != 0 && bytes.HasPrefix(val.key[offset:], it.checkID) {
// You may ask, why don't we check to see if it's a valid (not deleted) quad
// again?
//
@ -291,16 +290,15 @@ func (it *Iterator) Size() (int64, bool) {
return it.size, true
}
func (it *Iterator) DebugString(indent int) string {
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)",
strings.Repeat(" ", indent),
it.Type(),
it.UID(),
it.tags.Tags(),
it.dir,
it.size,
it.qs.NameOf(&Token{it.bucket, it.checkID}),
)
func (it *Iterator) Describe() graph.Description {
return graph.Description{
UID: it.UID(),
Name: it.qs.NameOf(&Token{it.bucket, it.checkID}),
Type: it.Type(),
Tags: it.tags.Tags(),
Size: it.size,
Direction: it.dir,
}
}
func (it *Iterator) Type() graph.Type { return boltType }

41
graph/iterator.go
View file

@ -17,10 +17,12 @@ package graph
// Define the general iterator interface.
import (
"fmt"
"strings"
"sync"
"github.com/barakmich/glog"
"github.com/google/cayley/quad"
)
type Tagger struct {
@ -130,8 +132,8 @@ type Iterator interface {
// Return a slice of the subiterators for this iterator.
SubIterators() []Iterator
// Return a string representation of the iterator, indented by the given amount.
DebugString(int) string
// Return a string representation of the iterator.
Describe() Description
// Close the iterator and do internal cleanup.
Close()
@ -140,6 +142,17 @@ type Iterator interface {
UID() uint64
}
type Description struct {
UID uint64 `json:",omitempty"`
Name string `json:",omitempty"`
Type Type `json:",omitempty"`
Tags []string `json:",omitempty"`
Size int64 `json:",omitempty"`
Direction quad.Direction `json:",omitempty"`
Iterator *Description `json:",omitempty"`
Iterators []Description `json:",omitempty"`
}
type Nexter interface {
// Next advances the iterator to the next value, which will then be available through
// the Result method. It returns false if no further advancement is possible.
@ -255,17 +268,35 @@ func (t Type) String() string {
return types[t]
}
func (t *Type) MarshalText() (text []byte, err error) {
if *t < 0 || int(*t) >= len(types) {
return nil, fmt.Errorf("graph: illegal iterator type: %d", *t)
}
return []byte(types[*t]), nil
}
func (t *Type) UnmarshalText(text []byte) error {
s := string(text)
for i, c := range types[1:] {
if c == s {
*t = Type(i + 1)
return nil
}
}
return fmt.Errorf("graph: unknown iterator label: %q", text)
}
type StatsContainer struct {
UID uint64
Type Type
IteratorStats
Kind string
UID uint64
SubIts []StatsContainer
}
func DumpStats(it Iterator) StatsContainer {
var out StatsContainer
out.IteratorStats = it.Stats()
out.Kind = it.Type().String()
out.Type = it.Type()
out.UID = it.UID()
for _, sub := range it.SubIterators() {
out.SubIts = append(out.SubIts, DumpStats(sub))

12
graph/iterator/all_iterator.go
View file

@ -23,9 +23,6 @@ package iterator
// the base iterators, and it helps just to see it here.
import (
"fmt"
"strings"
"github.com/google/cayley/graph"
)
@ -81,9 +78,12 @@ func (it *Int64) TagResults(dst map[string]graph.Value) {
}
}
// Prints the All iterator as just an "all".
func (it *Int64) DebugString(indent int) string {
return fmt.Sprintf("%s(%s tags: %v)", strings.Repeat(" ", indent), it.Type(), it.tags.Tags())
func (it *Int64) Describe() graph.Description {
return graph.Description{
UID: it.UID(),
Type: it.Type(),
Tags: it.tags.Tags(),
}
}
// Next() on an Int64 all iterator is a simple incrementing counter.

34
graph/iterator/and_iterator.go
View file

@ -16,9 +16,6 @@
package iterator
import (
"fmt"
"strings"
"github.com/google/cayley/graph"
)
@ -111,30 +108,19 @@ func (it *And) ResultTree() *graph.ResultTree {
return tree
}
// Prints information about this iterator.
func (it *And) DebugString(indent int) string {
var total string
func (it *And) Describe() graph.Description {
subIts := make([]graph.Description, len(it.internalIterators))
for i, sub := range it.internalIterators {
total += strings.Repeat(" ", indent+2)
total += fmt.Sprintf("%d:\n%s\n", i, sub.DebugString(indent+4))
subIts[i] = sub.Describe()
}
var tags string
for _, k := range it.tags.Tags() {
tags += fmt.Sprintf("%s;", k)
primary := it.primaryIt.Describe()
return graph.Description{
UID: it.UID(),
Type: it.Type(),
Tags: it.tags.Tags(),
Iterator: &primary,
Iterators: subIts,
}
spaces := strings.Repeat(" ", indent+2)
return fmt.Sprintf("%s(%s %d\n%stags:%s\n%sprimary_it:\n%s\n%sother_its:\n%s)",
strings.Repeat(" ", indent),
it.Type(),
it.UID(),
spaces,
tags,
spaces,
it.primaryIt.DebugString(indent+4),
spaces,
total,
)
}
// Add a subiterator to this And iterator.

27
graph/iterator/fixed_iterator.go
View file

@ -22,7 +22,7 @@ package iterator
import (
"fmt"
"strings"
"sort"
"github.com/google/cayley/graph"
)
@ -94,20 +94,23 @@ func (it *Fixed) Add(v graph.Value) {
it.values = append(it.values, v)
}
// Print some information about the iterator.
func (it *Fixed) DebugString(indent int) string {
value := ""
func (it *Fixed) Describe() graph.Description {
var value string
if len(it.values) > 0 {
value = fmt.Sprint(it.values[0])
}
return fmt.Sprintf("%s(%s %d tags: %s Size: %d id0: %s)",
strings.Repeat(" ", indent),
it.Type(),
it.UID(),
it.tags.Fixed(),
len(it.values),
value,
)
fixed := make([]string, 0, len(it.tags.Fixed()))
for k := range it.tags.Fixed() {
fixed = append(fixed, k)
}
sort.Strings(fixed)
return graph.Description{
UID: it.UID(),
Name: value,
Type: it.Type(),
Tags: fixed,
Size: int64(len(it.values)),
}
}
// Register this iterator as a Fixed iterator.

17
graph/iterator/hasa_iterator.go
View file

@ -34,9 +34,6 @@ package iterator
// Alternatively, can be seen as the dual of the LinksTo iterator.
import (
"fmt"
"strings"
"github.com/barakmich/glog"
"github.com/google/cayley/graph"
@ -130,13 +127,15 @@ func (it *HasA) ResultTree() *graph.ResultTree {
return tree
}
// Print some information about this iterator.
func (it *HasA) DebugString(indent int) string {
var tags string
for _, k := range it.tags.Tags() {
tags += fmt.Sprintf("%s;", k)
func (it *HasA) Describe() graph.Description {
primary := it.primaryIt.Describe()
return graph.Description{
UID: it.UID(),
Type: it.Type(),
Tags: it.tags.Tags(),
Direction: it.dir,
Iterator: &primary,
}
return fmt.Sprintf("%s(%s %d tags:%s direction:%s\n%s)", strings.Repeat(" ", indent), it.Type(), it.UID(), tags, it.dir, it.primaryIt.DebugString(indent+4))
}
// Check a value against our internal iterator. In order to do this, we must first open a new

9
graph/iterator/iterator.go
View file

@ -17,7 +17,6 @@ package iterator
// Define the general iterator interface.
import (
"strings"
"sync/atomic"
"github.com/google/cayley/graph"
@ -78,9 +77,11 @@ func (it *Null) Type() graph.Type { return graph.Null }
// Null has nothing it needs to do.
func (it *Null) Optimize() (graph.Iterator, bool) { return it, false }
// Print the null iterator.
func (it *Null) DebugString(indent int) string {
return strings.Repeat(" ", indent) + "(null)"
func (it *Null) Describe() graph.Description {
return graph.Description{
UID: it.UID(),
Type: it.Type(),
}
}
func (it *Null) Next() bool {

16
graph/iterator/linksto_iterator.go
View file

@ -30,9 +30,6 @@ package iterator
// Can be seen as the dual of the HasA iterator.
import (
"fmt"
"strings"
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
@ -108,11 +105,14 @@ func (it *LinksTo) ResultTree() *graph.ResultTree {
return tree
}
// Print the iterator.
func (it *LinksTo) DebugString(indent int) string {
return fmt.Sprintf("%s(%s %d direction:%s\n%s)",
strings.Repeat(" ", indent),
it.Type(), it.UID(), it.dir, it.primaryIt.DebugString(indent+4))
func (it *LinksTo) Describe() graph.Description {
primary := it.primaryIt.Describe()
return graph.Description{
UID: it.UID(),
Type: it.Type(),
Direction: it.dir,
Iterator: &primary,
}
}
// If it checks in the right direction for the subiterator, it is a valid link

21
graph/iterator/materialize_iterator.go
View file

@ -17,9 +17,6 @@ package iterator
// A simple iterator that, when first called Contains() or Next() upon, materializes the whole subiterator, stores it locally, and responds. Essentially a cache.
import (
"fmt"
"strings"
"github.com/barakmich/glog"
"github.com/google/cayley/graph"
@ -118,15 +115,15 @@ func (it *Materialize) Clone() graph.Iterator {
return out
}
// Print some information about the iterator.
func (it *Materialize) DebugString(indent int) string {
return fmt.Sprintf("%s(%s tags: %s Size: %d\n%s)",
strings.Repeat(" ", indent),
it.Type(),
it.tags.Tags(),
len(it.values),
it.subIt.DebugString(indent+4),
)
func (it *Materialize) Describe() graph.Description {
primary := it.subIt.Describe()
return graph.Description{
UID: it.UID(),
Type: it.Type(),
Tags: it.tags.Tags(),
Size: int64(len(it.values)),
Iterator: &primary,
}
}
// Register this iterator as a Materialize iterator.

18
graph/iterator/optional_iterator.go
View file

@ -27,9 +27,6 @@ package iterator
// -- all things in the graph. It matches everything (as does the regex "(a)?")
import (
"fmt"
"strings"
"github.com/google/cayley/graph"
)
@ -120,13 +117,14 @@ func (it *Optional) TagResults(dst map[string]graph.Value) {
// Registers the optional iterator.
func (it *Optional) Type() graph.Type { return graph.Optional }
// Prints the optional and it's subiterator.
func (it *Optional) DebugString(indent int) string {
return fmt.Sprintf("%s(%s tags:%s\n%s)",
strings.Repeat(" ", indent),
it.Type(),
it.tags.Tags(),
it.subIt.DebugString(indent+4))
func (it *Optional) Describe() graph.Description {
primary := it.subIt.Describe()
return graph.Description{
UID: it.UID(),
Type: it.Type(),
Tags: it.tags.Tags(),
Iterator: &primary,
}
}
// There's nothing to optimize for an optional. Optimize the subiterator and

28
graph/iterator/or_iterator.go
View file

@ -22,9 +22,6 @@ package iterator
// May return the same value twice -- once for each branch.
import (
"fmt"
"strings"
"github.com/google/cayley/graph"
)
@ -113,26 +110,17 @@ func (it *Or) ResultTree() *graph.ResultTree {
return tree
}
// Prints information about this graph.iterator.
func (it *Or) DebugString(indent int) string {
var total string
func (it *Or) Describe() graph.Description {
var subIts []graph.Description
for i, sub := range it.internalIterators {
total += strings.Repeat(" ", indent+2)
total += fmt.Sprintf("%d:\n%s\n", i, sub.DebugString(indent+4))
subIts[i] = sub.Describe()
}
var tags string
for _, k := range it.tags.Tags() {
tags += fmt.Sprintf("%s;", k)
return graph.Description{
UID: it.UID(),
Type: it.Type(),
Tags: it.tags.Tags(),
Iterators: subIts,
}
spaces := strings.Repeat(" ", indent+2)
return fmt.Sprintf("%s(%s\n%stags:%s\n%sits:\n%s)",
strings.Repeat(" ", indent),
it.Type(),
spaces,
tags,
spaces,
total)
}
// Add a subiterator to this Or graph.iterator. Order matters.

14
graph/iterator/value_comparison_iterator.go
View file

@ -27,10 +27,8 @@ package iterator
// In MQL terms, this is the [{"age>=": 21}] concept.
import (
"fmt"
"log"
"strconv"
"strings"
"github.com/google/cayley/graph"
)
@ -190,11 +188,13 @@ func (it *Comparison) TagResults(dst map[string]graph.Value) {
// Registers the value-comparison iterator.
func (it *Comparison) Type() graph.Type { return graph.Comparison }
// Prints the value-comparison and its subiterator.
func (it *Comparison) DebugString(indent int) string {
return fmt.Sprintf("%s(%s\n%s)",
strings.Repeat(" ", indent),
it.Type(), it.subIt.DebugString(indent+4))
func (it *Comparison) Describe() graph.Description {
primary := it.subIt.Describe()
return graph.Description{
UID: it.UID(),
Type: it.Type(),
Iterator: &primary,
}
}
// There's nothing to optimize, locally, for a value-comparison iterator.

45
graph/iterator_test.go Normal file
View file

@ -0,0 +1,45 @@
// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package graph
import (
"testing"
)
func TestTypeEncodingRoundtrip(t *testing.T) {
for i := Invalid; i <= Materialize; i++ {
text, err := i.MarshalText()
if err != nil {
t.Errorf("Unexpected error when marshaling %s: %v", i, err)
}
if string(text) != i.String() {
t.Errorf("Unexpected MarshalText result, got:%q expect:%q", i, text)
}
var m Type
err = m.UnmarshalText(text)
if i == Invalid {
if err == nil || err.Error() != `graph: unknown iterator label: "invalid"` {
t.Errorf("Unexpected error when unmarshaling %q: %v", text, err)
}
} else {
if err != nil {
t.Errorf("Unexpected error when unmarshaling %q: %v", text, err)
}
}
if m != i {
t.Errorf("Unexpected UnmarshalText result, got:Type(%d) expect:Type(%d)", m, i)
}
}
}

12
graph/leveldb/all_iterator.go
View file

@ -16,8 +16,6 @@ package leveldb
import (
"bytes"
"fmt"
"strings"
ldbit "github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
@ -159,9 +157,15 @@ func (it *AllIterator) Size() (int64, bool) {
return int64(^uint64(0) >> 1), false
}
func (it *AllIterator) DebugString(indent int) string {
func (it *AllIterator) Describe() graph.Description {
size, _ := it.Size()
return fmt.Sprintf("%s(%s tags: %v leveldb size:%d %s %p)", strings.Repeat(" ", indent), it.Type(), it.tags.Tags(), size, it.dir, it)
return graph.Description{
UID: it.UID(),
Type: it.Type(),
Tags: it.tags.Tags(),
Size: size,
Direction: it.dir,
}
}
func (it *AllIterator) Type() graph.Type { return graph.All }

21
graph/leveldb/iterator.go
View file

@ -17,8 +17,6 @@ package leveldb
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"github.com/barakmich/glog"
ldbit "github.com/syndtr/goleveldb/leveldb/iterator"
@ -250,17 +248,16 @@ func (it *Iterator) Size() (int64, bool) {
return it.qs.SizeOf(Token(it.checkID)), true
}
func (it *Iterator) DebugString(indent int) string {
func (it *Iterator) Describe() graph.Description {
size, _ := it.Size()
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)",
strings.Repeat(" ", indent),
it.Type(),
it.UID(),
it.tags.Tags(),
it.dir,
size,
it.qs.NameOf(Token(it.checkID)),
)
return graph.Description{
UID: it.UID(),
Name: it.qs.NameOf(Token(it.checkID)),
Type: it.Type(),
Tags: it.tags.Tags(),
Size: size,
Direction: it.dir,
}
}
var levelDBType graph.Type

12
graph/memstore/iterator.go
View file

@ -15,9 +15,7 @@
package memstore
import (
"fmt"
"math"
"strings"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
@ -159,9 +157,15 @@ func (it *Iterator) Contains(v graph.Value) bool {
return graph.ContainsLogOut(it, v, false)
}
func (it *Iterator) DebugString(indent int) string {
func (it *Iterator) Describe() graph.Description {
size, _ := it.Size()
return fmt.Sprintf("%s(%s tags:%s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.tags.Tags(), size, it.data)
return graph.Description{
UID: it.UID(),
Name: it.data,
Type: it.Type(),
Tags: it.tags.Tags(),
Size: size,
}
}
var memType graph.Type

7
graph/memstore/quadstore_test.go
View file

@ -165,8 +165,11 @@ func TestLinksToOptimization(t *testing.T) {
v := newIt.(*Iterator)
vClone := v.Clone()
if vClone.DebugString(0) != v.DebugString(0) {
t.Fatal("Wrong iterator. Got ", vClone.DebugString(0))
origDesc := v.Describe()
cloneDesc := vClone.Describe()
origDesc.UID, cloneDesc.UID = 0, 0 // We are more strict now, so fake UID equality.
if !reflect.DeepEqual(cloneDesc, origDesc) {
t.Fatalf("Unexpected iterator description.\ngot: %#v\nexpect: %#v", cloneDesc, origDesc)
}
vt := vClone.Tagger()
if len(vt.Tags()) < 1 || vt.Tags()[0] != "foo" {

10
graph/mongo/iterator.go
View file

@ -16,7 +16,6 @@ package mongo
import (
"fmt"
"strings"
"github.com/barakmich/glog"
"gopkg.in/mgo.v2"
@ -213,9 +212,14 @@ func (it *Iterator) Type() graph.Type {
func (it *Iterator) Sorted() bool { return true }
func (it *Iterator) Optimize() (graph.Iterator, bool) { return it, false }
func (it *Iterator) DebugString(indent int) string {
func (it *Iterator) Describe() graph.Description {
size, _ := it.Size()
return fmt.Sprintf("%s(%s size:%d %s %s)", strings.Repeat(" ", indent), it.Type(), size, it.hash, it.name)
return graph.Description{
UID: it.UID(),
Name: fmt.Sprintf("%s/%s", it.name, it.hash),
Type: it.Type(),
Size: size,
}
}
func (it *Iterator) Stats() graph.IteratorStats {