Merge branch 'declassify' into tip
Conflicts: graph/iterator/hasa_iterator.go graph/iterator/linksto_iterator.go graph/iterator/query_shape_test.go graph/leveldb/all_iterator.go graph/leveldb/iterator.go graph/leveldb/leveldb_test.go graph/memstore/triplestore_test.go graph/mongo/iterator.go
This commit is contained in:
kortschak
commit
2dbbd17fe1
34 changed files with 823 additions and 457 deletions
|
|
@ -28,33 +28,47 @@ import (
|
|||
)
|
||||
|
||||
type AllIterator struct {
|
||||
iterator.Base
|
||||
uid uint64
|
||||
tags graph.Tagger
|
||||
prefix []byte
|
||||
dir quad.Direction
|
||||
open bool
|
||||
iter ldbit.Iterator
|
||||
ts *TripleStore
|
||||
ro *opt.ReadOptions
|
||||
result graph.Value
|
||||
}
|
||||
|
||||
func NewAllIterator(prefix string, d quad.Direction, ts *TripleStore) *AllIterator {
|
||||
var it AllIterator
|
||||
iterator.BaseInit(&it.Base)
|
||||
it.ro = &opt.ReadOptions{}
|
||||
it.ro.DontFillCache = true
|
||||
it.iter = ts.db.NewIterator(nil, it.ro)
|
||||
it.prefix = []byte(prefix)
|
||||
it.dir = d
|
||||
it.open = true
|
||||
it.ts = ts
|
||||
opts := &opt.ReadOptions{
|
||||
DontFillCache: true,
|
||||
}
|
||||
|
||||
it := AllIterator{
|
||||
uid: iterator.NextUID(),
|
||||
ro: opts,
|
||||
iter: ts.db.NewIterator(nil, opts),
|
||||
prefix: []byte(prefix),
|
||||
dir: d,
|
||||
open: true,
|
||||
ts: ts,
|
||||
}
|
||||
|
||||
it.iter.Seek(it.prefix)
|
||||
if !it.iter.Valid() {
|
||||
// FIXME(kortschak) What are the semantics here? Is this iterator usable?
|
||||
// If not, we should return nil *Iterator and an error.
|
||||
it.open = false
|
||||
it.iter.Release()
|
||||
}
|
||||
|
||||
return &it
|
||||
}
|
||||
|
||||
func (it *AllIterator) UID() uint64 {
|
||||
return it.uid
|
||||
}
|
||||
|
||||
func (it *AllIterator) Reset() {
|
||||
if !it.open {
|
||||
it.iter = it.ts.db.NewIterator(nil, it.ro)
|
||||
|
|
@ -67,15 +81,29 @@ func (it *AllIterator) Reset() {
|
|||
}
|
||||
}
|
||||
|
||||
func (it *AllIterator) Tagger() *graph.Tagger {
|
||||
return &it.tags
|
||||
}
|
||||
|
||||
func (it *AllIterator) TagResults(dst map[string]graph.Value) {
|
||||
for _, tag := range it.tags.Tags() {
|
||||
dst[tag] = it.Result()
|
||||
}
|
||||
|
||||
for tag, value := range it.tags.Fixed() {
|
||||
dst[tag] = value
|
||||
}
|
||||
}
|
||||
|
||||
func (it *AllIterator) Clone() graph.Iterator {
|
||||
out := NewAllIterator(string(it.prefix), it.dir, it.ts)
|
||||
out.CopyTagsFrom(it)
|
||||
out.tags.CopyFrom(it)
|
||||
return out
|
||||
}
|
||||
|
||||
func (it *AllIterator) Next() (graph.Value, bool) {
|
||||
if !it.open {
|
||||
it.Last = nil
|
||||
it.result = nil
|
||||
return nil, false
|
||||
}
|
||||
var out []byte
|
||||
|
|
@ -89,12 +117,29 @@ func (it *AllIterator) Next() (graph.Value, bool) {
|
|||
it.Close()
|
||||
return nil, false
|
||||
}
|
||||
it.Last = out
|
||||
it.result = out
|
||||
return out, true
|
||||
}
|
||||
|
||||
func (it *AllIterator) ResultTree() *graph.ResultTree {
|
||||
return graph.NewResultTree(it.Result())
|
||||
}
|
||||
|
||||
func (it *AllIterator) Result() graph.Value {
|
||||
return it.result
|
||||
}
|
||||
|
||||
func (it *AllIterator) NextResult() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// No subiterators.
|
||||
func (it *AllIterator) SubIterators() []graph.Iterator {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (it *AllIterator) Check(v graph.Value) bool {
|
||||
it.Last = v
|
||||
it.result = v
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
@ -116,7 +161,7 @@ func (it *AllIterator) Size() (int64, bool) {
|
|||
|
||||
func (it *AllIterator) DebugString(indent int) string {
|
||||
size, _ := it.Size()
|
||||
return fmt.Sprintf("%s(%s tags: %v leveldb size:%d %s %p)", strings.Repeat(" ", indent), it.Type(), it.Tags(), size, it.dir, it)
|
||||
return fmt.Sprintf("%s(%s tags: %v leveldb size:%d %s %p)", strings.Repeat(" ", indent), it.Type(), it.tags.Tags(), size, it.dir, it)
|
||||
}
|
||||
|
||||
func (it *AllIterator) Type() graph.Type { return graph.All }
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@ import (
|
|||
)
|
||||
|
||||
type Iterator struct {
|
||||
iterator.Base
|
||||
uid uint64
|
||||
tags graph.Tagger
|
||||
nextPrefix []byte
|
||||
checkId []byte
|
||||
dir quad.Direction
|
||||
|
|
@ -37,30 +38,46 @@ type Iterator struct {
|
|||
qs *TripleStore
|
||||
ro *opt.ReadOptions
|
||||
originalPrefix string
|
||||
result graph.Value
|
||||
}
|
||||
|
||||
func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator {
|
||||
var it Iterator
|
||||
iterator.BaseInit(&it.Base)
|
||||
it.checkId = value.([]byte)
|
||||
it.dir = d
|
||||
it.originalPrefix = prefix
|
||||
it.nextPrefix = make([]byte, 0, 2+qs.hasher.Size())
|
||||
it.nextPrefix = append(it.nextPrefix, []byte(prefix)...)
|
||||
it.nextPrefix = append(it.nextPrefix, []byte(it.checkId[1:])...)
|
||||
it.ro = &opt.ReadOptions{}
|
||||
it.ro.DontFillCache = true
|
||||
it.iter = qs.db.NewIterator(nil, it.ro)
|
||||
it.open = true
|
||||
it.qs = qs
|
||||
vb := value.([]byte)
|
||||
p := make([]byte, 0, 2+qs.hasher.Size())
|
||||
p = append(p, []byte(prefix)...)
|
||||
p = append(p, []byte(vb[1:])...)
|
||||
|
||||
opts := &opt.ReadOptions{
|
||||
DontFillCache: true,
|
||||
}
|
||||
|
||||
it := Iterator{
|
||||
uid: iterator.NextUID(),
|
||||
nextPrefix: p,
|
||||
checkId: vb,
|
||||
dir: d,
|
||||
originalPrefix: prefix,
|
||||
ro: opts,
|
||||
iter: qs.db.NewIterator(nil, opts),
|
||||
open: true,
|
||||
qs: qs,
|
||||
}
|
||||
|
||||
ok := it.iter.Seek(it.nextPrefix)
|
||||
if !ok {
|
||||
// FIXME(kortschak) What are the semantics here? Is this iterator usable?
|
||||
// If not, we should return nil *Iterator and an error.
|
||||
it.open = false
|
||||
it.iter.Release()
|
||||
}
|
||||
|
||||
return &it
|
||||
}
|
||||
|
||||
func (it *Iterator) UID() uint64 {
|
||||
return it.uid
|
||||
}
|
||||
|
||||
func (it *Iterator) Reset() {
|
||||
if !it.open {
|
||||
it.iter = it.qs.db.NewIterator(nil, it.ro)
|
||||
|
|
@ -73,9 +90,23 @@ func (it *Iterator) Reset() {
|
|||
}
|
||||
}
|
||||
|
||||
func (it *Iterator) Tagger() *graph.Tagger {
|
||||
return &it.tags
|
||||
}
|
||||
|
||||
func (it *Iterator) TagResults(dst map[string]graph.Value) {
|
||||
for _, tag := range it.tags.Tags() {
|
||||
dst[tag] = it.Result()
|
||||
}
|
||||
|
||||
for tag, value := range it.tags.Fixed() {
|
||||
dst[tag] = value
|
||||
}
|
||||
}
|
||||
|
||||
func (it *Iterator) Clone() graph.Iterator {
|
||||
out := NewIterator(it.originalPrefix, it.dir, it.checkId, it.qs)
|
||||
out.CopyTagsFrom(it)
|
||||
out.tags.CopyFrom(it)
|
||||
return out
|
||||
}
|
||||
|
||||
|
|
@ -88,22 +119,22 @@ func (it *Iterator) Close() {
|
|||
|
||||
func (it *Iterator) Next() (graph.Value, bool) {
|
||||
if it.iter == nil {
|
||||
it.Last = nil
|
||||
it.result = nil
|
||||
return nil, false
|
||||
}
|
||||
if !it.open {
|
||||
it.Last = nil
|
||||
it.result = nil
|
||||
return nil, false
|
||||
}
|
||||
if !it.iter.Valid() {
|
||||
it.Last = nil
|
||||
it.result = nil
|
||||
it.Close()
|
||||
return nil, false
|
||||
}
|
||||
if bytes.HasPrefix(it.iter.Key(), it.nextPrefix) {
|
||||
out := make([]byte, len(it.iter.Key()))
|
||||
copy(out, it.iter.Key())
|
||||
it.Last = out
|
||||
it.result = out
|
||||
ok := it.iter.Next()
|
||||
if !ok {
|
||||
it.Close()
|
||||
|
|
@ -111,10 +142,27 @@ func (it *Iterator) Next() (graph.Value, bool) {
|
|||
return out, true
|
||||
}
|
||||
it.Close()
|
||||
it.Last = nil
|
||||
it.result = nil
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (it *Iterator) ResultTree() *graph.ResultTree {
|
||||
return graph.NewResultTree(it.Result())
|
||||
}
|
||||
|
||||
func (it *Iterator) Result() graph.Value {
|
||||
return it.result
|
||||
}
|
||||
|
||||
func (it *Iterator) NextResult() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// No subiterators.
|
||||
func (it *Iterator) SubIterators() []graph.Iterator {
|
||||
return nil
|
||||
}
|
||||
|
||||
func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
|
||||
if bytes.Equal(prefix, []byte("sp")) {
|
||||
switch d {
|
||||
|
|
@ -193,7 +241,7 @@ func (it *Iterator) Size() (int64, bool) {
|
|||
|
||||
func (it *Iterator) DebugString(indent int) string {
|
||||
size, _ := it.Size()
|
||||
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.UID(), it.Tags(), it.dir, size, it.qs.NameOf(it.checkId))
|
||||
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.UID(), it.tags.Tags(), it.dir, size, it.qs.NameOf(it.checkId))
|
||||
}
|
||||
|
||||
var levelDBType graph.Type
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ func makeTripleSet() []*quad.Quad {
|
|||
func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []*quad.Quad {
|
||||
var res ordered
|
||||
for {
|
||||
val, ok := it.Next()
|
||||
val, ok := graph.Next(it)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
|
@ -86,7 +86,7 @@ func (o ordered) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
|
|||
func iteratedNames(qs graph.TripleStore, it graph.Iterator) []string {
|
||||
var res []string
|
||||
for {
|
||||
val, ok := it.Next()
|
||||
val, ok := graph.Next(it)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
|
@ -266,7 +266,7 @@ func TestIterator(t *testing.T) {
|
|||
it.Reset()
|
||||
|
||||
it = qs.TriplesAllIterator()
|
||||
edge, _ := it.Next()
|
||||
edge, _ := graph.Next(it)
|
||||
triple := qs.Quad(edge)
|
||||
set := makeTripleSet()
|
||||
var ok bool
|
||||
|
|
@ -416,7 +416,7 @@ func TestOptimize(t *testing.T) {
|
|||
// With an linksto-fixed pair
|
||||
fixed := qs.FixedIterator()
|
||||
fixed.Add(qs.ValueOf("F"))
|
||||
fixed.AddTag("internal")
|
||||
fixed.Tagger().Add("internal")
|
||||
lto := iterator.NewLinksTo(qs, fixed, quad.Object)
|
||||
|
||||
oldIt := lto.Clone()
|
||||
|
|
@ -434,10 +434,10 @@ func TestOptimize(t *testing.T) {
|
|||
t.Errorf("Optimized iteration does not match original")
|
||||
}
|
||||
|
||||
oldIt.Next()
|
||||
graph.Next(oldIt)
|
||||
oldResults := make(map[string]graph.Value)
|
||||
oldIt.TagResults(oldResults)
|
||||
newIt.Next()
|
||||
graph.Next(newIt)
|
||||
newResults := make(map[string]graph.Value)
|
||||
newIt.TagResults(newResults)
|
||||
if !reflect.DeepEqual(newResults, oldResults) {
|
||||
|
|
|
|||
|
|
@ -37,14 +37,15 @@ func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bo
|
|||
if primary.Type() == graph.Fixed {
|
||||
size, _ := primary.Size()
|
||||
if size == 1 {
|
||||
val, ok := primary.Next()
|
||||
val, ok := graph.Next(primary)
|
||||
if !ok {
|
||||
panic("Sizes lie")
|
||||
}
|
||||
newIt := ts.TripleIterator(it.Direction(), val)
|
||||
newIt.CopyTagsFrom(it)
|
||||
for _, tag := range primary.Tags() {
|
||||
newIt.AddFixedTag(tag, val)
|
||||
nt := newIt.Tagger()
|
||||
nt.CopyFrom(it)
|
||||
for _, tag := range primary.Tagger().Tags() {
|
||||
nt.AddFixed(tag, val)
|
||||
}
|
||||
it.Close()
|
||||
return newIt, true
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue