Merge branch 'log_database' into b
Conflicts: graph/memstore/iterator.go graph/memstore/triplestore.go
This commit is contained in:
kortschak
commit
4a92ae9758
24 changed files with 863 additions and 531 deletions
|
|
@ -24,25 +24,46 @@ type AllIterator struct {
|
|||
ts *TripleStore
|
||||
}
|
||||
|
||||
func NewMemstoreAllIterator(ts *TripleStore) *AllIterator {
|
||||
var out AllIterator
|
||||
type NodesAllIterator AllIterator
|
||||
type QuadsAllIterator AllIterator
|
||||
|
||||
func NewMemstoreNodesAllIterator(ts *TripleStore) *NodesAllIterator {
|
||||
var out NodesAllIterator
|
||||
out.Int64 = *iterator.NewInt64(1, ts.idCounter-1)
|
||||
out.ts = ts
|
||||
return &out
|
||||
}
|
||||
|
||||
// No subiterators.
|
||||
func (it *AllIterator) SubIterators() []graph.Iterator {
|
||||
func (nit *NodesAllIterator) SubIterators() []graph.Iterator {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (it *AllIterator) Next() bool {
|
||||
if !it.Int64.Next() {
|
||||
func (nit *NodesAllIterator) Next() bool {
|
||||
if !nit.Int64.Next() {
|
||||
return false
|
||||
}
|
||||
_, ok := it.ts.revIdMap[it.Int64.Result().(int64)]
|
||||
_, ok := nit.ts.revIdMap[nit.Int64.Result().(int64)]
|
||||
if !ok {
|
||||
return it.Next()
|
||||
return nit.Next()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func NewMemstoreQuadsAllIterator(ts *TripleStore) *QuadsAllIterator {
|
||||
var out QuadsAllIterator
|
||||
out.Int64 = *iterator.NewInt64(1, ts.quadIdCounter-1)
|
||||
out.ts = ts
|
||||
return &out
|
||||
}
|
||||
|
||||
func (qit *QuadsAllIterator) Next() bool {
|
||||
out := qit.Int64.Next()
|
||||
if out {
|
||||
i64 := qit.Int64.Result().(int64)
|
||||
if qit.ts.log[i64].DeletedBy != 0 || qit.ts.log[i64].Action == graph.Delete {
|
||||
return qit.Next()
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import (
|
|||
|
||||
type Iterator struct {
|
||||
uid uint64
|
||||
ts *TripleStore
|
||||
tags graph.Tagger
|
||||
tree *b.Tree
|
||||
iter *b.Enumerator
|
||||
|
|
@ -37,13 +38,14 @@ func cmp(a, b int64) int {
|
|||
return int(a - b)
|
||||
}
|
||||
|
||||
func NewIterator(tree *b.Tree, data string) *Iterator {
|
||||
func NewIterator(tree *b.Tree, data string, ts *TripleStore) *Iterator {
|
||||
iter, err := tree.SeekFirst()
|
||||
if err != nil {
|
||||
iter = nil
|
||||
}
|
||||
return &Iterator{
|
||||
uid: iterator.NextUID(),
|
||||
ts: ts,
|
||||
tree: tree,
|
||||
iter: iter,
|
||||
data: data,
|
||||
|
|
@ -94,6 +96,7 @@ func (it *Iterator) Clone() graph.Iterator {
|
|||
|
||||
m := &Iterator{
|
||||
uid: iterator.NextUID(),
|
||||
ts: it.ts,
|
||||
tree: it.tree,
|
||||
iter: iter,
|
||||
data: it.data,
|
||||
|
|
@ -103,11 +106,10 @@ func (it *Iterator) Clone() graph.Iterator {
|
|||
return m
|
||||
}
|
||||
|
||||
func (it *Iterator) Close() {
|
||||
if it.iter != nil {
|
||||
it.iter.Close()
|
||||
it.iter = nil
|
||||
}
|
||||
func (it *Iterator) Close() {}
|
||||
|
||||
func (it *Iterator) checkValid(index int64) bool {
|
||||
return it.ts.log[index].DeletedBy == 0
|
||||
}
|
||||
|
||||
func (it *Iterator) Next() bool {
|
||||
|
|
@ -120,8 +122,10 @@ func (it *Iterator) Next() bool {
|
|||
if err != nil {
|
||||
return graph.NextLogOut(it, nil, false)
|
||||
}
|
||||
if !it.checkValid(result) {
|
||||
return it.Next()
|
||||
}
|
||||
it.result = result
|
||||
|
||||
return graph.NextLogOut(it, it.result, true)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -31,12 +31,12 @@ func init() {
|
|||
}, nil)
|
||||
}
|
||||
|
||||
type TripleDirectionIndex struct {
|
||||
type QuadDirectionIndex struct {
|
||||
index [4]map[int64]*b.Tree
|
||||
}
|
||||
|
||||
func NewTripleDirectionIndex() TripleDirectionIndex {
|
||||
return TripleDirectionIndex{[...]map[int64]*b.Tree{
|
||||
func NewQuadDirectionIndex() QuadDirectionIndex {
|
||||
return QuadDirectionIndex{[...]map[int64]*b.Tree{
|
||||
quad.Subject - 1: make(map[int64]*b.Tree),
|
||||
quad.Predicate - 1: make(map[int64]*b.Tree),
|
||||
quad.Object - 1: make(map[int64]*b.Tree),
|
||||
|
|
@ -44,34 +44,39 @@ func NewTripleDirectionIndex() TripleDirectionIndex {
|
|||
}}
|
||||
}
|
||||
|
||||
func (tdi TripleDirectionIndex) Tree(d quad.Direction, id int64) *b.Tree {
|
||||
func (qdi QuadDirectionIndex) Tree(d quad.Direction, id int64) *b.Tree {
|
||||
if d < quad.Subject || d > quad.Label {
|
||||
panic("illegal direction")
|
||||
}
|
||||
tree, ok := tdi.index[d-1][id]
|
||||
tree, ok := qdi.index[d-1][id]
|
||||
if !ok {
|
||||
tree = b.TreeNew(cmp)
|
||||
tdi.index[d-1][id] = tree
|
||||
qdi.index[d-1][id] = tree
|
||||
}
|
||||
return tree
|
||||
}
|
||||
|
||||
func (tdi TripleDirectionIndex) Get(d quad.Direction, id int64) (*b.Tree, bool) {
|
||||
func (qdi QuadDirectionIndex) Get(d quad.Direction, id int64) (*b.Tree, bool) {
|
||||
if d < quad.Subject || d > quad.Label {
|
||||
panic("illegal direction")
|
||||
}
|
||||
tree, ok := tdi.index[d-1][id]
|
||||
tree, ok := qdi.index[d-1][id]
|
||||
return tree, ok
|
||||
}
|
||||
|
||||
type LogEntry struct {
|
||||
graph.Delta
|
||||
DeletedBy int64
|
||||
}
|
||||
|
||||
type TripleStore struct {
|
||||
idCounter int64
|
||||
tripleIdCounter int64
|
||||
idMap map[string]int64
|
||||
revIdMap map[int64]string
|
||||
triples []quad.Quad
|
||||
size int64
|
||||
index TripleDirectionIndex
|
||||
idCounter int64
|
||||
quadIdCounter int64
|
||||
idMap map[string]int64
|
||||
revIdMap map[int64]string
|
||||
log []LogEntry
|
||||
size int64
|
||||
index QuadDirectionIndex
|
||||
// vip_index map[string]map[int64]map[string]map[int64]*b.Tree
|
||||
}
|
||||
|
||||
|
|
@ -80,20 +85,28 @@ func newTripleStore() *TripleStore {
|
|||
idMap: make(map[string]int64),
|
||||
revIdMap: make(map[int64]string),
|
||||
|
||||
// Sentinel null triple so triple indices start at 1
|
||||
triples: make([]quad.Quad, 1, 200),
|
||||
// Sentinel null entry so indices start at 1
|
||||
log: make([]LogEntry, 1, 200),
|
||||
|
||||
size: 1,
|
||||
index: NewTripleDirectionIndex(),
|
||||
idCounter: 1,
|
||||
tripleIdCounter: 1,
|
||||
index: NewQuadDirectionIndex(),
|
||||
idCounter: 1,
|
||||
quadIdCounter: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func (ts *TripleStore) AddTripleSet(triples []quad.Quad) {
|
||||
for _, t := range triples {
|
||||
ts.AddTriple(t)
|
||||
func (ts *TripleStore) ApplyDeltas(deltas []*graph.Delta) error {
|
||||
for _, d := range deltas {
|
||||
var err error
|
||||
if d.Action == graph.Add {
|
||||
err = ts.AddDelta(d)
|
||||
} else {
|
||||
err = ts.RemoveDelta(d)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const maxInt = int(^uint(0) >> 1)
|
||||
|
|
@ -120,29 +133,29 @@ func (ts *TripleStore) indexOf(t quad.Quad) (int64, bool) {
|
|||
min, tree = l, index
|
||||
}
|
||||
}
|
||||
it := NewIterator(tree, "")
|
||||
it := NewIterator(tree, "", ts)
|
||||
|
||||
for it.Next() {
|
||||
val := it.Result()
|
||||
if t == ts.triples[val.(int64)] {
|
||||
if t == ts.log[val.(int64)].Quad {
|
||||
return val.(int64), true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (ts *TripleStore) AddTriple(t quad.Quad) {
|
||||
if _, exists := ts.indexOf(t); exists {
|
||||
return
|
||||
func (ts *TripleStore) AddDelta(d *graph.Delta) error {
|
||||
if _, exists := ts.indexOf(d.Quad); exists {
|
||||
return graph.ErrQuadExists
|
||||
}
|
||||
ts.triples = append(ts.triples, t)
|
||||
tid := ts.tripleIdCounter
|
||||
qid := ts.quadIdCounter
|
||||
ts.log = append(ts.log, LogEntry{Delta: *d})
|
||||
ts.size++
|
||||
ts.tripleIdCounter++
|
||||
ts.quadIdCounter++
|
||||
|
||||
for d := quad.Subject; d <= quad.Label; d++ {
|
||||
sid := t.Get(d)
|
||||
if d == quad.Label && sid == "" {
|
||||
for dir := quad.Subject; dir <= quad.Label; dir++ {
|
||||
sid := d.Quad.Get(dir)
|
||||
if dir == quad.Label && sid == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := ts.idMap[sid]; !ok {
|
||||
|
|
@ -152,85 +165,60 @@ func (ts *TripleStore) AddTriple(t quad.Quad) {
|
|||
}
|
||||
}
|
||||
|
||||
for d := quad.Subject; d <= quad.Label; d++ {
|
||||
if d == quad.Label && t.Get(d) == "" {
|
||||
for dir := quad.Subject; dir <= quad.Label; dir++ {
|
||||
if dir == quad.Label && d.Quad.Get(dir) == "" {
|
||||
continue
|
||||
}
|
||||
id := ts.idMap[t.Get(d)]
|
||||
tree := ts.index.Tree(d, id)
|
||||
tree.Set(tid, struct{}{})
|
||||
id := ts.idMap[d.Quad.Get(dir)]
|
||||
tree := ts.index.Tree(dir, id)
|
||||
tree.Set(qid, struct{}{})
|
||||
}
|
||||
|
||||
// TODO(barakmich): Add VIP indexing
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ts *TripleStore) RemoveTriple(t quad.Quad) {
|
||||
tid, ok := ts.indexOf(t)
|
||||
if !ok {
|
||||
return
|
||||
func (ts *TripleStore) RemoveDelta(d *graph.Delta) error {
|
||||
prevQuadID, exists := ts.indexOf(d.Quad)
|
||||
if !exists {
|
||||
return graph.ErrQuadNotExist
|
||||
}
|
||||
|
||||
ts.triples[tid] = quad.Quad{}
|
||||
quadID := ts.quadIdCounter
|
||||
ts.log = append(ts.log, LogEntry{Delta: *d})
|
||||
ts.log[prevQuadID].DeletedBy = quadID
|
||||
ts.size--
|
||||
|
||||
for d := quad.Subject; d <= quad.Label; d++ {
|
||||
if d == quad.Label && t.Get(d) == "" {
|
||||
continue
|
||||
}
|
||||
id := ts.idMap[t.Get(d)]
|
||||
tree := ts.index.Tree(d, id)
|
||||
tree.Delete(tid)
|
||||
}
|
||||
|
||||
for d := quad.Subject; d <= quad.Label; d++ {
|
||||
if d == quad.Label && t.Get(d) == "" {
|
||||
continue
|
||||
}
|
||||
id, ok := ts.idMap[t.Get(d)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
stillExists := false
|
||||
for d := quad.Subject; d <= quad.Label; d++ {
|
||||
if d == quad.Label && t.Get(d) == "" {
|
||||
continue
|
||||
}
|
||||
nodeTree := ts.index.Tree(d, id)
|
||||
if nodeTree.Len() != 0 {
|
||||
stillExists = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !stillExists {
|
||||
delete(ts.idMap, t.Get(d))
|
||||
delete(ts.revIdMap, id)
|
||||
}
|
||||
}
|
||||
ts.quadIdCounter++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ts *TripleStore) Quad(index graph.Value) quad.Quad {
|
||||
return ts.triples[index.(int64)]
|
||||
return ts.log[index.(int64)].Quad
|
||||
}
|
||||
|
||||
func (ts *TripleStore) TripleIterator(d quad.Direction, value graph.Value) graph.Iterator {
|
||||
index, ok := ts.index.Get(d, value.(int64))
|
||||
data := fmt.Sprintf("dir:%s val:%d", d, value.(int64))
|
||||
if ok {
|
||||
return NewIterator(index, data)
|
||||
return NewIterator(index, data, ts)
|
||||
}
|
||||
return &iterator.Null{}
|
||||
}
|
||||
|
||||
func (ts *TripleStore) Horizon() int64 {
|
||||
return ts.log[len(ts.log)-1].ID
|
||||
}
|
||||
|
||||
func (ts *TripleStore) Size() int64 {
|
||||
return ts.size - 1 // Don't count the sentinel
|
||||
return ts.size
|
||||
}
|
||||
|
||||
func (ts *TripleStore) DebugPrint() {
|
||||
for i, t := range ts.triples {
|
||||
for i, l := range ts.log {
|
||||
if i == 0 {
|
||||
continue
|
||||
}
|
||||
glog.V(2).Infof("%d: %s", i, t)
|
||||
glog.V(2).Infof("%d: %#v", i, l)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -243,7 +231,7 @@ func (ts *TripleStore) NameOf(id graph.Value) string {
|
|||
}
|
||||
|
||||
func (ts *TripleStore) TriplesAllIterator() graph.Iterator {
|
||||
return iterator.NewInt64(0, ts.Size())
|
||||
return NewMemstoreQuadsAllIterator(ts)
|
||||
}
|
||||
|
||||
func (ts *TripleStore) FixedIterator() graph.FixedIterator {
|
||||
|
|
@ -256,7 +244,7 @@ func (ts *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.
|
|||
}
|
||||
|
||||
func (ts *TripleStore) NodesAllIterator() graph.Iterator {
|
||||
return NewMemstoreAllIterator(ts)
|
||||
return NewMemstoreNodesAllIterator(ts)
|
||||
}
|
||||
|
||||
func (ts *TripleStore) Close() {}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import (
|
|||
"github.com/google/cayley/graph"
|
||||
"github.com/google/cayley/graph/iterator"
|
||||
"github.com/google/cayley/quad"
|
||||
"github.com/google/cayley/writer"
|
||||
)
|
||||
|
||||
// This is a simple test graph.
|
||||
|
|
@ -51,13 +52,14 @@ var simpleGraph = []quad.Quad{
|
|||
{"G", "status", "cool", "status_graph"},
|
||||
}
|
||||
|
||||
func makeTestStore(data []quad.Quad) (*TripleStore, []pair) {
|
||||
func makeTestStore(data []quad.Quad) (*TripleStore, graph.QuadWriter, []pair) {
|
||||
seen := make(map[string]struct{})
|
||||
ts := newTripleStore()
|
||||
var (
|
||||
val int64
|
||||
ind []pair
|
||||
)
|
||||
writer, _ := writer.NewSingleReplication(ts, nil)
|
||||
for _, t := range data {
|
||||
for _, qp := range []string{t.Subject, t.Predicate, t.Object, t.Label} {
|
||||
if _, ok := seen[qp]; !ok && qp != "" {
|
||||
|
|
@ -66,9 +68,10 @@ func makeTestStore(data []quad.Quad) (*TripleStore, []pair) {
|
|||
seen[qp] = struct{}{}
|
||||
}
|
||||
}
|
||||
ts.AddTriple(t)
|
||||
|
||||
writer.AddQuad(t)
|
||||
}
|
||||
return ts, ind
|
||||
return ts, writer, ind
|
||||
}
|
||||
|
||||
type pair struct {
|
||||
|
|
@ -77,7 +80,7 @@ type pair struct {
|
|||
}
|
||||
|
||||
func TestMemstore(t *testing.T) {
|
||||
ts, index := makeTestStore(simpleGraph)
|
||||
ts, _, index := makeTestStore(simpleGraph)
|
||||
if size := ts.Size(); size != int64(len(simpleGraph)) {
|
||||
t.Errorf("Triple store has unexpected size, got:%d expected %d", size, len(simpleGraph))
|
||||
}
|
||||
|
|
@ -95,7 +98,7 @@ func TestMemstore(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestIteratorsAndNextResultOrderA(t *testing.T) {
|
||||
ts, _ := makeTestStore(simpleGraph)
|
||||
ts, _, _ := makeTestStore(simpleGraph)
|
||||
|
||||
fixed := ts.FixedIterator()
|
||||
fixed.Add(ts.ValueOf("C"))
|
||||
|
|
@ -144,7 +147,7 @@ func TestIteratorsAndNextResultOrderA(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestLinksToOptimization(t *testing.T) {
|
||||
ts, _ := makeTestStore(simpleGraph)
|
||||
ts, _, _ := makeTestStore(simpleGraph)
|
||||
|
||||
fixed := ts.FixedIterator()
|
||||
fixed.Add(ts.ValueOf("cool"))
|
||||
|
|
@ -172,9 +175,9 @@ func TestLinksToOptimization(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestRemoveTriple(t *testing.T) {
|
||||
ts, _ := makeTestStore(simpleGraph)
|
||||
ts, w, _ := makeTestStore(simpleGraph)
|
||||
|
||||
ts.RemoveTriple(quad.Quad{"E", "follows", "F", ""})
|
||||
w.RemoveQuad(quad.Quad{"E", "follows", "F", ""})
|
||||
|
||||
fixed := ts.FixedIterator()
|
||||
fixed.Add(ts.ValueOf("E"))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue