Merge branch 'log_database' into b
Conflicts: graph/memstore/iterator.go graph/memstore/triplestore.go
This commit is contained in:
kortschak
commit
4a92ae9758
24 changed files with 863 additions and 531 deletions
|
|
@ -31,12 +31,12 @@ func init() {
|
|||
}, nil)
|
||||
}
|
||||
|
||||
type TripleDirectionIndex struct {
|
||||
type QuadDirectionIndex struct {
|
||||
index [4]map[int64]*b.Tree
|
||||
}
|
||||
|
||||
func NewTripleDirectionIndex() TripleDirectionIndex {
|
||||
return TripleDirectionIndex{[...]map[int64]*b.Tree{
|
||||
func NewQuadDirectionIndex() QuadDirectionIndex {
|
||||
return QuadDirectionIndex{[...]map[int64]*b.Tree{
|
||||
quad.Subject - 1: make(map[int64]*b.Tree),
|
||||
quad.Predicate - 1: make(map[int64]*b.Tree),
|
||||
quad.Object - 1: make(map[int64]*b.Tree),
|
||||
|
|
@ -44,34 +44,39 @@ func NewTripleDirectionIndex() TripleDirectionIndex {
|
|||
}}
|
||||
}
|
||||
|
||||
func (tdi TripleDirectionIndex) Tree(d quad.Direction, id int64) *b.Tree {
|
||||
func (qdi QuadDirectionIndex) Tree(d quad.Direction, id int64) *b.Tree {
|
||||
if d < quad.Subject || d > quad.Label {
|
||||
panic("illegal direction")
|
||||
}
|
||||
tree, ok := tdi.index[d-1][id]
|
||||
tree, ok := qdi.index[d-1][id]
|
||||
if !ok {
|
||||
tree = b.TreeNew(cmp)
|
||||
tdi.index[d-1][id] = tree
|
||||
qdi.index[d-1][id] = tree
|
||||
}
|
||||
return tree
|
||||
}
|
||||
|
||||
func (tdi TripleDirectionIndex) Get(d quad.Direction, id int64) (*b.Tree, bool) {
|
||||
func (qdi QuadDirectionIndex) Get(d quad.Direction, id int64) (*b.Tree, bool) {
|
||||
if d < quad.Subject || d > quad.Label {
|
||||
panic("illegal direction")
|
||||
}
|
||||
tree, ok := tdi.index[d-1][id]
|
||||
tree, ok := qdi.index[d-1][id]
|
||||
return tree, ok
|
||||
}
|
||||
|
||||
type LogEntry struct {
|
||||
graph.Delta
|
||||
DeletedBy int64
|
||||
}
|
||||
|
||||
type TripleStore struct {
|
||||
idCounter int64
|
||||
tripleIdCounter int64
|
||||
idMap map[string]int64
|
||||
revIdMap map[int64]string
|
||||
triples []quad.Quad
|
||||
size int64
|
||||
index TripleDirectionIndex
|
||||
idCounter int64
|
||||
quadIdCounter int64
|
||||
idMap map[string]int64
|
||||
revIdMap map[int64]string
|
||||
log []LogEntry
|
||||
size int64
|
||||
index QuadDirectionIndex
|
||||
// vip_index map[string]map[int64]map[string]map[int64]*b.Tree
|
||||
}
|
||||
|
||||
|
|
@ -80,20 +85,28 @@ func newTripleStore() *TripleStore {
|
|||
idMap: make(map[string]int64),
|
||||
revIdMap: make(map[int64]string),
|
||||
|
||||
// Sentinel null triple so triple indices start at 1
|
||||
triples: make([]quad.Quad, 1, 200),
|
||||
// Sentinel null entry so indices start at 1
|
||||
log: make([]LogEntry, 1, 200),
|
||||
|
||||
size: 1,
|
||||
index: NewTripleDirectionIndex(),
|
||||
idCounter: 1,
|
||||
tripleIdCounter: 1,
|
||||
index: NewQuadDirectionIndex(),
|
||||
idCounter: 1,
|
||||
quadIdCounter: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func (ts *TripleStore) AddTripleSet(triples []quad.Quad) {
|
||||
for _, t := range triples {
|
||||
ts.AddTriple(t)
|
||||
func (ts *TripleStore) ApplyDeltas(deltas []*graph.Delta) error {
|
||||
for _, d := range deltas {
|
||||
var err error
|
||||
if d.Action == graph.Add {
|
||||
err = ts.AddDelta(d)
|
||||
} else {
|
||||
err = ts.RemoveDelta(d)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const maxInt = int(^uint(0) >> 1)
|
||||
|
|
@ -120,29 +133,29 @@ func (ts *TripleStore) indexOf(t quad.Quad) (int64, bool) {
|
|||
min, tree = l, index
|
||||
}
|
||||
}
|
||||
it := NewIterator(tree, "")
|
||||
it := NewIterator(tree, "", ts)
|
||||
|
||||
for it.Next() {
|
||||
val := it.Result()
|
||||
if t == ts.triples[val.(int64)] {
|
||||
if t == ts.log[val.(int64)].Quad {
|
||||
return val.(int64), true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (ts *TripleStore) AddTriple(t quad.Quad) {
|
||||
if _, exists := ts.indexOf(t); exists {
|
||||
return
|
||||
func (ts *TripleStore) AddDelta(d *graph.Delta) error {
|
||||
if _, exists := ts.indexOf(d.Quad); exists {
|
||||
return graph.ErrQuadExists
|
||||
}
|
||||
ts.triples = append(ts.triples, t)
|
||||
tid := ts.tripleIdCounter
|
||||
qid := ts.quadIdCounter
|
||||
ts.log = append(ts.log, LogEntry{Delta: *d})
|
||||
ts.size++
|
||||
ts.tripleIdCounter++
|
||||
ts.quadIdCounter++
|
||||
|
||||
for d := quad.Subject; d <= quad.Label; d++ {
|
||||
sid := t.Get(d)
|
||||
if d == quad.Label && sid == "" {
|
||||
for dir := quad.Subject; dir <= quad.Label; dir++ {
|
||||
sid := d.Quad.Get(dir)
|
||||
if dir == quad.Label && sid == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := ts.idMap[sid]; !ok {
|
||||
|
|
@ -152,85 +165,60 @@ func (ts *TripleStore) AddTriple(t quad.Quad) {
|
|||
}
|
||||
}
|
||||
|
||||
for d := quad.Subject; d <= quad.Label; d++ {
|
||||
if d == quad.Label && t.Get(d) == "" {
|
||||
for dir := quad.Subject; dir <= quad.Label; dir++ {
|
||||
if dir == quad.Label && d.Quad.Get(dir) == "" {
|
||||
continue
|
||||
}
|
||||
id := ts.idMap[t.Get(d)]
|
||||
tree := ts.index.Tree(d, id)
|
||||
tree.Set(tid, struct{}{})
|
||||
id := ts.idMap[d.Quad.Get(dir)]
|
||||
tree := ts.index.Tree(dir, id)
|
||||
tree.Set(qid, struct{}{})
|
||||
}
|
||||
|
||||
// TODO(barakmich): Add VIP indexing
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ts *TripleStore) RemoveTriple(t quad.Quad) {
|
||||
tid, ok := ts.indexOf(t)
|
||||
if !ok {
|
||||
return
|
||||
func (ts *TripleStore) RemoveDelta(d *graph.Delta) error {
|
||||
prevQuadID, exists := ts.indexOf(d.Quad)
|
||||
if !exists {
|
||||
return graph.ErrQuadNotExist
|
||||
}
|
||||
|
||||
ts.triples[tid] = quad.Quad{}
|
||||
quadID := ts.quadIdCounter
|
||||
ts.log = append(ts.log, LogEntry{Delta: *d})
|
||||
ts.log[prevQuadID].DeletedBy = quadID
|
||||
ts.size--
|
||||
|
||||
for d := quad.Subject; d <= quad.Label; d++ {
|
||||
if d == quad.Label && t.Get(d) == "" {
|
||||
continue
|
||||
}
|
||||
id := ts.idMap[t.Get(d)]
|
||||
tree := ts.index.Tree(d, id)
|
||||
tree.Delete(tid)
|
||||
}
|
||||
|
||||
for d := quad.Subject; d <= quad.Label; d++ {
|
||||
if d == quad.Label && t.Get(d) == "" {
|
||||
continue
|
||||
}
|
||||
id, ok := ts.idMap[t.Get(d)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
stillExists := false
|
||||
for d := quad.Subject; d <= quad.Label; d++ {
|
||||
if d == quad.Label && t.Get(d) == "" {
|
||||
continue
|
||||
}
|
||||
nodeTree := ts.index.Tree(d, id)
|
||||
if nodeTree.Len() != 0 {
|
||||
stillExists = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !stillExists {
|
||||
delete(ts.idMap, t.Get(d))
|
||||
delete(ts.revIdMap, id)
|
||||
}
|
||||
}
|
||||
ts.quadIdCounter++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ts *TripleStore) Quad(index graph.Value) quad.Quad {
|
||||
return ts.triples[index.(int64)]
|
||||
return ts.log[index.(int64)].Quad
|
||||
}
|
||||
|
||||
func (ts *TripleStore) TripleIterator(d quad.Direction, value graph.Value) graph.Iterator {
|
||||
index, ok := ts.index.Get(d, value.(int64))
|
||||
data := fmt.Sprintf("dir:%s val:%d", d, value.(int64))
|
||||
if ok {
|
||||
return NewIterator(index, data)
|
||||
return NewIterator(index, data, ts)
|
||||
}
|
||||
return &iterator.Null{}
|
||||
}
|
||||
|
||||
func (ts *TripleStore) Horizon() int64 {
|
||||
return ts.log[len(ts.log)-1].ID
|
||||
}
|
||||
|
||||
func (ts *TripleStore) Size() int64 {
|
||||
return ts.size - 1 // Don't count the sentinel
|
||||
return ts.size
|
||||
}
|
||||
|
||||
func (ts *TripleStore) DebugPrint() {
|
||||
for i, t := range ts.triples {
|
||||
for i, l := range ts.log {
|
||||
if i == 0 {
|
||||
continue
|
||||
}
|
||||
glog.V(2).Infof("%d: %s", i, t)
|
||||
glog.V(2).Infof("%d: %#v", i, l)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -243,7 +231,7 @@ func (ts *TripleStore) NameOf(id graph.Value) string {
|
|||
}
|
||||
|
||||
func (ts *TripleStore) TriplesAllIterator() graph.Iterator {
|
||||
return iterator.NewInt64(0, ts.Size())
|
||||
return NewMemstoreQuadsAllIterator(ts)
|
||||
}
|
||||
|
||||
func (ts *TripleStore) FixedIterator() graph.FixedIterator {
|
||||
|
|
@ -256,7 +244,7 @@ func (ts *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.
|
|||
}
|
||||
|
||||
func (ts *TripleStore) NodesAllIterator() graph.Iterator {
|
||||
return NewMemstoreAllIterator(ts)
|
||||
return NewMemstoreNodesAllIterator(ts)
|
||||
}
|
||||
|
||||
func (ts *TripleStore) Close() {}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue