barak/cayley
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #277 from barakmich/upgrade_db

Browse source 
Add migration tool and use protos in Bolt store
This commit is contained in:
Barak Michener 2015-10-05 17:22:05 -04:00
commit 0c5f1bc364
13 changed files with 1228 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

12
graph/bolt/bolt_test.go
View file

@ -104,7 +104,7 @@ func TestCreateDatabase(t *testing.T) {
qs, err := newQuadStore(tmpFile.Name(), nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb QuadStore.")
t.Error("Failed to create bolt QuadStore.")
}
if s := qs.Size(); s != 0 {
t.Errorf("Unexpected size, got:%d expected:0", s)
@ -167,7 +167,7 @@ func TestLoadDatabase(t *testing.T) {
ts2, didConvert := qs.(*QuadStore)
if !didConvert {
t.Errorf("Could not convert from generic to LevelDB QuadStore")
t.Errorf("Could not convert from generic to Bolt QuadStore")
}
//Test horizon
@ -219,7 +219,7 @@ func TestIterator(t *testing.T) {
qs, err := newQuadStore(tmpFile.Name(), nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb QuadStore.")
t.Error("Failed to create bolt QuadStore.")
}
w, _ := writer.NewSingleReplication(qs, nil)
@ -313,7 +313,7 @@ func TestSetIterator(t *testing.T) {
qs, err := newQuadStore(tmpFile.Name(), nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb QuadStore.")
t.Error("Failed to create bolt QuadStore.")
}
defer qs.Close()
@ -428,7 +428,7 @@ func TestOptimize(t *testing.T) {
}
qs, err := newQuadStore(tmpFile.Name(), nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb QuadStore.")
t.Error("Failed to create bolt QuadStore.")
}
w, _ := writer.NewSingleReplication(qs, nil)
@ -478,7 +478,7 @@ func TestDeletedFromIterator(t *testing.T) {
qs, err := newQuadStore(tmpFile.Name(), nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb QuadStore.")
t.Error("Failed to create bolt QuadStore.")
}
defer qs.Close()

6
graph/bolt/iterator.go
View file

@ -16,7 +16,6 @@ package bolt
import (
"bytes"
"encoding/json"
"errors"
"fmt"
@ -25,6 +24,7 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/graph/proto"
"github.com/google/cayley/quad"
)
@ -114,8 +114,8 @@ func (it *Iterator) Close() error {
}
func (it *Iterator) isLiveValue(val []byte) bool {
var entry IndexEntry
json.Unmarshal(val, &entry)
var entry proto.HistoryEntry
entry.Unmarshal(val)
return len(entry.History)%2 != 0
}

162
graph/bolt/migrate.go Normal file
View file

@ -0,0 +1,162 @@
// Copyright 2015 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bolt
import (
"encoding/json"
"fmt"
"github.com/barakmich/glog"
"github.com/boltdb/bolt"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/proto"
)
const latestDataVersion = 2
const nilDataVersion = 1
type upgradeFunc func(*bolt.DB) error
var migrateFunctions = []upgradeFunc{
nil,
upgrade1To2,
}
func upgradeBolt(path string, opts graph.Options) error {
db, err := bolt.Open(path, 0600, nil)
defer db.Close()
if err != nil {
glog.Errorln("Error, couldn't open! ", err)
return err
}
var version int64
err = db.View(func(tx *bolt.Tx) error {
version, err = getInt64ForMetaKey(tx, "version", nilDataVersion)
return err
})
if err != nil {
glog.Errorln("error:", err)
return err
}
if version == latestDataVersion {
fmt.Printf("Already at latest version: %d\n", latestDataVersion)
return nil
}
if version > latestDataVersion {
err := fmt.Errorf("Unknown data version: %d -- upgrade this tool", version)
glog.Errorln("error:", err)
return err
}
for i := version; i < latestDataVersion; i++ {
err := migrateFunctions[i](db)
if err != nil {
return err
}
setVersion(db, i+1)
}
return nil
}
type v1ValueData struct {
Name string
Size int64
}
type v1IndexEntry struct {
History []int64
}
func upgrade1To2(db *bolt.DB) error {
fmt.Println("Upgrading v1 to v2...")
tx, err := db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
fmt.Println("Upgrading bucket", string(logBucket))
lb := tx.Bucket(logBucket)
c := lb.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var delta graph.Delta
err := json.Unmarshal(v, &delta)
if err != nil {
return err
}
newd := deltaToProto(delta)
data, err := newd.Marshal()
if err != nil {
return err
}
lb.Put(k, data)
}
if err := tx.Commit(); err != nil {
return err
}
tx, err = db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
fmt.Println("Upgrading bucket", string(nodeBucket))
nb := tx.Bucket(nodeBucket)
c = nb.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var vd proto.NodeData
err := json.Unmarshal(v, &vd)
if err != nil {
return err
}
data, err := vd.Marshal()
if err != nil {
return err
}
nb.Put(k, data)
}
if err := tx.Commit(); err != nil {
return err
}
for _, bucket := range [4][]byte{spoBucket, ospBucket, posBucket, cpsBucket} {
tx, err = db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
fmt.Println("Upgrading bucket", string(bucket))
b := tx.Bucket(bucket)
cur := b.Cursor()
for k, v := cur.First(); k != nil; k, v = cur.Next() {
var h proto.HistoryEntry
err := json.Unmarshal(v, &h)
if err != nil {
return err
}
data, err := h.Marshal()
if err != nil {
return err
}
b.Put(k, data)
}
if err := tx.Commit(); err != nil {
return err
}
}
return nil
}

128
graph/bolt/quadstore.go
View file

@ -18,7 +18,6 @@ import (
"bytes"
"crypto/sha1"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"hash"
@ -29,11 +28,18 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/graph/proto"
"github.com/google/cayley/quad"
)
func init() {
graph.RegisterQuadStore("bolt", true, newQuadStore, createNewBolt, nil)
graph.RegisterQuadStore(QuadStoreType, graph.QuadStoreRegistration{
NewFunc: newQuadStore,
NewForRequestFunc: nil,
UpgradeFunc: upgradeBolt,
InitFunc: createNewBolt,
IsPersistent: true,
})
}
var (
@ -67,6 +73,7 @@ type QuadStore struct {
open bool
size int64
horizon int64
version int64
}
func createNewBolt(path string, _ graph.Options) error {
@ -82,6 +89,10 @@ func createNewBolt(path string, _ graph.Options) error {
if err != nil {
return err
}
err = setVersion(qs.db, latestDataVersion)
if err != nil {
return err
}
qs.Close()
return nil
}
@ -106,6 +117,9 @@ func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
} else if err != nil {
return nil, err
}
if qs.version != latestDataVersion {
return nil, errors.New("bolt: data version is out of date. Run cayleyupgrade for your config to update the data.")
}
return &qs, nil
}
@ -134,6 +148,24 @@ func (qs *QuadStore) createBuckets() error {
})
}
func setVersion(db *bolt.DB, version int64) error {
return db.Update(func(tx *bolt.Tx) error {
buf := new(bytes.Buffer)
err := binary.Write(buf, binary.LittleEndian, version)
if err != nil {
glog.Errorf("Couldn't convert version!")
return err
}
b := tx.Bucket(metaBucket)
werr := b.Put([]byte("version"), buf.Bytes())
if werr != nil {
glog.Error("Couldn't write version!")
return werr
}
return nil
})
}
func (qs *QuadStore) Size() int64 {
return qs.size
}
@ -175,10 +207,6 @@ func (qs *QuadStore) createValueKeyFor(s string) []byte {
return key
}
type IndexEntry struct {
History []int64
}
var (
// Short hand for direction permutations.
spo = [4]quad.Direction{quad.Subject, quad.Predicate, quad.Object, quad.Label}
@ -196,6 +224,20 @@ var (
metaBucket = []byte("meta")
)
func deltaToProto(delta graph.Delta) proto.LogDelta {
var newd proto.LogDelta
newd.ID = uint64(delta.ID.Int())
newd.Action = int32(delta.Action)
newd.Timestamp = delta.Timestamp.UnixNano()
newd.Quad = &proto.Quad{
Subject: delta.Quad.Subject,
Predicate: delta.Quad.Predicate,
Object: delta.Quad.Object,
Label: delta.Quad.Label,
}
return newd
}
func (qs *QuadStore) ApplyDeltas(deltas []graph.Delta, ignoreOpts graph.IgnoreOpts) error {
oldSize := qs.size
oldHorizon := qs.horizon
@ -208,7 +250,8 @@ func (qs *QuadStore) ApplyDeltas(deltas []graph.Delta, ignoreOpts graph.IgnoreOp
if d.Action != graph.Add && d.Action != graph.Delete {
return errors.New("bolt: invalid action")
}
bytes, err := json.Marshal(d)
p := deltaToProto(d)
bytes, err := p.Marshal()
if err != nil {
return err
}
@ -263,13 +306,13 @@ func (qs *QuadStore) ApplyDeltas(deltas []graph.Delta, ignoreOpts graph.IgnoreOp
}
func (qs *QuadStore) buildQuadWrite(tx *bolt.Tx, q quad.Quad, id int64, isAdd bool) error {
var entry IndexEntry
var entry proto.HistoryEntry
b := tx.Bucket(spoBucket)
b.FillPercent = localFillPercent
data := b.Get(qs.createKeyFor(spo, q))
if data != nil {
// We got something.
err := json.Unmarshal(data, &entry)
err := entry.Unmarshal(data)
if err != nil {
return err
}
@ -284,9 +327,9 @@ func (qs *QuadStore) buildQuadWrite(tx *bolt.Tx, q quad.Quad, id int64, isAdd bo
return graph.ErrQuadNotExist
}
entry.History = append(entry.History, id)
entry.History = append(entry.History, uint64(id))
jsonbytes, err := json.Marshal(entry)
bytes, err := entry.Marshal()
if err != nil {
glog.Errorf("Couldn't write to buffer for entry %#v: %s", entry, err)
return err
@ -297,7 +340,7 @@ func (qs *QuadStore) buildQuadWrite(tx *bolt.Tx, q quad.Quad, id int64, isAdd bo
}
b := tx.Bucket(bucketFor(index))
b.FillPercent = localFillPercent
err = b.Put(qs.createKeyFor(index, q), jsonbytes)
err = b.Put(qs.createKeyFor(index, q), bytes)
if err != nil {
return err
}
@ -305,13 +348,11 @@ func (qs *QuadStore) buildQuadWrite(tx *bolt.Tx, q quad.Quad, id int64, isAdd bo
return nil
}
type ValueData struct {
Name string
Size int64
}
func (qs *QuadStore) UpdateValueKeyBy(name string, amount int64, tx *bolt.Tx) error {
value := ValueData{name, amount}
value := proto.NodeData{
Name: name,
Size_: amount,
}
b := tx.Bucket(nodeBucket)
b.FillPercent = localFillPercent
key := qs.createValueKeyFor(name)
@ -319,21 +360,23 @@ func (qs *QuadStore) UpdateValueKeyBy(name string, amount int64, tx *bolt.Tx) er
if data != nil {
// Node exists in the database -- unmarshal and update.
err := json.Unmarshal(data, &value)
var oldvalue proto.NodeData
err := oldvalue.Unmarshal(data)
if err != nil {
glog.Errorf("Error: couldn't reconstruct value: %v", err)
return err
}
value.Size += amount
oldvalue.Size_ += amount
value = oldvalue
}
// Are we deleting something?
if value.Size <= 0 {
value.Size = 0
if value.Size_ <= 0 {
value.Size_ = 0
}
// Repackage and rewrite.
bytes, err := json.Marshal(&value)
bytes, err := value.Marshal()
if err != nil {
glog.Errorf("Couldn't write to buffer for value %s: %s", name, err)
return err
@ -381,7 +424,7 @@ func (qs *QuadStore) Close() {
}
func (qs *QuadStore) Quad(k graph.Value) quad.Quad {
var d graph.Delta
var d proto.LogDelta
tok := k.(*Token)
err := qs.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(tok.bucket)
@ -389,8 +432,8 @@ func (qs *QuadStore) Quad(k graph.Value) quad.Quad {
if data == nil {
return nil
}
var in IndexEntry
err := json.Unmarshal(data, &in)
var in proto.HistoryEntry
err := in.Unmarshal(data)
if err != nil {
return err
}
@ -398,18 +441,23 @@ func (qs *QuadStore) Quad(k graph.Value) quad.Quad {
return nil
}
b = tx.Bucket(logBucket)
data = b.Get(qs.createDeltaKeyFor(in.History[len(in.History)-1]))
data = b.Get(qs.createDeltaKeyFor(int64(in.History[len(in.History)-1])))
if data == nil {
// No harm, no foul.
return nil
}
return json.Unmarshal(data, &d)
return d.Unmarshal(data)
})
if err != nil {
glog.Error("Error getting quad: ", err)
return quad.Quad{}
}
return d.Quad
return quad.Quad{
d.Quad.Subject,
d.Quad.Predicate,
d.Quad.Object,
d.Quad.Label,
}
}
func (qs *QuadStore) ValueOf(s string) graph.Value {
@ -419,8 +467,8 @@ func (qs *QuadStore) ValueOf(s string) graph.Value {
}
}
func (qs *QuadStore) valueData(t *Token) ValueData {
var out ValueData
func (qs *QuadStore) valueData(t *Token) proto.NodeData {
var out proto.NodeData
if glog.V(3) {
glog.V(3).Infof("%s %v", string(t.bucket), t.key)
}
@ -428,13 +476,13 @@ func (qs *QuadStore) valueData(t *Token) ValueData {
b := tx.Bucket(t.bucket)
data := b.Get(t.key)
if data != nil {
return json.Unmarshal(data, &out)
return out.Unmarshal(data)
}
return nil
})
if err != nil {
glog.Errorln("Error: couldn't get value")
return ValueData{}
return proto.NodeData{}
}
return out
}
@ -449,12 +497,12 @@ func (qs *QuadStore) NameOf(k graph.Value) string {
func (qs *QuadStore) SizeOf(k graph.Value) int64 {
if k == nil {
return 0
return -1
}
return int64(qs.valueData(k.(*Token)).Size)
return int64(qs.valueData(k.(*Token)).Size_)
}
func (qs *QuadStore) getInt64ForKey(tx *bolt.Tx, key string, empty int64) (int64, error) {
func getInt64ForMetaKey(tx *bolt.Tx, key string, empty int64) (int64, error) {
var out int64
b := tx.Bucket(metaBucket)
if b == nil {
@ -475,11 +523,15 @@ func (qs *QuadStore) getInt64ForKey(tx *bolt.Tx, key string, empty int64) (int64
func (qs *QuadStore) getMetadata() error {
err := qs.db.View(func(tx *bolt.Tx) error {
var err error
qs.size, err = qs.getInt64ForKey(tx, "size", 0)
qs.size, err = getInt64ForMetaKey(tx, "size", 0)
if err != nil {
return err
}
qs.horizon, err = qs.getInt64ForKey(tx, "horizon", 0)
qs.version, err = getInt64ForMetaKey(tx, "version", nilDataVersion)
if err != nil {
return err
}
qs.horizon, err = getInt64ForMetaKey(tx, "horizon", 0)
return err
})
return err