Add plumbing for bolt upgrades

cmd/cayleyupgrade/cayleyupgrade.go

@@ -17,19 +17,20 @@
 package main
 
 import (
-	"errors"
 	"flag"
+	"fmt"
+	"os"
 
+	"github.com/barakmich/glog"
 	"github.com/google/cayley/config"
-	_ "github.com/google/cayley/graph"
+	"github.com/google/cayley/graph"
 
 	// Load all supported backends.
-	"github.com/google/cayley/db"
+
 	_ "github.com/google/cayley/graph/bolt"
 	_ "github.com/google/cayley/graph/leveldb"
 	_ "github.com/google/cayley/graph/memstore"
 	_ "github.com/google/cayley/graph/mongo"
-	"github.com/google/cayley/internal"
 )
 
 var (
@@ -39,6 +40,23 @@ var (
 )
 
 func configFrom(file string) *config.Config {
+	// Find the file...
+	if file != "" {
+		if _, err := os.Stat(file); os.IsNotExist(err) {
+			glog.Fatalln("Cannot find specified configuration file", file, ", aborting.")
+		}
+	} else if _, err := os.Stat(os.Getenv("CAYLEY_CFG")); err == nil {
+		file = os.Getenv("CAYLEY_CFG")
+	} else if _, err := os.Stat("/etc/cayley.cfg"); err == nil {
+		file = "/etc/cayley.cfg"
+	}
+	if file == "" {
+		glog.Infoln("Couldn't find a config file in either $CAYLEY_CFG or /etc/cayley.cfg. Going by flag defaults only.")
+	}
+	cfg, err := config.Load(file)
+	if err != nil {
+		glog.Fatalln(err)
+	}
 	if cfg.DatabasePath == "" {
 		cfg.DatabasePath = *databasePath
 	}
@@ -53,23 +71,9 @@ func main() {
 	flag.Parse()
 	cfg := configFrom(*configFile)
 
-	r, registered := storeRegistry[*databaseBackend]
+	err := graph.UpgradeQuadStore(cfg.DatabaseType, cfg.DatabasePath, nil)
-	if registered {
-		return r.initFunc(dbpath, opts)
-	}
-	return errors.New("quadstore: name '" + name + "' is not registered")
 	if err != nil {
-		break
+		fmt.Fprintln(os.Stderr, err)
-	}
+		os.Exit(1)
-	if *quadFile != "" {
-		handle, err = db.Open(cfg)
-		if err != nil {
-			break
-		}
-		err = internal.Load(handle.QuadWriter, cfg, *quadFile, *quadType)
-		if err != nil {
-			break
-		}
-		handle.Close()
 	}
 }

graph/bolt/migrate.go

@@ -0,0 +1,84 @@
+// Copyright 2015 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bolt
+
+import (
+	"fmt"
+
+	"github.com/barakmich/glog"
+	"github.com/boltdb/bolt"
+	"github.com/google/cayley/graph"
+)
+
+const latestDataVersion = 1
+const nilDataVersion = 1
+
+type upgradeFunc func(*bolt.DB) error
+
+var migrateFunctions = []upgradeFunc{
+	nil,
+	upgrade1To2,
+}
+
+func upgradeBolt(path string, opts graph.Options) error {
+	db, err := bolt.Open(path, 0600, nil)
+	defer db.Close()
+	if err != nil {
+		glog.Errorln("Error, couldn't open! ", err)
+		return err
+	}
+	var version int64
+	err = db.View(func(tx *bolt.Tx) error {
+		version, err = getInt64ForMetaKey(tx, "version", nilDataVersion)
+		return err
+	})
+	if err != nil {
+		glog.Errorln("error:", err)
+		return err
+	}
+
+	if version == latestDataVersion {
+		fmt.Printf("Already at latest version: %d\n", latestDataVersion)
+		return nil
+	}
+
+	if version > latestDataVersion {
+		err := fmt.Errorf("Unknown data version: %d -- upgrade this tool", version)
+		glog.Errorln("error:", err)
+		return err
+	}
+
+	for i := version; i < latestDataVersion; i++ {
+		err := migrateFunctions[i](db)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func upgrade1To2(db *bolt.DB) error {
+	fmt.Println("Upgrading v1 to v2...")
+	tx, err := db.Begin(true)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+	return nil
+}

graph/bolt/quadstore.go

@@ -36,7 +36,7 @@ func init() {
 	graph.RegisterQuadStore(QuadStoreType, graph.QuadStoreRegistration{
 		NewFunc:           newQuadStore,
 		NewForRequestFunc: nil,
-		UpgradeFunc:       nil,
+		UpgradeFunc:       upgradeBolt,
 		InitFunc:          createNewBolt,
 		IsPersistent:      true,
 	})
@@ -73,6 +73,7 @@ type QuadStore struct {
 	open    bool
 	size    int64
 	horizon int64
+	version int64
 }
 
 func createNewBolt(path string, _ graph.Options) error {
@@ -88,6 +89,10 @@ func createNewBolt(path string, _ graph.Options) error {
 	if err != nil {
 		return err
 	}
+	err = qs.setVersion(latestDataVersion)
+	if err != nil {
+		return err
+	}
 	qs.Close()
 	return nil
 }
@@ -112,6 +117,9 @@ func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
 	} else if err != nil {
 		return nil, err
 	}
+	if qs.version != latestDataVersion {
+		return nil, errors.New("bolt: data version is out of date. Run cayleyupgrade for your config to update the data.")
+	}
 	return &qs, nil
 }
 
@@ -140,6 +148,24 @@ func (qs *QuadStore) createBuckets() error {
 	})
 }
 
+func (qs *QuadStore) setVersion(version int) error {
+	return qs.db.Update(func(tx *bolt.Tx) error {
+		buf := new(bytes.Buffer)
+		err := binary.Write(buf, binary.LittleEndian, version)
+		if err != nil {
+			glog.Errorf("Couldn't convert version!")
+			return err
+		}
+		b := tx.Bucket(metaBucket)
+		werr := b.Put([]byte("version"), buf.Bytes())
+		if werr != nil {
+			glog.Error("Couldn't write version!")
+			return werr
+		}
+		return nil
+	})
+}
+
 func (qs *QuadStore) Size() int64 {
 	return qs.size
 }
@@ -460,7 +486,7 @@ func (qs *QuadStore) SizeOf(k graph.Value) int64 {
 	return int64(qs.valueData(k.(*Token)).Size)
 }
 
-func (qs *QuadStore) getInt64ForKey(tx *bolt.Tx, key string, empty int64) (int64, error) {
+func getInt64ForMetaKey(tx *bolt.Tx, key string, empty int64) (int64, error) {
 	var out int64
 	b := tx.Bucket(metaBucket)
 	if b == nil {
@@ -481,11 +507,15 @@ func (qs *QuadStore) getInt64ForKey(tx *bolt.Tx, key string, empty int64) (int64
 func (qs *QuadStore) getMetadata() error {
 	err := qs.db.View(func(tx *bolt.Tx) error {
 		var err error
-		qs.size, err = qs.getInt64ForKey(tx, "size", 0)
+		qs.size, err = getInt64ForMetaKey(tx, "size", 0)
 		if err != nil {
 			return err
 		}
-		qs.horizon, err = qs.getInt64ForKey(tx, "horizon", 0)
+		qs.version, err = getInt64ForMetaKey(tx, "version", nilDataVersion)
+		if err != nil {
+			return err
+		}
+		qs.horizon, err = getInt64ForMetaKey(tx, "horizon", 0)
 		return err
 	})
 	return err

graph/quadstore.go

@@ -197,7 +197,11 @@ func NewQuadStoreForRequest(qs QuadStore, opts Options) (QuadStore, error) {
 func UpgradeQuadStore(name, dbpath string, opts Options) error {
 	r, registered := storeRegistry[name]
 	if registered {
-		return r.UpgradeFunc(dbpath, opts)
+		if r.UpgradeFunc != nil {
+			return r.UpgradeFunc(dbpath, opts)
+		} else {
+			return nil
+		}
 	}
 	return errors.New("quadstore: name '" + name + "' is not registered")