Added command line options to ignore duplicate quad in add or missing quad in delete

graph/bolt/quadstore.go

@@ -43,6 +43,7 @@ var (
 	}
 	hashSize         = sha1.Size
 	localFillPercent = 0.7
+
 )
 
 type Token struct {
@@ -208,6 +209,12 @@ func (qs *QuadStore) ApplyDeltas(deltas []graph.Delta) error {
 		for _, d := range deltas {
 			err := qs.buildQuadWrite(tx, d.Quad, d.ID.Int(), d.Action == graph.Add)
 			if err != nil {
+				if err == graph.ErrQuadExists && *graph.NoErrorDup{
+					continue
+				}
+				if err == graph.ErrQuadNotExist && *graph.NoErrorDel{
+					continue
+				}
 				return err
 			}
 			delta := int64(1)

graph/leveldb/quadstore.go

@@ -49,6 +49,7 @@ var (
 		New: func() interface{} { return sha1.New() },
 	}
 	hashSize = sha1.Size
+
 )
 
 type Token []byte
@@ -195,6 +196,12 @@ func (qs *QuadStore) ApplyDeltas(deltas []graph.Delta) error {
 		batch.Put(keyFor(d), bytes)
 		err = qs.buildQuadWrite(batch, d.Quad, d.ID.Int(), d.Action == graph.Add)
 		if err != nil {
+			if err == graph.ErrQuadExists && *graph.NoErrorDup{
+				continue
+			}
+			if err == graph.ErrQuadNotExist && *graph.NoErrorDel{
+				continue
+			}
 			return err
 		}
 		delta := int64(1)
@@ -243,6 +250,7 @@ func (qs *QuadStore) buildQuadWrite(batch *leveldb.Batch, q quad.Quad, id int64,
 	}
 	if err == nil {
 		// We got something.
+		fmt.Printf("Got something")
 		err = json.Unmarshal(data, &entry)
 		if err != nil {
 			return err
@@ -250,12 +258,17 @@ func (qs *QuadStore) buildQuadWrite(batch *leveldb.Batch, q quad.Quad, id int64,
 	} else {
 		entry.Quad = q
 	}
-	entry.History = append(entry.History, id)
 
-	if isAdd && len(entry.History)%2 == 0 {
-		glog.Error("Entry History is out of sync for", entry)
-		return errors.New("odd index history")
+	if isAdd && len(entry.History)%2 == 1 {
+		glog.Errorf("attempt to add existing quad %v: %#v", entry, q)
+		return graph.ErrQuadExists
 	}
+	if !isAdd && len(entry.History)%2 == 0 {
+		glog.Error("attempt to delete non-existent quad %v: %#c", entry, q)
+		return graph.ErrQuadNotExist
+	}
+
+	entry.History = append(entry.History, id)
 
 	bytes, err := json.Marshal(entry)
 	if err != nil {

graph/memstore/quadstore.go

@@ -155,7 +155,11 @@ func (qs *QuadStore) indexOf(t quad.Quad) (int64, bool) {
 
 func (qs *QuadStore) AddDelta(d graph.Delta) error {
 	if _, exists := qs.indexOf(d.Quad); exists {
-		return graph.ErrQuadExists
+		if *graph.NoErrorDup {
+			return nil
+		}else{
+			return graph.ErrQuadExists
+		}
 	}
 	qid := qs.nextQuadID
 	qs.log = append(qs.log, LogEntry{
@@ -194,7 +198,11 @@ func (qs *QuadStore) AddDelta(d graph.Delta) error {
 func (qs *QuadStore) RemoveDelta(d graph.Delta) error {
 	prevQuadID, exists := qs.indexOf(d.Quad)
 	if !exists {
-		return graph.ErrQuadNotExist
+		if *graph.NoErrorDel {
+			return nil
+		}else{
+			return graph.ErrQuadNotExist
+		}
 	}
 
 	quadID := qs.nextQuadID

graph/mongo/quadstore.go

@@ -226,11 +226,19 @@ func (qs *QuadStore) ApplyDeltas(in []graph.Delta) error {
 		switch d.Action {
 		case graph.Add:
 			if qs.checkValid(key) {
-				return graph.ErrQuadExists
+				if *graph.NoErrorDup {
+					continue
+				}else{
+					return graph.ErrQuadExists
+				}
 			}
 		case graph.Delete:
 			if !qs.checkValid(key) {
-				return graph.ErrQuadNotExist
+				if *graph.NoErrorDel {
+					continue
+				}else{
+					return graph.ErrQuadNotExist
+				}
 			}
 		}
 	}

graph/quadstore.go

@@ -23,6 +23,7 @@ package graph
 
 import (
 	"errors"
+	"flag"
 
 	"github.com/barakmich/glog"
 	"github.com/google/cayley/quad"
@@ -193,3 +194,8 @@ func QuadStores() []string {
 	}
 	return t
 }
+
+var (
+	NoErrorDup = flag.Bool("noerrdup", false, "Don't stop loading on duplicated key on add")
+	NoErrorDel = flag.Bool("noerrdel", false, "Don't stop loading on missing key on delete")
+)