diff --git a/state/upgrades.go b/state/upgrades.go
index a7c0cf5..aa3216a 100644
--- a/state/upgrades.go
+++ b/state/upgrades.go
@@ -715,6 +715,8 @@ func AddStatusHistoryPruneSettings(st *State) error {
return nil
}
+var splitBatchRemoveSize = 1000
+
// SplitLogCollections moves log entries from the old single log collection
// to the log collection per environment.
func SplitLogCollections(st *State) error {
@@ -726,8 +728,13 @@ func SplitLogCollections(st *State) error {
seen := set.NewStrings()
iter := oldLogs.Find(nil).Iter()
- var doc bson.M
+ var (
+ doc bson.M
+ ids []string
+ )
+
for iter.Next(&doc) {
+
newCollName := logCollection(doc["e"].(string))
newLogs := db.C(newCollName)
@@ -740,9 +747,18 @@ func SplitLogCollections(st *State) error {
delete(doc, "e") // old env uuid
delete(doc, "r") // version - not needed
+
if err := newLogs.Insert(doc); err != nil {
return errors.Annotate(err, "failed to insert log record")
}
+ ids = append(ids, doc["_id"].(string))
+ if len(ids) >= splitBatchRemoveSize {
+ if err := oldLogs.Remove(bson.D{{"_id", bson.D{{"$in", ids}}}}); err != nil {
+ return errors.Annotate(err, "failed to remove batch of logs")
+ }
+ ids = nil
+ }
+
doc = nil
}
diff --git a/state/upgrades_test.go b/state/upgrades_test.go
index 81f476a..f8fd768 100644
--- a/state/upgrades_test.go
+++ b/state/upgrades_test.go
@@ -1320,6 +1320,9 @@ func (s *upgradesSuite) TestSplitLogCollection(c *gc.C) {
expected := map[string][]bson.M{}
+ // set batch remove size to 10 to trigger removal.
+ splitBatchRemoveSize = 10
+
for i := 0; i < 15; i++ {
modelUUID := uuids[i%3]
logRow := bson.M{