Ubuntu Pastebin

Paste from tim at Fri, 2 Jun 2017 04:45:29 +0000

Download as text
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
diff --git a/state/upgrades.go b/state/upgrades.go
index a7c0cf5..aa3216a 100644
--- a/state/upgrades.go
+++ b/state/upgrades.go
@@ -715,6 +715,8 @@ func AddStatusHistoryPruneSettings(st *State) error {
 	return nil
 }
 
+var splitBatchRemoveSize = 1000
+
 // SplitLogCollections moves log entries from the old single log collection
 // to the log collection per environment.
 func SplitLogCollections(st *State) error {
@@ -726,8 +728,13 @@ func SplitLogCollections(st *State) error {
 	seen := set.NewStrings()
 
 	iter := oldLogs.Find(nil).Iter()
-	var doc bson.M
+	var (
+		doc bson.M
+		ids []string
+	)
+
 	for iter.Next(&doc) {
+
 		newCollName := logCollection(doc["e"].(string))
 		newLogs := db.C(newCollName)
 
@@ -740,9 +747,18 @@ func SplitLogCollections(st *State) error {
 
 		delete(doc, "e") // old env uuid
 		delete(doc, "r") // version - not needed
+
 		if err := newLogs.Insert(doc); err != nil {
 			return errors.Annotate(err, "failed to insert log record")
 		}
+		ids = append(ids, doc["_id"].(string))
+		if len(ids) >= splitBatchRemoveSize {
+			if err := oldLogs.Remove(bson.D{{"_id", bson.D{{"$in", ids}}}}); err != nil {
+				return errors.Annotate(err, "failed to remove batch of logs")
+			}
+			ids = nil
+		}
+
 		doc = nil
 	}
 
diff --git a/state/upgrades_test.go b/state/upgrades_test.go
index 81f476a..f8fd768 100644
--- a/state/upgrades_test.go
+++ b/state/upgrades_test.go
@@ -1320,6 +1320,9 @@ func (s *upgradesSuite) TestSplitLogCollection(c *gc.C) {
 
 	expected := map[string][]bson.M{}
 
+	// set batch remove size to 10 to trigger removal.
+	splitBatchRemoveSize = 10
+
 	for i := 0; i < 15; i++ {
 		modelUUID := uuids[i%3]
 		logRow := bson.M{
Download as text