1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715 | HTTP/1.0 200 OK
Content-Type: text/plain; charset=utf-8
Date: Tue, 02 Feb 2016 02:24:25 GMT
goroutine profile: total 1303
1 @ 0xbade93 0xbadc73 0xbaa154 0x790bc9 0x790e46 0x8a672d 0x8a7ce4 0x8a8781 0x8a56d0 0x464e41
# 0xbade93 runtime/pprof.writeRuntimeProfile+0xb3 /home/dfc/go/src/runtime/pprof/pprof.go:545
# 0xbadc73 runtime/pprof.writeGoroutine+0xa3 /home/dfc/go/src/runtime/pprof/pprof.go:507
# 0xbaa154 runtime/pprof.(*Profile).WriteTo+0xe4 /home/dfc/go/src/runtime/pprof/pprof.go:236
# 0x790bc9 github.com/juju/juju/cmd/pprof.handler.ServeHTTP+0x1d9 /home/dfc/src/github.com/juju/juju/cmd/pprof/pprof.go:202
# 0x790e46 github.com/juju/juju/cmd/pprof.Index+0x136 /home/dfc/src/github.com/juju/juju/cmd/pprof/pprof.go:214
# 0x8a672d net/http.HandlerFunc.ServeHTTP+0x3d /home/dfc/go/src/net/http/server.go:1616
# 0x8a7ce4 net/http.(*ServeMux).ServeHTTP+0x154 /home/dfc/go/src/net/http/server.go:1908
# 0x8a8781 net/http.serverHandler.ServeHTTP+0x131 /home/dfc/go/src/net/http/server.go:2079
# 0x8a56d0 net/http.(*conn).serve+0xbf0 /home/dfc/go/src/net/http/server.go:1470
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa1ccd5 0x5846c6 0x58364f 0x5094d2 0x4ff941 0x407217 0x407bb3 0x407675 0x40887b 0x4377ac 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa1ccd5 github.com/juju/juju/worker.(*runner).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/runner.go:128
# 0x5846c6 github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).Run+0x796 /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:453
# 0x58364f github.com/juju/juju/cmd/jujud/agent.(*machineAgentCmd).Run+0x5f /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:254
# 0x5094d2 github.com/juju/cmd.(*SuperCommand).Run+0x4d2 /home/dfc/src/github.com/juju/cmd/supercommand.go:446
# 0x4ff941 github.com/juju/cmd.Main+0x281 /home/dfc/src/github.com/juju/cmd/cmd.go:279
# 0x407217 main.jujuDMain+0x6f7 /home/dfc/src/github.com/juju/juju/cmd/jujud/main.go:162
# 0x407bb3 main.Main+0x523 /home/dfc/src/github.com/juju/juju/cmd/jujud/main.go:199
# 0x407675 main.MainWrapper+0x35 /home/dfc/src/github.com/juju/juju/cmd/jujud/main.go:170
# 0x40887b main.main+0x3b /home/dfc/src/github.com/juju/juju/cmd/jujud/main_nix.go:22
# 0x4377ac runtime.main+0x22c /home/dfc/go/src/runtime/proc.go:188
1 @ 0x464e41
1 @ 0x437c2a 0x437cf3 0x437a36 0x464e41
# 0x437c2a runtime.gopark+0x14a /home/dfc/go/src/runtime/proc.go:248
# 0x437cf3 runtime.goparkunlock+0x53 /home/dfc/go/src/runtime/proc.go:268
# 0x437a36 runtime.forcegchelper+0xc6 /home/dfc/go/src/runtime/proc.go:229
1 @ 0x437c2a 0x437cf3 0x42841a 0x464e41
# 0x437c2a runtime.gopark+0x14a /home/dfc/go/src/runtime/proc.go:248
# 0x437cf3 runtime.goparkunlock+0x53 /home/dfc/go/src/runtime/proc.go:268
# 0x42841a runtime.bgsweep+0x12a /home/dfc/go/src/runtime/mgcsweep.go:79
1 @ 0x437c2a 0x437cf3 0x420e65 0x464e41
# 0x437c2a runtime.gopark+0x14a /home/dfc/go/src/runtime/proc.go:248
# 0x437cf3 runtime.goparkunlock+0x53 /home/dfc/go/src/runtime/proc.go:268
# 0x420e65 runtime.runfinq+0x3f5 /home/dfc/go/src/runtime/mfinal.go:158
1 @ 0x419fb2 0x44a217 0x803148 0x464e41
# 0x44a217 os/signal.signal_recv+0x107 /home/dfc/go/src/runtime/sigqueue.go:116
# 0x803148 os/signal.loop+0x18 /home/dfc/go/src/os/signal/signal_unix.go:22
9 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa1ecb5 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa1ecb5 github.com/juju/juju/worker.(*simpleWorker).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/simpleworker.go:34
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa28b19 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa28b19 github.com/juju/juju/worker/dependency.(*engine).Wait+0x39 /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:165
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
1 @ 0x437c2a 0x423d4f 0x464e41
# 0x437c2a runtime.gopark+0x14a /home/dfc/go/src/runtime/proc.go:248
# 0x423d4f runtime.gcBgMarkWorker+0xdf /home/dfc/go/src/runtime/mgc.go:1382
6 @ 0x437c2a 0x446482 0x445472 0xa1e0fe 0xa1edc9 0x464e41
# 0xa1e0fe github.com/juju/juju/worker.(*runner).run+0x12ae /home/dfc/src/github.com/juju/juju/worker/runner.go:166
# 0xa1edc9 github.com/juju/juju/worker.NewRunner.func1+0x49 /home/dfc/src/github.com/juju/juju/worker/runner.go:90
1 @ 0x437c2a 0x446482 0x445472 0x799e63 0x464e41
# 0x799e63 github.com/juju/juju/worker/logsender.(*BufferedLogWriter).loop+0x253 /home/dfc/src/github.com/juju/juju/worker/logsender/bufferedlogwriter.go:100
1 @ 0x437c2a 0x432b27 0x4321ee 0x495960 0x4959cc 0x498d7f 0x4bfc4b 0x4bff09 0x8a8a6f 0x791940 0x464e41
# 0x4321ee net.runtime_pollWait+0x5e /home/dfc/go/src/runtime/netpoll.go:160
# 0x495960 net.(*pollDesc).Wait+0x40 /home/dfc/go/src/net/fd_poll_runtime.go:73
# 0x4959cc net.(*pollDesc).WaitRead+0x3c /home/dfc/go/src/net/fd_poll_runtime.go:78
# 0x498d7f net.(*netFD).accept+0x1ff /home/dfc/go/src/net/fd_unix.go:426
# 0x4bfc4b net.(*UnixListener).AcceptUnix+0x5b /home/dfc/go/src/net/unixsock_posix.go:305
# 0x4bff09 net.(*UnixListener).Accept+0x49 /home/dfc/go/src/net/unixsock_posix.go:315
# 0x8a8a6f net/http.(*Server).Serve+0x10f /home/dfc/go/src/net/http/server.go:2115
# 0x791940 github.com/juju/juju/cmd/pprof.Start.func4+0x70 /home/dfc/src/github.com/juju/juju/cmd/pprof/socket.go:63
1 @ 0x437c2a 0x446482 0x445472 0x585a11 0x59444c 0xa1ee5f 0x464e41
# 0x585a11 github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).stateStarter+0x2f1 /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:647
# 0x59444c github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).(github.com/juju/juju/cmd/jujud/agent.stateStarter)-fm+0x3c /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:627
# 0xa1ee5f github.com/juju/juju/worker.NewSimpleWorker.func1+0x5f /home/dfc/src/github.com/juju/juju/worker/simpleworker.go:20
1 @ 0x437c2a 0x437cf3 0x44763d 0x7eeb39 0x97d267 0x594491 0x464e41
# 0x44763d sync.runtime_Syncsemacquire+0x1bd /home/dfc/go/src/runtime/sema.go:241
# 0x7eeb39 sync.(*Cond).Wait+0x69 /home/dfc/go/src/sync/cond.go:63
# 0x97d267 github.com/juju/utils/voyeur.(*Watcher).Next+0x107 /home/dfc/src/github.com/juju/utils/voyeur/value.go:129
# 0x594491 github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).stateStarter.func1+0x21 /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:642
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa1ccd5 0x59daaf 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa1ccd5 github.com/juju/juju/worker.(*runner).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/runner.go:128
# 0x59daaf github.com/juju/juju/cmd/jujud/util.(*CloseWorker).Wait+0x3f /home/dfc/src/github.com/juju/juju/cmd/jujud/util/util.go:222
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
4 @ 0x437c2a 0x437cf3 0x4519a0 0x9948d7 0x464e41
# 0x4519a0 time.Sleep+0xf0 /home/dfc/go/src/runtime/time.go:59
# 0x9948d7 gopkg.in/mgo%2ev2.(*mongoServer).pinger+0x187 /home/dfc/src/gopkg.in/mgo.v2/server.go:296
1 @ 0x437c2a 0x446482 0x445472 0xa28a71 0xa2d9d2 0x464e41
# 0xa28a71 github.com/juju/juju/worker/dependency.(*engine).loop+0x5d1 /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:133
# 0xa2d9d2 github.com/juju/juju/worker/dependency.NewEngine.func1+0x52 /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:81
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0x103eda5 0xa2e5e1 0xa2aeda 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0x103eda5 github.com/juju/juju/worker/terminationworker.(*terminationWorker).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/terminationworker/worker.go:46
# 0xa2e5e1 github.com/juju/juju/worker/dependency.(*engine).runWorker.func2+0x641 /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:428
# 0xa2aeda github.com/juju/juju/worker/dependency.(*engine).runWorker+0x13a /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:432
1 @ 0x437c2a 0x446482 0x445472 0x45b6b5 0x464e41
# 0x437c2a runtime.gopark+0x14a /home/dfc/go/src/runtime/proc.go:248
# 0x446482 runtime.selectgoImpl+0xfe2 /home/dfc/go/src/runtime/select.go:392
# 0x445472 runtime.selectgo+0x12 /home/dfc/go/src/runtime/select.go:215
# 0x45b6b5 runtime.ensureSigM.func1+0x385 /home/dfc/go/src/runtime/signal1_unix.go:279
2 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0xa40eec 0x464e41
# 0xa40eec github.com/juju/juju/worker/gate.ManifoldEx.func1.1+0x6c /home/dfc/src/github.com/juju/juju/worker/gate/manifold.go:38
2 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa40e55 0xa2e5e1 0xa2aeda 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa40e55 github.com/juju/juju/worker/gate.(*gate).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/gate/manifold.go:121
# 0xa2e5e1 github.com/juju/juju/worker/dependency.(*engine).runWorker.func2+0x641 /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:428
# 0xa2aeda github.com/juju/juju/worker/dependency.(*engine).runWorker+0x13a /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:432
57 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xe37b85 0xd53bed 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xe37b85 github.com/juju/juju/api/watcher.(*commonWatcher).Wait+0x35 /home/dfc/src/github.com/juju/juju/api/watcher/watcher.go:129
# 0xd53bed github.com/juju/juju/worker/catacomb.(*Catacomb).add.func1+0x7d /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:175
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x103c6bc 0x464e41
# 0x103c6bc github.com/juju/juju/worker/agent.startFunc.func1.1+0x6c /home/dfc/src/github.com/juju/juju/worker/agent/manifold.go:31
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0x103fa65 0xa2e5e1 0xa2aeda 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0x103fa65 github.com/juju/juju/worker/upgradewaiter.(*upgradeWaiter).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/upgradewaiter/manifold.go:132
# 0xa2e5e1 github.com/juju/juju/worker/dependency.(*engine).runWorker.func2+0x641 /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:428
# 0xa2aeda github.com/juju/juju/worker/dependency.(*engine).runWorker+0x13a /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:432
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0x103c625 0xa2e5e1 0xa2aeda 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0x103c625 github.com/juju/juju/worker/agent.(*agentWorker).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/agent/manifold.go:62
# 0xa2e5e1 github.com/juju/juju/worker/dependency.(*engine).runWorker.func2+0x641 /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:428
# 0xa2aeda github.com/juju/juju/worker/dependency.(*engine).runWorker+0x13a /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:432
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa22515 0xa2e5e1 0xa2aeda 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa22515 github.com/juju/juju/worker/apicaller.(*apiConnWorker).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/apicaller/worker.go:53
# 0xa2e5e1 github.com/juju/juju/worker/dependency.(*engine).runWorker.func2+0x641 /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:428
# 0xa2aeda github.com/juju/juju/worker/dependency.(*engine).runWorker+0x13a /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:432
2 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xd53565 0xa1b039 0xa2e5e1 0xa2aeda 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xd53565 github.com/juju/juju/worker/catacomb.(*Catacomb).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:202
# 0xa1b039 github.com/juju/juju/watcher.(*NotifyWorker).Wait+0x39 /home/dfc/src/github.com/juju/juju/watcher/notify.go:138
# 0xa2e5e1 github.com/juju/juju/worker/dependency.(*engine).runWorker.func2+0x641 /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:428
# 0xa2aeda github.com/juju/juju/worker/dependency.(*engine).runWorker+0x13a /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:432
1 @ 0x419fb2 0x452079 0x464e41
# 0x419fb2 runtime.notetsleepg+0x52 /home/dfc/go/src/runtime/lock_futex.go:199
# 0x452079 runtime.timerproc+0x119 /home/dfc/go/src/runtime/time.go:209
1 @ 0x437c2a 0x446482 0x445472 0x103f015 0x103f109 0x464e41
# 0x103f015 github.com/juju/juju/worker/terminationworker.(*terminationWorker).loop+0x245 /home/dfc/src/github.com/juju/juju/worker/terminationworker/worker.go:53
# 0x103f109 github.com/juju/juju/worker/terminationworker.NewWorker.func1+0x49 /home/dfc/src/github.com/juju/juju/worker/terminationworker/worker.go:36
21 @ 0x437c2a 0x437cf3 0x410815 0x410490 0xe383c4 0xe3a0cb 0x464e41
# 0xe383c4 github.com/juju/juju/api/watcher.(*stringsWatcher).loop+0x1e4 /home/dfc/src/github.com/juju/juju/api/watcher/watcher.go:222
# 0xe3a0cb github.com/juju/juju/api/watcher.NewStringsWatcher.func1+0x6b /home/dfc/src/github.com/juju/juju/api/watcher/watcher.go:202
39 @ 0x437c2a 0x446482 0x445472 0xd53aab 0x464e41
# 0xd53aab github.com/juju/juju/worker/catacomb.Invoke.func2+0x16b /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:100
2 @ 0x437c2a 0x446482 0x445472 0x98bb4c 0x464e41
# 0x98bb4c gopkg.in/mgo%2ev2.(*mongoCluster).syncServersLoop+0x48c /home/dfc/src/gopkg.in/mgo.v2/cluster.go:394
31 @ 0x437c2a 0x432b27 0x4321ee 0x495960 0x4959cc 0x497489 0x4aae36 0x83c3a3 0x83cb08 0x840256 0x9b609b 0x9b627d 0x464e41
# 0x4321ee net.runtime_pollWait+0x5e /home/dfc/go/src/runtime/netpoll.go:160
# 0x495960 net.(*pollDesc).Wait+0x40 /home/dfc/go/src/net/fd_poll_runtime.go:73
# 0x4959cc net.(*pollDesc).WaitRead+0x3c /home/dfc/go/src/net/fd_poll_runtime.go:78
# 0x497489 net.(*netFD).Read+0x1c9 /home/dfc/go/src/net/fd_unix.go:250
# 0x4aae36 net.(*conn).Read+0x86 /home/dfc/go/src/net/net.go:172
# 0x83c3a3 crypto/tls.(*block).readFromUntil+0xb3 /home/dfc/go/src/crypto/tls/conn.go:460
# 0x83cb08 crypto/tls.(*Conn).readRecord+0x1e8 /home/dfc/go/src/crypto/tls/conn.go:562
# 0x840256 crypto/tls.(*Conn).Read+0x106 /home/dfc/go/src/crypto/tls/conn.go:939
# 0x9b609b gopkg.in/mgo%2ev2.fill+0x5b /home/dfc/src/gopkg.in/mgo.v2/socket.go:529
# 0x9b627d gopkg.in/mgo%2ev2.(*mongoSocket).readLoop+0x10d /home/dfc/src/gopkg.in/mgo.v2/socket.go:546
4 @ 0x437c2a 0x446482 0x445472 0xb57eaf 0xb577f2 0xb59662 0xd53b36 0x464e41
# 0xb57eaf github.com/juju/juju/worker/lease.(*Manager).choose+0x56f /home/dfc/src/github.com/juju/juju/worker/lease/manager.go:92
# 0xb577f2 github.com/juju/juju/worker/lease.(*Manager).loop+0xb2 /home/dfc/src/github.com/juju/juju/worker/lease/manager.go:77
# 0xb59662 github.com/juju/juju/worker/lease.(*Manager).(github.com/juju/juju/worker/lease.loop)-fm+0x32 /home/dfc/src/github.com/juju/juju/worker/lease/manager.go:38
# 0xd53b36 github.com/juju/juju/worker/catacomb.Invoke.func3+0x76 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:115
2 @ 0x437c2a 0x446482 0x445472 0xb4d672 0xb5038d 0x464e41
# 0xb4d672 github.com/juju/juju/state/watcher.(*Watcher).loop+0x2f2 /home/dfc/src/github.com/juju/juju/state/watcher/watcher.go:231
# 0xb5038d github.com/juju/juju/state/watcher.New.func1+0x2d /home/dfc/src/github.com/juju/juju/state/watcher/watcher.go:115
2 @ 0x437c2a 0x446482 0x445472 0xb5f040 0xb6558d 0x464e41
# 0xb5f040 github.com/juju/juju/state/presence.(*Watcher).loop+0x3b0 /home/dfc/src/github.com/juju/juju/state/presence/presence.go:256
# 0xb6558d github.com/juju/juju/state/presence.NewWatcher.func1+0x2d /home/dfc/src/github.com/juju/juju/state/presence/presence.go:139
7 @ 0x437c2a 0x446482 0x445472 0x599152 0xa1ee5f 0x464e41
# 0x599152 github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).upgradeWaiterWorker.func1+0x752 /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:1831
# 0xa1ee5f github.com/juju/juju/worker.NewSimpleWorker.func1+0x5f /home/dfc/src/github.com/juju/juju/worker/simpleworker.go:20
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0x9d1955 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0x9d1955 github.com/juju/juju/apiserver.(*Server).Wait+0x35 /home/dfc/src/github.com/juju/juju/apiserver/apiserver.go:231
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
1 @ 0x437c2a 0x446482 0x445472 0x9d0562 0x9ea9d2 0x464e41
# 0x9d0562 github.com/juju/juju/apiserver.(*changeCertListener).processCertChanges+0x232 /home/dfc/src/github.com/juju/juju/apiserver/apiserver.go:123
# 0x9ea9d2 github.com/juju/juju/apiserver.newChangeCertListener.func1+0x52 /home/dfc/src/github.com/juju/juju/apiserver/apiserver.go:96
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9d3603 0x464e41
# 0x9d3603 github.com/juju/juju/apiserver.(*Server).run+0x8c3 /home/dfc/src/github.com/juju/juju/apiserver/apiserver.go:411
1 @ 0x437c2a 0x446482 0x445472 0x9d4a97 0x9eaaa1 0x464e41
# 0x9d4a97 github.com/juju/juju/apiserver.(*Server).mongoPinger+0x297 /home/dfc/src/github.com/juju/juju/apiserver/apiserver.go:498
# 0x9eaaa1 github.com/juju/juju/apiserver.(*Server).run.func2+0x21 /home/dfc/src/github.com/juju/juju/apiserver/apiserver.go:326
1 @ 0x437c2a 0x432b27 0x4321ee 0x495960 0x4959cc 0x498d7f 0x4b6275 0x4b64e9 0x9d00e1 0x8a8a6f 0x8a8596 0x9eabb6 0x464e41
# 0x4321ee net.runtime_pollWait+0x5e /home/dfc/go/src/runtime/netpoll.go:160
# 0x495960 net.(*pollDesc).Wait+0x40 /home/dfc/go/src/net/fd_poll_runtime.go:73
# 0x4959cc net.(*pollDesc).WaitRead+0x3c /home/dfc/go/src/net/fd_poll_runtime.go:78
# 0x498d7f net.(*netFD).accept+0x1ff /home/dfc/go/src/net/fd_unix.go:426
# 0x4b6275 net.(*TCPListener).AcceptTCP+0x55 /home/dfc/go/src/net/tcpsock_posix.go:254
# 0x4b64e9 net.(*TCPListener).Accept+0x49 /home/dfc/go/src/net/tcpsock_posix.go:264
# 0x9d00e1 github.com/juju/juju/apiserver.(*changeCertListener).Accept+0x51 /home/dfc/src/github.com/juju/juju/apiserver/apiserver.go:103
# 0x8a8a6f net/http.(*Server).Serve+0x10f /home/dfc/go/src/net/http/server.go:2115
# 0x8a8596 net/http.Serve+0x96 /home/dfc/go/src/net/http/server.go:1974
# 0x9eabb6 github.com/juju/juju/apiserver.(*Server).run.func3+0x46 /home/dfc/src/github.com/juju/juju/apiserver/apiserver.go:408
2 @ 0x437c2a 0x432b27 0x4321ee 0x495960 0x4959cc 0x497489 0x4aae36 0x83c3a3 0x83cb08 0x840256 0x8de3fd 0x8dec22 0xdee5ad 0xdf4bc7 0xe0604f 0xe05309 0xe022e7 0xe01fc0 0x464e41
# 0x4321ee net.runtime_pollWait+0x5e /home/dfc/go/src/runtime/netpoll.go:160
# 0x495960 net.(*pollDesc).Wait+0x40 /home/dfc/go/src/net/fd_poll_runtime.go:73
# 0x4959cc net.(*pollDesc).WaitRead+0x3c /home/dfc/go/src/net/fd_poll_runtime.go:78
# 0x497489 net.(*netFD).Read+0x1c9 /home/dfc/go/src/net/fd_unix.go:250
# 0x4aae36 net.(*conn).Read+0x86 /home/dfc/go/src/net/net.go:172
# 0x83c3a3 crypto/tls.(*block).readFromUntil+0xb3 /home/dfc/go/src/crypto/tls/conn.go:460
# 0x83cb08 crypto/tls.(*Conn).readRecord+0x1e8 /home/dfc/go/src/crypto/tls/conn.go:562
# 0x840256 crypto/tls.(*Conn).Read+0x106 /home/dfc/go/src/crypto/tls/conn.go:939
# 0x8de3fd bufio.(*Reader).fill+0x15d /home/dfc/go/src/bufio/bufio.go:97
# 0x8dec22 bufio.(*Reader).ReadByte+0x82 /home/dfc/go/src/bufio/bufio.go:229
# 0xdee5ad golang.org/x/net/websocket.hybiFrameReaderFactory.NewFrameReader+0x9d /home/dfc/src/golang.org/x/net/websocket/hybi.go:126
# 0xdf4bc7 golang.org/x/net/websocket.Codec.Receive+0x147 /home/dfc/src/golang.org/x/net/websocket/websocket.go:315
# 0xe0604f github.com/juju/juju/rpc/jsoncodec.wsJSONConn.Receive+0x5f /home/dfc/src/github.com/juju/juju/rpc/jsoncodec/conn.go:25
# 0xe05309 github.com/juju/juju/rpc/jsoncodec.(*Codec).ReadHeader+0x699 /home/dfc/src/github.com/juju/juju/rpc/jsoncodec/codec.go:115
# 0xe022e7 github.com/juju/juju/rpc.(*Conn).loop+0xa7 /home/dfc/src/github.com/juju/juju/rpc/server.go:408
# 0xe01fc0 github.com/juju/juju/rpc.(*Conn).input+0x30 /home/dfc/src/github.com/juju/juju/rpc/server.go:381
2 @ 0x437c2a 0x446482 0x445472 0x94cdd8 0x464e41
# 0x94cdd8 github.com/juju/juju/api.(*state).heartbeatMonitor+0x158 /home/dfc/src/github.com/juju/juju/api/apiclient.go:525
20 @ 0x437c2a 0x446482 0x445472 0x9e20d7 0x9eca19 0x464e41
# 0x9e20d7 github.com/juju/juju/apiserver.(*pingTimeout).loop+0x2a7 /home/dfc/src/github.com/juju/juju/apiserver/pinger.go:85
# 0x9eca19 github.com/juju/juju/apiserver.newPingTimeout.func1+0x49 /home/dfc/src/github.com/juju/juju/apiserver/pinger.go:59
20 @ 0x437c2a 0x446482 0x445472 0x9d42c7 0x9eae4e 0xdf3a1d 0xdf37a1 0x9d3e0b 0x9eab5e 0x8a672d 0x100c3f9 0x8a8781 0x8a56d0 0x464e41
# 0x9d42c7 github.com/juju/juju/apiserver.(*Server).serveConn+0x417 /home/dfc/src/github.com/juju/juju/apiserver/apiserver.go:469
# 0x9eae4e github.com/juju/juju/apiserver.(*Server).apiHandler.func1+0x24e /home/dfc/src/github.com/juju/juju/apiserver/apiserver.go:432
# 0xdf3a1d golang.org/x/net/websocket.Server.serveWebSocket+0x26d /home/dfc/src/golang.org/x/net/websocket/server.go:89
# 0xdf37a1 golang.org/x/net/websocket.Server.ServeHTTP+0x61 /home/dfc/src/golang.org/x/net/websocket/server.go:70
# 0x9d3e0b github.com/juju/juju/apiserver.(*Server).apiHandler+0x15b /home/dfc/src/github.com/juju/juju/apiserver/apiserver.go:437
# 0x9eab5e github.com/juju/juju/apiserver.(*Server).(github.com/juju/juju/apiserver.apiHandler)-fm+0x3e /home/dfc/src/github.com/juju/juju/apiserver/apiserver.go:376
# 0x8a672d net/http.HandlerFunc.ServeHTTP+0x3d /home/dfc/go/src/net/http/server.go:1616
# 0x100c3f9 github.com/bmizerany/pat.(*PatternServeMux).ServeHTTP+0x199 /home/dfc/src/github.com/bmizerany/pat/mux.go:109
# 0x8a8781 net/http.serverHandler.ServeHTTP+0x131 /home/dfc/go/src/net/http/server.go:2079
# 0x8a56d0 net/http.(*conn).serve+0xbf0 /home/dfc/go/src/net/http/server.go:1470
20 @ 0x437c2a 0x432b27 0x4321ee 0x495960 0x4959cc 0x497489 0x4aae36 0x83c3a3 0x83cb08 0x840256 0x8a00b6 0x8de3fd 0x8dec22 0xdee5ad 0xdf4bc7 0xe0604f 0xe05309 0xe022e7 0xe01fc0 0x464e41
# 0x4321ee net.runtime_pollWait+0x5e /home/dfc/go/src/runtime/netpoll.go:160
# 0x495960 net.(*pollDesc).Wait+0x40 /home/dfc/go/src/net/fd_poll_runtime.go:73
# 0x4959cc net.(*pollDesc).WaitRead+0x3c /home/dfc/go/src/net/fd_poll_runtime.go:78
# 0x497489 net.(*netFD).Read+0x1c9 /home/dfc/go/src/net/fd_unix.go:250
# 0x4aae36 net.(*conn).Read+0x86 /home/dfc/go/src/net/net.go:172
# 0x83c3a3 crypto/tls.(*block).readFromUntil+0xb3 /home/dfc/go/src/crypto/tls/conn.go:460
# 0x83cb08 crypto/tls.(*Conn).readRecord+0x1e8 /home/dfc/go/src/crypto/tls/conn.go:562
# 0x840256 crypto/tls.(*Conn).Read+0x106 /home/dfc/go/src/crypto/tls/conn.go:939
# 0x8a00b6 net/http.(*connReader).Read+0x196 /home/dfc/go/src/net/http/server.go:526
# 0x8de3fd bufio.(*Reader).fill+0x15d /home/dfc/go/src/bufio/bufio.go:97
# 0x8dec22 bufio.(*Reader).ReadByte+0x82 /home/dfc/go/src/bufio/bufio.go:229
# 0xdee5ad golang.org/x/net/websocket.hybiFrameReaderFactory.NewFrameReader+0x9d /home/dfc/src/golang.org/x/net/websocket/hybi.go:126
# 0xdf4bc7 golang.org/x/net/websocket.Codec.Receive+0x147 /home/dfc/src/golang.org/x/net/websocket/websocket.go:315
# 0xe0604f github.com/juju/juju/rpc/jsoncodec.wsJSONConn.Receive+0x5f /home/dfc/src/github.com/juju/juju/rpc/jsoncodec/conn.go:25
# 0xe05309 github.com/juju/juju/rpc/jsoncodec.(*Codec).ReadHeader+0x699 /home/dfc/src/github.com/juju/juju/rpc/jsoncodec/codec.go:115
# 0xe022e7 github.com/juju/juju/rpc.(*Conn).loop+0xa7 /home/dfc/src/github.com/juju/juju/rpc/server.go:408
# 0xe01fc0 github.com/juju/juju/rpc.(*Conn).input+0x30 /home/dfc/src/github.com/juju/juju/rpc/server.go:381
20 @ 0x437c2a 0x446482 0x445472 0xb63934 0xb6574d 0x464e41
# 0xb63934 github.com/juju/juju/state/presence.(*Pinger).loop+0x1c4 /home/dfc/src/github.com/juju/juju/state/presence/presence.go:625
# 0xb6574d github.com/juju/juju/state/presence.(*Pinger).Start.func1+0x2d /home/dfc/src/github.com/juju/juju/state/presence/presence.go:536
47 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0xe01581 0x94d06b 0xe39ae9 0xe39da0 0x464e41
# 0xe01581 github.com/juju/juju/rpc.(*Conn).Call+0xf1 /home/dfc/src/github.com/juju/juju/rpc/client.go:152
# 0x94d06b github.com/juju/juju/api.(*state).APICall+0x13b /home/dfc/src/github.com/juju/juju/api/apiclient.go:547
# 0xe39ae9 github.com/juju/juju/api/watcher.makeWatcherAPICaller.func1+0x159 /home/dfc/src/github.com/juju/juju/api/watcher/watcher.go:49
# 0xe39da0 github.com/juju/juju/api/watcher.(*commonWatcher).commonLoop.func2+0xe0 /home/dfc/src/github.com/juju/juju/api/watcher/watcher.go:95
1 @ 0x437c2a 0x446482 0x445472 0xa226fe 0xa22ec9 0x464e41
# 0xa226fe github.com/juju/juju/worker/apicaller.(*apiConnWorker).loop+0x1be /home/dfc/src/github.com/juju/juju/worker/apicaller/worker.go:77
# 0xa22ec9 github.com/juju/juju/worker/apicaller.newApiConnWorker.func1+0x49 /home/dfc/src/github.com/juju/juju/worker/apicaller/worker.go:36
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xd53565 0xa9f315 0xa2e5e1 0xa2aeda 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xd53565 github.com/juju/juju/worker/catacomb.(*Catacomb).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:202
# 0xa9f315 github.com/juju/juju/worker/upgrader.(*Upgrader).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/upgrader/upgrader.go:85
# 0xa2e5e1 github.com/juju/juju/worker/dependency.(*engine).runWorker.func2+0x641 /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:428
# 0xa2aeda github.com/juju/juju/worker/dependency.(*engine).runWorker+0x13a /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:432
173 @ 0x437c2a 0x446482 0x445472 0x711462 0x73707e 0x464e41
# 0x711462 github.com/juju/juju/state.(*docWatcher).loop+0x5b2 /home/dfc/src/github.com/juju/juju/state/watcher.go:1582
# 0x73707e github.com/juju/juju/state.newDocWatcher.func1+0x9e /home/dfc/src/github.com/juju/juju/state/watcher.go:1541
47 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0xe39bbc 0x464e41
# 0xe39bbc github.com/juju/juju/api/watcher.(*commonWatcher).commonLoop.func1+0x7c /home/dfc/src/github.com/juju/juju/api/watcher/watcher.go:82
1 @ 0x437c2a 0x446482 0x445472 0xaa0294 0xaa13d2 0xd53b36 0x464e41
# 0xaa0294 github.com/juju/juju/worker/upgrader.(*Upgrader).loop+0xee4 /home/dfc/src/github.com/juju/juju/worker/upgrader/upgrader.go:153
# 0xaa13d2 github.com/juju/juju/worker/upgrader.(*Upgrader).(github.com/juju/juju/worker/upgrader.loop)-fm+0x32 /home/dfc/src/github.com/juju/juju/worker/upgrader/upgrader.go:70
# 0xd53b36 github.com/juju/juju/worker/catacomb.Invoke.func3+0x76 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:115
47 @ 0x437c2a 0x437cf3 0x446d3c 0x446a26 0x7f038b 0xe37add 0x464e41
# 0x446a26 sync.runtime_Semacquire+0x26 /home/dfc/go/src/runtime/sema.go:47
# 0x7f038b sync.(*WaitGroup).Wait+0x8b /home/dfc/go/src/sync/waitgroup.go:127
# 0xe37add github.com/juju/juju/api/watcher.(*commonWatcher).commonLoop+0xfd /home/dfc/src/github.com/juju/juju/api/watcher/watcher.go:119
201 @ 0x437c2a 0x437cf3 0x410815 0x410490 0x9e8fa3 0x46276e 0x7deaf7 0x7de17b 0x1035e39 0x9e2dee 0xe0382c 0x464e41
# 0x9e8fa3 github.com/juju/juju/apiserver.(*srvNotifyWatcher).Next+0x63 /home/dfc/src/github.com/juju/juju/apiserver/watcher.go:126
# 0x46276e runtime.call32+0x3e /home/dfc/go/src/runtime/asm_amd64.s:472
# 0x7deaf7 reflect.Value.call+0x837 /home/dfc/go/src/reflect/value.go:435
# 0x7de17b reflect.Value.Call+0xbb /home/dfc/go/src/reflect/value.go:303
# 0x1035e39 github.com/juju/juju/rpc/rpcreflect.newMethod.func4+0x129 /home/dfc/src/github.com/juju/juju/rpc/rpcreflect/type.go:306
# 0x9e2dee github.com/juju/juju/apiserver.(*srvCaller).Call+0x11e /home/dfc/src/github.com/juju/juju/apiserver/root.go:123
# 0xe0382c github.com/juju/juju/rpc.(*Conn).runRequest+0xcc /home/dfc/src/github.com/juju/juju/rpc/server.go:555
22 @ 0x437c2a 0x437cf3 0x410815 0x410490 0xe37f15 0xe3a009 0x464e41
# 0xe37f15 github.com/juju/juju/api/watcher.(*notifyWatcher).loop+0x1b5 /home/dfc/src/github.com/juju/juju/api/watcher/watcher.go:171
# 0xe3a009 github.com/juju/juju/api/watcher.NewNotifyWatcher.func1+0x49 /home/dfc/src/github.com/juju/juju/api/watcher/watcher.go:151
67 @ 0x437c2a 0x446482 0x445472 0xd53d26 0x464e41
# 0xd53d26 github.com/juju/juju/worker/catacomb.(*Catacomb).add.func2+0xf6 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:180
1 @ 0x437c2a 0x446482 0x445472 0xa94182 0xa1ee5f 0x464e41
# 0xa94182 github.com/juju/juju/worker/txnpruner.New.func1+0x1e2 /home/dfc/src/github.com/juju/juju/worker/txnpruner/txnpruner.go:30
# 0xa1ee5f github.com/juju/juju/worker.NewSimpleWorker.func1+0x5f /home/dfc/src/github.com/juju/juju/worker/simpleworker.go:20
3 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa1ecb5 0x5989b7 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa1ecb5 github.com/juju/juju/worker.(*simpleWorker).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/simpleworker.go:34
# 0x5989b7 github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).upgradeWaiterWorker.func1.1+0x27 /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:1829
1 @ 0x437c2a 0x446482 0x445472 0x103f94a 0x103fad9 0x464e41
# 0x103f94a github.com/juju/juju/worker/upgradewaiter.(*upgradeWaiter).wait+0x29a /home/dfc/src/github.com/juju/juju/worker/upgradewaiter/manifold.go:106
# 0x103fad9 github.com/juju/juju/worker/upgradewaiter.Manifold.func1.1+0x49 /home/dfc/src/github.com/juju/juju/worker/upgradewaiter/manifold.go:65
1 @ 0x437c2a 0x446482 0x445472 0x58563c 0x594395 0xa1ee5f 0x464e41
# 0x58563c github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).restoreStateWatcher+0x1bc /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:613
# 0x594395 github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).newRestoreStateWatcherWorker.func1+0x45 /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:581
# 0xa1ee5f github.com/juju/juju/worker.NewSimpleWorker.func1+0x5f /home/dfc/src/github.com/juju/juju/worker/simpleworker.go:20
1 @ 0x437c2a 0x446482 0x445472 0xa362e5 0xa36ac2 0x464e41
# 0xa362e5 github.com/juju/juju/worker/envworkermanager.(*envWorkerManager).loop+0x2b5 /home/dfc/src/github.com/juju/juju/worker/envworkermanager/envworkermanager.go:91
# 0xa36ac2 github.com/juju/juju/worker/envworkermanager.NewEnvWorkerManager.func1+0x52 /home/dfc/src/github.com/juju/juju/worker/envworkermanager/envworkermanager.go:42
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa36009 0x5989b7 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa36009 github.com/juju/juju/worker/envworkermanager.(*envWorkerManager).Wait+0x39 /home/dfc/src/github.com/juju/juju/worker/envworkermanager/envworkermanager.go:74
# 0x5989b7 github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).upgradeWaiterWorker.func1.1+0x27 /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:1829
1 @ 0x437c2a 0x446482 0x445472 0x1062250 0x1062839 0x464e41
# 0x1062250 github.com/juju/juju/watcher/legacy.(*notifyWorker).loop+0x380 /home/dfc/src/github.com/juju/juju/watcher/legacy/notifyworker.go:95
# 0x1062839 github.com/juju/juju/watcher/legacy.NewNotifyWorker.func1+0x49 /home/dfc/src/github.com/juju/juju/watcher/legacy/notifyworker.go:55
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0x1061e45 0x5989b7 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0x1061e45 github.com/juju/juju/watcher/legacy.(*notifyWorker).Wait+0x35 /home/dfc/src/github.com/juju/juju/watcher/legacy/notifyworker.go:67
# 0x5989b7 github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).upgradeWaiterWorker.func1.1+0x27 /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:1829
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa1ccd5 0xa36b29 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa1ccd5 github.com/juju/juju/worker.(*runner).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/runner.go:128
# 0xa36b29 github.com/juju/juju/worker/envworkermanager.(*envWorkerManager).loop.func1+0x29 /home/dfc/src/github.com/juju/juju/worker/envworkermanager/envworkermanager.go:80
75 @ 0x437c2a 0x446482 0x445472 0x7070fd 0x736580 0x464e41
# 0x7070fd github.com/juju/juju/state.(*lifecycleWatcher).loop+0x60d /home/dfc/src/github.com/juju/juju/state/watcher.go:549
# 0x736580 github.com/juju/juju/state.newLifecycleWatcher.func1+0x80 /home/dfc/src/github.com/juju/juju/state/watcher.go:431
1 @ 0x437c2a 0x446482 0x445472 0xa27dcc 0xa27e1c 0xa1ee5f 0x464e41
# 0xa27dcc github.com/juju/juju/worker/dblogpruner.(*pruneWorker).loop+0x2ac /home/dfc/src/github.com/juju/juju/worker/dblogpruner/worker.go:56
# 0xa27e1c github.com/juju/juju/worker/dblogpruner.(*pruneWorker).(github.com/juju/juju/worker/dblogpruner.loop)-fm+0x3c /home/dfc/src/github.com/juju/juju/worker/dblogpruner/worker.go:45
# 0xa1ee5f github.com/juju/juju/worker.NewSimpleWorker.func1+0x5f /home/dfc/src/github.com/juju/juju/worker/simpleworker.go:20
1 @ 0x437c2a 0x446482 0x445472 0x76ed0e 0x772189 0x464e41
# 0x76ed0e github.com/juju/juju/worker/peergrouper.(*pgWorker).loop+0x5ce /home/dfc/src/github.com/juju/juju/worker/peergrouper/worker.go:165
# 0x772189 github.com/juju/juju/worker/peergrouper.newWorker.func1+0x59 /home/dfc/src/github.com/juju/juju/worker/peergrouper/worker.go:137
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0x76e715 0x5989b7 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0x76e715 github.com/juju/juju/worker/peergrouper.(*pgWorker).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/peergrouper/worker.go:155
# 0x5989b7 github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).upgradeWaiterWorker.func1.1+0x27 /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:1829
2 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa1ccd5 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa1ccd5 github.com/juju/juju/worker.(*runner).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/runner.go:128
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
1 @ 0x437c2a 0x446482 0x445472 0x7709eb 0x772362 0x7722ec 0x464e41
# 0x7709eb github.com/juju/juju/worker/peergrouper.(*serverInfoWatcher).loop+0x20b /home/dfc/src/github.com/juju/juju/worker/peergrouper/worker.go:384
# 0x772362 github.com/juju/juju/worker/peergrouper.(*serverInfoWatcher).(github.com/juju/juju/worker/peergrouper.loop)-fm+0x32 /home/dfc/src/github.com/juju/juju/worker/peergrouper/worker.go:378
# 0x7722ec github.com/juju/juju/worker/peergrouper.(*pgWorker).start.func1+0x4c /home/dfc/src/github.com/juju/juju/worker/peergrouper/worker.go:345
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa1ccd5 0xa2e5e1 0xa2aeda 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa1ccd5 github.com/juju/juju/worker.(*runner).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/runner.go:128
# 0xa2e5e1 github.com/juju/juju/worker/dependency.(*engine).runWorker.func2+0x641 /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:428
# 0xa2aeda github.com/juju/juju/worker/dependency.(*engine).runWorker+0x13a /home/dfc/src/github.com/juju/juju/worker/dependency/engine.go:432
7 @ 0x437c2a 0x446482 0x445472 0xa1ad9d 0xa1b8f2 0xd53b36 0x464e41
# 0xa1ad9d github.com/juju/juju/watcher.(*NotifyWorker).loop+0x25d /home/dfc/src/github.com/juju/juju/watcher/notify.go:90
# 0xa1b8f2 github.com/juju/juju/watcher.(*NotifyWorker).(github.com/juju/juju/watcher.loop)-fm+0x32 /home/dfc/src/github.com/juju/juju/watcher/notify.go:71
# 0xd53b36 github.com/juju/juju/worker/catacomb.Invoke.func3+0x76 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:115
1 @ 0x437c2a 0x446482 0x445472 0x771a47 0x772422 0x7722ec 0x464e41
# 0x771a47 github.com/juju/juju/worker/peergrouper.(*machine).loop+0x207 /home/dfc/src/github.com/juju/juju/worker/peergrouper/worker.go:481
# 0x772422 github.com/juju/juju/worker/peergrouper.(*machine).(github.com/juju/juju/worker/peergrouper.loop)-fm+0x32 /home/dfc/src/github.com/juju/juju/worker/peergrouper/worker.go:475
# 0x7722ec github.com/juju/juju/worker/peergrouper.(*pgWorker).start.func1+0x4c /home/dfc/src/github.com/juju/juju/worker/peergrouper/worker.go:345
4 @ 0x437c2a 0x437cf3 0x410815 0x410490 0xe3932a 0xe3a302 0x464e41
# 0xe3932a github.com/juju/juju/api/watcher.(*machineAttachmentsWatcher).loop+0x23a /home/dfc/src/github.com/juju/juju/api/watcher/watcher.go:371
# 0xe3a302 github.com/juju/juju/api/watcher.newMachineStorageIdsWatcher.func1+0x82 /home/dfc/src/github.com/juju/juju/api/watcher/watcher.go:340
10 @ 0x437c2a 0x446482 0x445472 0x713c47 0x737460 0x464e41
# 0x713c47 github.com/juju/juju/state.(*machineAddressesWatcher).loop+0x5d7 /home/dfc/src/github.com/juju/juju/state/watcher.go:1807
# 0x737460 github.com/juju/juju/state.newMachineAddressesWatcher.func1+0x80 /home/dfc/src/github.com/juju/juju/state/watcher.go:1784
10 @ 0x437c2a 0x446482 0x445472 0x71b327 0x737d20 0x464e41
# 0x71b327 github.com/juju/juju/state.(*rebootWatcher).loop+0x417 /home/dfc/src/github.com/juju/juju/state/watcher.go:2658
# 0x737d20 github.com/juju/juju/state.newRebootWatcher.func1+0x80 /home/dfc/src/github.com/juju/juju/state/watcher.go:2631
5 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xd53565 0xa1b039 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xd53565 github.com/juju/juju/worker/catacomb.(*Catacomb).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:202
# 0xa1b039 github.com/juju/juju/watcher.(*NotifyWorker).Wait+0x39 /home/dfc/src/github.com/juju/juju/watcher/notify.go:138
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
1 @ 0x437c2a 0x446482 0x445472 0x79abc5 0xa1ee5f 0x464e41
# 0x79abc5 github.com/juju/juju/worker/logsender.New.func1+0x575 /home/dfc/src/github.com/juju/juju/worker/logsender/worker.go:31
# 0xa1ee5f github.com/juju/juju/worker.NewSimpleWorker.func1+0x5f /home/dfc/src/github.com/juju/juju/worker/simpleworker.go:20
19 @ 0x437c2a 0x432b27 0x4321ee 0x495960 0x4959cc 0x497489 0x4aae36 0x83c3a3 0x83cb08 0x840256 0x8a00b6 0x8de3fd 0x8dec22 0xdee5ad 0xdf4bc7 0x9ec248 0xdf3a1d 0xdf37a1 0x9e1126 0x100c3f9 0x8a8781 0x8a56d0 0x464e41
# 0x4321ee net.runtime_pollWait+0x5e /home/dfc/go/src/runtime/netpoll.go:160
# 0x495960 net.(*pollDesc).Wait+0x40 /home/dfc/go/src/net/fd_poll_runtime.go:73
# 0x4959cc net.(*pollDesc).WaitRead+0x3c /home/dfc/go/src/net/fd_poll_runtime.go:78
# 0x497489 net.(*netFD).Read+0x1c9 /home/dfc/go/src/net/fd_unix.go:250
# 0x4aae36 net.(*conn).Read+0x86 /home/dfc/go/src/net/net.go:172
# 0x83c3a3 crypto/tls.(*block).readFromUntil+0xb3 /home/dfc/go/src/crypto/tls/conn.go:460
# 0x83cb08 crypto/tls.(*Conn).readRecord+0x1e8 /home/dfc/go/src/crypto/tls/conn.go:562
# 0x840256 crypto/tls.(*Conn).Read+0x106 /home/dfc/go/src/crypto/tls/conn.go:939
# 0x8a00b6 net/http.(*connReader).Read+0x196 /home/dfc/go/src/net/http/server.go:526
# 0x8de3fd bufio.(*Reader).fill+0x15d /home/dfc/go/src/bufio/bufio.go:97
# 0x8dec22 bufio.(*Reader).ReadByte+0x82 /home/dfc/go/src/bufio/bufio.go:229
# 0xdee5ad golang.org/x/net/websocket.hybiFrameReaderFactory.NewFrameReader+0x9d /home/dfc/src/golang.org/x/net/websocket/hybi.go:126
# 0xdf4bc7 golang.org/x/net/websocket.Codec.Receive+0x147 /home/dfc/src/golang.org/x/net/websocket/websocket.go:315
# 0x9ec248 github.com/juju/juju/apiserver.(*logSinkHandler).ServeHTTP.func1+0x348 /home/dfc/src/github.com/juju/juju/apiserver/logsink.go:82
# 0xdf3a1d golang.org/x/net/websocket.Server.serveWebSocket+0x26d /home/dfc/src/golang.org/x/net/websocket/server.go:89
# 0xdf37a1 golang.org/x/net/websocket.Server.ServeHTTP+0x61 /home/dfc/src/golang.org/x/net/websocket/server.go:70
# 0x9e1126 github.com/juju/juju/apiserver.(*logSinkHandler).ServeHTTP+0xe6 /home/dfc/src/github.com/juju/juju/apiserver/logsink.go:104
# 0x100c3f9 github.com/bmizerany/pat.(*PatternServeMux).ServeHTTP+0x199 /home/dfc/src/github.com/bmizerany/pat/mux.go:109
# 0x8a8781 net/http.serverHandler.ServeHTTP+0x131 /home/dfc/go/src/net/http/server.go:2079
# 0x8a56d0 net/http.(*conn).serve+0xbf0 /home/dfc/go/src/net/http/server.go:1470
5 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa1c6c5 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa1c6c5 github.com/juju/juju/worker.(*periodicWorker).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/periodicworker.go:106
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
2 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xd53565 0xa80aa5 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xd53565 github.com/juju/juju/worker/catacomb.(*Catacomb).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:202
# 0xa80aa5 github.com/juju/juju/worker/storageprovisioner.(*storageProvisioner).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/storageprovisioner/storageprovisioner.go:208
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
2 @ 0x437c2a 0x446482 0x445472 0xa81d02 0xa907a2 0xd53b36 0x464e41
# 0xa81d02 github.com/juju/juju/worker/storageprovisioner.(*storageProvisioner).loop+0x1232 /home/dfc/src/github.com/juju/juju/worker/storageprovisioner/storageprovisioner.go:309
# 0xa907a2 github.com/juju/juju/worker/storageprovisioner.(*storageProvisioner).(github.com/juju/juju/worker/storageprovisioner.loop)-fm+0x32 /home/dfc/src/github.com/juju/juju/worker/storageprovisioner/storageprovisioner.go:188
# 0xd53b36 github.com/juju/juju/worker/catacomb.Invoke.func3+0x76 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:115
5 @ 0x437c2a 0x446482 0x445472 0xa1c631 0xa1ed3d 0x464e41
# 0xa1c631 github.com/juju/juju/worker.(*periodicWorker).run+0x261 /home/dfc/src/github.com/juju/juju/worker/periodicworker.go:82
# 0xa1ed3d github.com/juju/juju/worker.NewPeriodicWorker.func1+0x5d /home/dfc/src/github.com/juju/juju/worker/periodicworker.go:73
10 @ 0x437c2a 0x446482 0x445472 0x71bd55 0x738000 0x464e41
# 0x71bd55 github.com/juju/juju/state.(*blockDevicesWatcher).loop+0x835 /home/dfc/src/github.com/juju/juju/state/watcher.go:2721
# 0x738000 github.com/juju/juju/state.newBlockDevicesWatcher.func1+0x80 /home/dfc/src/github.com/juju/juju/state/watcher.go:2694
83 @ 0x437c2a 0x437cf3 0x410815 0x410490 0x9e93b6 0x4628c1 0x7deaf7 0x7de17b 0x10361e9 0x9e2dee 0xe0382c 0x464e41
# 0x9e93b6 github.com/juju/juju/apiserver.(*srvStringsWatcher).Next+0xa6 /home/dfc/src/github.com/juju/juju/apiserver/watcher.go:170
# 0x4628c1 runtime.call128+0x51 /home/dfc/go/src/runtime/asm_amd64.s:474
# 0x7deaf7 reflect.Value.call+0x837 /home/dfc/go/src/reflect/value.go:435
# 0x7de17b reflect.Value.Call+0xbb /home/dfc/go/src/reflect/value.go:303
# 0x10361e9 github.com/juju/juju/rpc/rpcreflect.newMethod.func6+0x129 /home/dfc/src/github.com/juju/juju/rpc/rpcreflect/type.go:323
# 0x9e2dee github.com/juju/juju/apiserver.(*srvCaller).Call+0x11e /home/dfc/src/github.com/juju/juju/apiserver/root.go:123
# 0xe0382c github.com/juju/juju/rpc.(*Conn).runRequest+0xcc /home/dfc/src/github.com/juju/juju/rpc/server.go:555
22 @ 0x437c2a 0x437cf3 0x410815 0x410490 0x9e9f56 0x4628c1 0x7deaf7 0x7de17b 0x10361e9 0x9e2dee 0xe0382c 0x464e41
# 0x9e9f56 github.com/juju/juju/apiserver.(*srvMachineStorageIdsWatcher).Next+0xa6 /home/dfc/src/github.com/juju/juju/apiserver/watcher.go:289
# 0x4628c1 runtime.call128+0x51 /home/dfc/go/src/runtime/asm_amd64.s:474
# 0x7deaf7 reflect.Value.call+0x837 /home/dfc/go/src/reflect/value.go:435
# 0x7de17b reflect.Value.Call+0xbb /home/dfc/go/src/reflect/value.go:303
# 0x10361e9 github.com/juju/juju/rpc/rpcreflect.newMethod.func6+0x129 /home/dfc/src/github.com/juju/juju/rpc/rpcreflect/type.go:323
# 0x9e2dee github.com/juju/juju/apiserver.(*srvCaller).Call+0x11e /home/dfc/src/github.com/juju/juju/apiserver/root.go:123
# 0xe0382c github.com/juju/juju/rpc.(*Conn).runRequest+0xcc /home/dfc/src/github.com/juju/juju/rpc/server.go:555
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa1ccd5 0x596f4e 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa1ccd5 github.com/juju/juju/worker.(*runner).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/runner.go:128
# 0x596f4e github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).startEnvWorkers.func2+0x2e /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:1178
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0x10623f5 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0x10623f5 github.com/juju/juju/watcher/legacy.(*stringsWorker).Wait+0x35 /home/dfc/src/github.com/juju/juju/watcher/legacy/stringsworker.go:59
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xd53565 0xa598e9 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xd53565 github.com/juju/juju/worker/catacomb.(*Catacomb).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:202
# 0xa598e9 github.com/juju/juju/worker/provisioner.(*provisioner).Wait+0x39 /home/dfc/src/github.com/juju/juju/worker/provisioner/provisioner.go:110
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
1 @ 0x437c2a 0x446482 0x445472 0xa5ab12 0xa65e12 0xd53b36 0x464e41
# 0xa5ab12 github.com/juju/juju/worker/provisioner.(*environProvisioner).loop+0x6c2 /home/dfc/src/github.com/juju/juju/worker/provisioner/provisioner.go:223
# 0xa65e12 github.com/juju/juju/worker/provisioner.(*environProvisioner).(github.com/juju/juju/worker/provisioner.loop)-fm+0x32 /home/dfc/src/github.com/juju/juju/worker/provisioner/provisioner.go:185
# 0xd53b36 github.com/juju/juju/worker/catacomb.Invoke.func3+0x76 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:115
1 @ 0x437c2a 0x446482 0x445472 0x10627db 0x10628b9 0x464e41
# 0x10627db github.com/juju/juju/watcher/legacy.(*stringsWorker).loop+0x3bb /home/dfc/src/github.com/juju/juju/watcher/legacy/stringsworker.go:75
# 0x10628b9 github.com/juju/juju/watcher/legacy.NewStringsWorker.func1+0x49 /home/dfc/src/github.com/juju/juju/watcher/legacy/stringsworker.go:47
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa26925 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa26925 github.com/juju/juju/worker/charmrevision.(*revisionUpdateWorker).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/charmrevision/worker.go:105
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
1 @ 0x437c2a 0x446482 0x445472 0xa2688c 0xa26999 0x464e41
# 0xa2688c github.com/juju/juju/worker/charmrevision.(*revisionUpdateWorker).loop+0x1bc /home/dfc/src/github.com/juju/juju/worker/charmrevision/worker.go:85
# 0xa26999 github.com/juju/juju/worker/charmrevision.NewWorker.func1+0x49 /home/dfc/src/github.com/juju/juju/worker/charmrevision/worker.go:72
1 @ 0x437c2a 0x446482 0x445472 0x707e1b 0x736640 0x464e41
# 0x707e1b github.com/juju/juju/state.(*minUnitsWatcher).loop+0x51b /home/dfc/src/github.com/juju/juju/state/watcher.go:649
# 0x736640 github.com/juju/juju/state.newMinUnitsWatcher.func1+0x80 /home/dfc/src/github.com/juju/juju/state/watcher.go:594
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xd53565 0xa45529 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xd53565 github.com/juju/juju/worker/catacomb.(*Catacomb).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:202
# 0xa45529 github.com/juju/juju/worker/instancepoller.(*updaterWorker).Wait+0x39 /home/dfc/src/github.com/juju/juju/worker/instancepoller/worker.go:47
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
1 @ 0x437c2a 0x446482 0x445472 0xa42eba 0xa45857 0xa45fb2 0xd53b36 0x464e41
# 0xa42eba github.com/juju/juju/worker/instancepoller.watchMachinesLoop+0x54a /home/dfc/src/github.com/juju/juju/worker/instancepoller/updater.go:105
# 0xa45857 github.com/juju/juju/worker/instancepoller.(*updaterWorker).loop+0x307 /home/dfc/src/github.com/juju/juju/worker/instancepoller/worker.go:75
# 0xa45fb2 github.com/juju/juju/worker/instancepoller.(*updaterWorker).(github.com/juju/juju/worker/instancepoller.loop)-fm+0x32 /home/dfc/src/github.com/juju/juju/worker/instancepoller/worker.go:32
# 0xd53b36 github.com/juju/juju/worker/catacomb.Invoke.func3+0x76 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:115
2 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xd53565 0xa1b899 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xd53565 github.com/juju/juju/worker/catacomb.(*Catacomb).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:202
# 0xa1b899 github.com/juju/juju/watcher.(*StringsWorker).Wait+0x39 /home/dfc/src/github.com/juju/juju/watcher/strings.go:141
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
3 @ 0x437c2a 0x446482 0x445472 0xa1b5fb 0xa1b952 0xd53b36 0x464e41
# 0xa1b5fb github.com/juju/juju/watcher.(*StringsWorker).loop+0x29b /home/dfc/src/github.com/juju/juju/watcher/strings.go:91
# 0xa1b952 github.com/juju/juju/watcher.(*StringsWorker).(github.com/juju/juju/watcher.loop)-fm+0x32 /home/dfc/src/github.com/juju/juju/watcher/strings.go:72
# 0xd53b36 github.com/juju/juju/worker/catacomb.Invoke.func3+0x76 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:115
1 @ 0x437c2a 0x446482 0x445472 0x71425a 0x737520 0x464e41
# 0x71425a github.com/juju/juju/state.(*cleanupWatcher).loop+0x41a /home/dfc/src/github.com/juju/juju/state/watcher.go:1865
# 0x737520 github.com/juju/juju/state.newCleanupWatcher.func1+0x80 /home/dfc/src/github.com/juju/juju/state/watcher.go:1848
9 @ 0x437c2a 0x446482 0x445472 0x71723a 0x73775e 0x464e41
# 0x71723a github.com/juju/juju/state.(*collectionWatcher).loop+0x54a /home/dfc/src/github.com/juju/juju/state/watcher.go:2154
# 0x73775e github.com/juju/juju/state.newcollectionWatcher.func2+0xae /home/dfc/src/github.com/juju/juju/state/watcher.go:2125
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xd53565 0xa3ef85 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xd53565 github.com/juju/juju/worker/catacomb.(*Catacomb).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:202
# 0xa3ef85 github.com/juju/juju/worker/firewaller.(*Firewaller).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/firewaller/firewaller.go:737
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
1 @ 0x437c2a 0x446482 0x445472 0x106349f 0x10638f2 0xd53b36 0x464e41
# 0x106349f github.com/juju/juju/worker/environ.(*Tracker).loop+0x40f /home/dfc/src/github.com/juju/juju/worker/environ/environ.go:100
# 0x10638f2 github.com/juju/juju/worker/environ.(*Tracker).(github.com/juju/juju/worker/environ.loop)-fm+0x32 /home/dfc/src/github.com/juju/juju/worker/environ/environ.go:76
# 0xd53b36 github.com/juju/juju/worker/catacomb.Invoke.func3+0x76 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:115
1 @ 0x437c2a 0x446482 0x445472 0xa42506 0xa45d62 0x464e41
# 0xa42506 github.com/juju/juju/worker/instancepoller.(*aggregator).loop+0x826 /home/dfc/src/github.com/juju/juju/worker/instancepoller/aggregate.go:74
# 0xa45d62 github.com/juju/juju/worker/instancepoller.newAggregator.func1+0x52 /home/dfc/src/github.com/juju/juju/worker/instancepoller/aggregate.go:34
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa1ccd5 0xa36da7 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa1ccd5 github.com/juju/juju/worker.(*runner).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/runner.go:128
# 0xa36da7 github.com/juju/juju/worker/envworkermanager.(*envWorkerManager).envIsAlive.func1.2+0x27 /home/dfc/src/github.com/juju/juju/worker/envworkermanager/envworkermanager.go:147
1 @ 0x437c2a 0x446482 0x445472 0xa3933f 0xa401c2 0xd53b36 0x464e41
# 0xa3933f github.com/juju/juju/worker/firewaller.(*Firewaller).loop+0xabf /home/dfc/src/github.com/juju/juju/worker/firewaller/firewaller.go:126
# 0xa401c2 github.com/juju/juju/worker/firewaller.(*Firewaller).(github.com/juju/juju/worker/firewaller.loop)-fm+0x32 /home/dfc/src/github.com/juju/juju/worker/firewaller/firewaller.go:60
# 0xd53b36 github.com/juju/juju/worker/catacomb.Invoke.func3+0x76 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:115
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xd53565 0x1063539 0xd53bed 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xd53565 github.com/juju/juju/worker/catacomb.(*Catacomb).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:202
# 0x1063539 github.com/juju/juju/worker/environ.(*Tracker).Wait+0x39 /home/dfc/src/github.com/juju/juju/worker/environ/environ.go:126
# 0xd53bed github.com/juju/juju/worker/catacomb.(*Catacomb).add.func1+0x7d /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:175
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa42949 0xd53bed 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa42949 github.com/juju/juju/worker/instancepoller.(*aggregator).Wait+0x39 /home/dfc/src/github.com/juju/juju/worker/instancepoller/aggregate.go:128
# 0xd53bed github.com/juju/juju/worker/catacomb.(*Catacomb).add.func1+0x7d /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:175
10 @ 0x437c2a 0x446482 0x445472 0xa4388c 0xa43461 0x464e41
# 0xa4388c github.com/juju/juju/worker/instancepoller.machineLoop+0x3dc /home/dfc/src/github.com/juju/juju/worker/instancepoller/updater.go:221
# 0xa43461 github.com/juju/juju/worker/instancepoller.runMachine+0x91 /home/dfc/src/github.com/juju/juju/worker/instancepoller/updater.go:175
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xa6b3a5 0xa1eaa1 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xa6b3a5 github.com/juju/juju/worker/resumer.(*Resumer).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/resumer/resumer.go:59
# 0xa1eaa1 github.com/juju/juju/worker.(*runner).runWorker+0x691 /home/dfc/src/github.com/juju/juju/worker/runner.go:279
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xd53565 0xa1b899 0x5989b7 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xd53565 github.com/juju/juju/worker/catacomb.(*Catacomb).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:202
# 0xa1b899 github.com/juju/juju/watcher.(*StringsWorker).Wait+0x39 /home/dfc/src/github.com/juju/juju/watcher/strings.go:141
# 0x5989b7 github.com/juju/juju/cmd/jujud/agent.(*MachineAgent).upgradeWaiterWorker.func1.1+0x27 /home/dfc/src/github.com/juju/juju/cmd/jujud/agent/machine.go:1829
1 @ 0x437c2a 0x446482 0x445472 0xa6b5d5 0xa6b639 0x464e41
# 0xa6b5d5 github.com/juju/juju/worker/resumer.(*Resumer).loop+0x205 /home/dfc/src/github.com/juju/juju/worker/resumer/resumer.go:64
# 0xa6b639 github.com/juju/juju/worker/resumer.NewResumer.func1+0x49 /home/dfc/src/github.com/juju/juju/worker/resumer/resumer.go:40
20 @ 0x437c2a 0x446482 0x445472 0x713377 0x737140 0x464e41
# 0x713377 github.com/juju/juju/state.(*machineUnitsWatcher).loop+0x807 /home/dfc/src/github.com/juju/juju/state/watcher.go:1729
# 0x737140 github.com/juju/juju/state.newMachineUnitsWatcher.func1+0x80 /home/dfc/src/github.com/juju/juju/state/watcher.go:1634
1 @ 0x437c2a 0x446482 0x445472 0x66c9a4 0x726ca9 0x464e41
# 0x66c9a4 github.com/juju/juju/state.(*storeManager).loop+0x3a4 /home/dfc/src/github.com/juju/juju/state/multiwatcher.go:188
# 0x726ca9 github.com/juju/juju/state.newStoreManager.func1+0x59 /home/dfc/src/github.com/juju/juju/state/multiwatcher.go:161
1 @ 0x437c2a 0x446482 0x445472 0x71a5e5 0x737c60 0x464e41
# 0x71a5e5 github.com/juju/juju/state.(*openedPortsWatcher).loop+0x515 /home/dfc/src/github.com/juju/juju/state/watcher.go:2550
# 0x737c60 github.com/juju/juju/state.newOpenedPortsWatcher.func1+0x80 /home/dfc/src/github.com/juju/juju/state/watcher.go:2493
1 @ 0x437c2a 0x446482 0x445472 0xf4dd84 0xf5f177 0x464e41
# 0xf4dd84 github.com/juju/juju/apiserver/provisioner.(*machineErrorRetry).loop+0x194 /home/dfc/src/github.com/juju/juju/apiserver/provisioner/machineerror.go:70
# 0xf5f177 github.com/juju/juju/apiserver/provisioner.newWatchMachineErrorRetry.func1+0x77 /home/dfc/src/github.com/juju/juju/apiserver/provisioner/machineerror.go:28
10 @ 0x437c2a 0x446482 0x445472 0xa3f46f 0xa40234 0xd53b36 0x464e41
# 0xa3f46f github.com/juju/juju/worker/firewaller.(*machineData).watchLoop+0x43f /home/dfc/src/github.com/juju/juju/worker/firewaller/firewaller.go:767
# 0xa40234 github.com/juju/juju/worker/firewaller.(*Firewaller).startMachine.func1+0x44 /home/dfc/src/github.com/juju/juju/worker/firewaller/firewaller.go:243
# 0xd53b36 github.com/juju/juju/worker/catacomb.Invoke.func3+0x76 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:115
1 @ 0x437c2a 0x446482 0x445472 0xa5cae3 0xa65ed2 0xd53b36 0x464e41
# 0xa5cae3 github.com/juju/juju/worker/provisioner.(*provisionerTask).loop+0x5e3 /home/dfc/src/github.com/juju/juju/worker/provisioner/provisioner_task.go:149
# 0xa65ed2 github.com/juju/juju/worker/provisioner.(*provisionerTask).(github.com/juju/juju/worker/provisioner.loop)-fm+0x32 /home/dfc/src/github.com/juju/juju/worker/provisioner/provisioner_task.go:98
# 0xd53b36 github.com/juju/juju/worker/catacomb.Invoke.func3+0x76 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:115
1 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xd53565 0xa5c4d9 0xd53bed 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xd53565 github.com/juju/juju/worker/catacomb.(*Catacomb).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:202
# 0xa5c4d9 github.com/juju/juju/worker/provisioner.(*provisionerTask).Wait+0x39 /home/dfc/src/github.com/juju/juju/worker/provisioner/provisioner_task.go:134
# 0xd53bed github.com/juju/juju/worker/catacomb.(*Catacomb).add.func1+0x7d /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:175
7 @ 0x437c2a 0x446482 0x445472 0xa3fa45 0xa40299 0xd53b36 0x464e41
# 0xa3fa45 github.com/juju/juju/worker/firewaller.(*serviceData).watchLoop+0x515 /home/dfc/src/github.com/juju/juju/worker/firewaller/firewaller.go:827
# 0xa40299 github.com/juju/juju/worker/firewaller.(*Firewaller).startService.func1+0x39 /home/dfc/src/github.com/juju/juju/worker/firewaller/firewaller.go:321
# 0xd53b36 github.com/juju/juju/worker/catacomb.Invoke.func3+0x76 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:115
7 @ 0x437c2a 0x437cf3 0x410815 0x41045b 0x9c29ae 0xd53565 0xa3fae5 0xd53bed 0x464e41
# 0x9c29ae launchpad.net/tomb.(*Tomb).Wait+0x5e /home/dfc/src/launchpad.net/tomb/tomb.go:108
# 0xd53565 github.com/juju/juju/worker/catacomb.(*Catacomb).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:202
# 0xa3fae5 github.com/juju/juju/worker/firewaller.(*serviceData).Wait+0x35 /home/dfc/src/github.com/juju/juju/worker/firewaller/firewaller.go:865
# 0xd53bed github.com/juju/juju/worker/catacomb.(*Catacomb).add.func1+0x7d /home/dfc/src/github.com/juju/juju/worker/catacomb/catacomb.go:175
1 @ 0x437c2a 0x446482 0x445472 0xa1eae6 0x464e41
# 0xa1eae6 github.com/juju/juju/worker.(*runner).runWorker+0x6d6 /home/dfc/src/github.com/juju/juju/worker/runner.go:268
6 @ 0x437c2a 0x446482 0x445472 0x70a154 0x736700 0x464e41
# 0x70a154 github.com/juju/juju/state.(*RelationScopeWatcher).loop+0x674 /home/dfc/src/github.com/juju/juju/state/watcher.go:848
# 0x736700 github.com/juju/juju/state.newRelationScopeWatcher.func1+0x80 /home/dfc/src/github.com/juju/juju/state/watcher.go:755
6 @ 0x437c2a 0x446482 0x445472 0x70b95e 0x736877 0x464e41
# 0x70b95e github.com/juju/juju/state.(*relationUnitsWatcher).loop+0x67e /home/dfc/src/github.com/juju/juju/state/watcher.go:998
# 0x736877 github.com/juju/juju/state.newRelationUnitsWatcher.func1+0x47 /home/dfc/src/github.com/juju/juju/state/watcher.go:901
6 @ 0x437c2a 0x437cf3 0x410815 0x410490 0x9e986b 0x4628c1 0x7deaf7 0x7de17b 0x10361e9 0x9e2dee 0xe0382c 0x464e41
# 0x9e986b github.com/juju/juju/apiserver.(*srvRelationUnitsWatcher).Next+0xbb /home/dfc/src/github.com/juju/juju/apiserver/watcher.go:215
# 0x4628c1 runtime.call128+0x51 /home/dfc/go/src/runtime/asm_amd64.s:474
# 0x7deaf7 reflect.Value.call+0x837 /home/dfc/go/src/reflect/value.go:435
# 0x7de17b reflect.Value.Call+0xbb /home/dfc/go/src/reflect/value.go:303
# 0x10361e9 github.com/juju/juju/rpc/rpcreflect.newMethod.func6+0x129 /home/dfc/src/github.com/juju/juju/rpc/rpcreflect/type.go:323
# 0x9e2dee github.com/juju/juju/apiserver.(*srvCaller).Call+0x11e /home/dfc/src/github.com/juju/juju/apiserver/root.go:123
# 0xe0382c github.com/juju/juju/rpc.(*Conn).runRequest+0xcc /home/dfc/src/github.com/juju/juju/rpc/server.go:555
2 @ 0x437c2a 0x446482 0x445472 0xb564aa 0xb58dc4 0x6499e7 0x15525ca 0x46280e 0x7deaf7 0x7de17b 0x10361e9 0x9e2dee 0xe0382c 0x464e41
# 0xb564aa github.com/juju/juju/worker/lease.block.invoke+0x19a /home/dfc/src/github.com/juju/juju/worker/lease/block.go:18
# 0xb58dc4 github.com/juju/juju/worker/lease.(*Manager).WaitUntilExpired+0x1f4 /home/dfc/src/github.com/juju/juju/worker/lease/manager.go:201
# 0x6499e7 github.com/juju/juju/state.leadershipClaimer.BlockUntilLeadershipReleased+0x47 /home/dfc/src/github.com/juju/juju/state/leadership.go:154
# 0x15525ca github.com/juju/juju/apiserver/leadership.(*leadershipService).BlockUntilLeadershipReleased+0x8a /home/dfc/src/github.com/juju/juju/apiserver/leadership/leadership.go:117
# 0x46280e runtime.call64+0x3e /home/dfc/go/src/runtime/asm_amd64.s:473
# 0x7deaf7 reflect.Value.call+0x837 /home/dfc/go/src/reflect/value.go:435
# 0x7de17b reflect.Value.Call+0xbb /home/dfc/go/src/reflect/value.go:303
# 0x10361e9 github.com/juju/juju/rpc/rpcreflect.newMethod.func6+0x129 /home/dfc/src/github.com/juju/juju/rpc/rpcreflect/type.go:323
# 0x9e2dee github.com/juju/juju/apiserver.(*srvCaller).Call+0x11e /home/dfc/src/github.com/juju/juju/apiserver/root.go:123
# 0xe0382c github.com/juju/juju/rpc.(*Conn).runRequest+0xcc /home/dfc/src/github.com/juju/juju/rpc/server.go:555
|