Live profiling stops updating at some point + Crashes on repeated viewer reconnects

[wzli]

In my use case, there is an algorithm that ends up stalling after running for a long time. I would like to use live-profiling to find the exact cause when the stall happens. It would be inconvenient to dump and post analyze since the stall happens sporadically, and restarting the run is relatively expensive.

The issue I'm facing with this tool is that the profiling updates stop after a while, even though the main program is running as normal. One work around I found was to run in remote-profile mode, where even if the profiling updates stop, I can restart it again by reconnecting the viewer. However, repeating doing this eventually causes the below exception.

Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 35, in serve_forever
self.dispatch_sockets()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 105, in dispatch_sockets
self.connected(sock)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/init.py", line 228, in connected
self._start_profiling()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 53, in _start_profiling
self.profile_periodically()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 57, in profile_periodically
self.dispatch_sockets(self.interval)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 105, in dispatch_sockets
self.connected(sock)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/init.py", line 228, in connected
self._start_profiling()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 53, in _start_profiling
self.profile_periodically()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 57, in profile_periodically
self.dispatch_sockets(self.interval)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 105, in dispatch_sockets
self.connected(sock)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/init.py", line 228, in connected
self._start_profiling()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 53, in _start_profiling
self.profile_periodically()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 57, in profile_periodically
self.dispatch_sockets(self.interval)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 105, in dispatch_sockets
self.connected(sock)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/init.py", line 228, in connected
self._start_profiling()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 53, in _start_profiling
self.profile_periodically()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 57, in profile_periodically
self.dispatch_sockets(self.interval)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 105, in dispatch_sockets
self.connected(sock)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/init.py", line 228, in connected
self._start_profiling()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 53, in _start_profiling
self.profile_periodically()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 57, in profile_periodically
self.dispatch_sockets(self.interval)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 105, in dispatch_sockets
self.connected(sock)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/init.py", line 228, in connected
self._start_profiling()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 53, in _start_profiling
self.profile_periodically()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 57, in profile_periodically
self.dispatch_sockets(self.interval)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 105, in dispatch_sockets
self.connected(sock)
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/init.py", line 228, in connected
self._start_profiling()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 53, in _start_profiling
self.profile_periodically()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/select.py", line 56, in profile_periodically
for __ in self.profiling():
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/remote/init.py", line 180, in profiling
self.profiler.stop()
File "/home/wzli/.local/lib/python3.5/site-packages/profiling/utils.py", line 63, in stop
raise RuntimeError('Not started')
RuntimeError: Not started