Optofidelity: Graceful exit upon keyboard interrupts

Make sure that devices get back into a good state when a keyboard
interrupt is received.

BUG=chromium:395174
TEST=None

Change-Id: I90a14b1674418e5a22936d3e5b638e4146e88841
Reviewed-on: https://chromium-review.googlesource.com/233933
Reviewed-by: Dennis Kempin <denniskempin@chromium.org>
Commit-Queue: Dennis Kempin <denniskempin@chromium.org>
Tested-by: Dennis Kempin <denniskempin@chromium.org>
diff --git a/optofidelity/optofidelity/detection/processor.py b/optofidelity/optofidelity/detection/processor.py
index c372ae6..6ed3672 100644
--- a/optofidelity/optofidelity/detection/processor.py
+++ b/optofidelity/optofidelity/detection/processor.py
@@ -7,6 +7,8 @@
 import gc
 import multiprocessing
 import numpy as np
+import os
+import signal
 import sys
 import time
 
@@ -274,6 +276,9 @@
     self.processor = processor
 
   def run(self):
+    # Ignore keyboard interrupts. The parent process is killing all subprocess.
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
     debugger = ProcessorDebugger(None)
 
     while True:
@@ -321,54 +326,65 @@
     frame_queue = multiprocessing.Queue()
     data_queue = multiprocessing.Queue()
     processes = []
-    for i in range(self.num_processes):
-      process = PreprocessWorker(self, buffer_list, frame_queue, data_queue)
-      process.start()
-      processes.append(process)
 
-    # Read frames into shared memory buffer and create jobs to process them.
-    num_frames = 0
-    current_idx = 0
-    prev_idx = None
+    try:
+      for i in range(self.num_processes):
+        process = PreprocessWorker(self, buffer_list, frame_queue, data_queue)
+        process.start()
+        processes.append(process)
 
-    for i, frame in self.ReadRelevantFrames(video, debug):
-      def visualize(waiting_on=""):
-        sys.stdout.write("\r")
-        for b in buffer_list:
-          sys.stdout.write("*" if b.done.is_set() else "-")
-        sys.stdout.write(" %d / %d" % (i, video.num_frames))
-        if debug["perf"] and waiting_on:
-          sys.stdout.write(" (waiting on %s)" % waiting_on)
-        sys.stdout.flush()
-      visualize()
+      # Read frames into shared memory buffer and create jobs to process them.
+      num_frames = 0
+      current_idx = 0
+      prev_idx = None
 
-      # Wait for all operations accessing the current buffer to be done.
-      # We are waiting for next_idx too since it will use the current_idx
-      # as a prev_image.
-      next_idx = (current_idx + 1) % len(buffer_list)
-      while not buffer_list[current_idx].done.wait():
-        visualize("current buffer")
-      while not buffer_list[next_idx].done.wait():
-        visualize("next buffer")
+      for i, frame in self.ReadRelevantFrames(video, debug):
+        def visualize(waiting_on=""):
+          sys.stdout.write("\r")
+          for b in buffer_list:
+            sys.stdout.write("*" if b.done.is_set() else "-")
+          sys.stdout.write(" %d / %d" % (i, video.num_frames))
+          if debug["perf"] and waiting_on:
+            sys.stdout.write(" (waiting on %s)" % waiting_on)
+          sys.stdout.flush()
+        visualize()
 
-      # Write into current buffer and create a job to process it.
-      visualize("queue")
-      buffer_list[current_idx].image[:] = frame[:]
-      buffer_list[current_idx].done.clear()
-      frame_queue.put((i, current_idx, prev_idx))
+        # Wait for all operations accessing the current buffer to be done.
+        # We are waiting for next_idx too since it will use the current_idx
+        # as a prev_image.
+        next_idx = (current_idx + 1) % len(buffer_list)
+        while not buffer_list[current_idx].done.wait():
+          visualize("current buffer")
+        while not buffer_list[next_idx].done.wait():
+          visualize("next buffer")
 
-      # Update counters.
-      prev_idx = current_idx
-      current_idx = next_idx
-      num_frames += 1
-      visualize("phantom")
+        # Write into current buffer and create a job to process it.
+        visualize("queue")
+        buffer_list[current_idx].image[:] = frame[:]
+        buffer_list[current_idx].done.clear()
+        frame_queue.put((i, current_idx, prev_idx))
 
-    print
+        # Update counters.
+        prev_idx = current_idx
+        current_idx = next_idx
+        num_frames += 1
+        visualize("phantom")
+      print
 
-    # Terminate worker threads by passing None into the queue
-    for i in range(self.num_processes):
-      frame_queue.put((None, None, None))
-    frame_queue.close()
+      # Terminate worker threads by passing None into the queue
+      for i in range(self.num_processes):
+        frame_queue.put((None, None, None))
+      frame_queue.close()
+
+    except:
+      # Make sure all processes are killed in case of errors
+      print
+      print "Killing child processes"
+      for process in processes:
+        if process.pid:
+          os.kill(process.pid, signal.SIGKILL)
+      raise
+
 
     # Read pre-processing results and put them into a dict by frame index
     data_map = {}
diff --git a/optofidelity/optofidelity/test_runner.py b/optofidelity/optofidelity/test_runner.py
index 0162de7..3de9a54 100644
--- a/optofidelity/optofidelity/test_runner.py
+++ b/optofidelity/optofidelity/test_runner.py
@@ -155,6 +155,11 @@
         app_results = self._RunAppTests(results, dut, app, cases, debug, safe)
         dut.ExitApp()
       return results
+    except KeyboardInterrupt as e:
+      print "Keyboard interrupt received."
+      print "Resetting device before exiting app."
+      dut.Reset()
+      raise e
     except:
       traceback.print_exc()
       if not safe:
@@ -194,6 +199,8 @@
             case_results = self._RunTest(results, dut, app, case, debug)
             num_failures = 0
             break
+          except KeyboardInterrupt as e:
+            raise e
           except:
             num_failures += 1
             traceback.print_exc()