servo/drv/ina2xx: Retry reading from INA on error

When taking power measurements using CCD, the H1 could be busy when an
i2c request comes in. This results in a timeout error being thrown.

BUG=b:117604702
TEST=Ran day long power measurements using CCD. Noticed the timeouts
were retried successfully.

Change-Id: I7743f82c54d2a09fd06ac4d0a8f650a9d8e8591a
Reviewed-on: https://chromium-review.googlesource.com/1529168
Commit-Ready: Raul E Rangel <rrangel@chromium.org>
Tested-by: Raul E Rangel <rrangel@chromium.org>
Reviewed-by: Todd Broch <tbroch@chromium.org>
diff --git a/servo/drv/ina2xx.py b/servo/drv/ina2xx.py
index 0b2de37..4123dc5 100644
--- a/servo/drv/ina2xx.py
+++ b/servo/drv/ina2xx.py
@@ -7,8 +7,11 @@
   INA219
   INA231
 """
+import errno
 import logging
 import numpy
+import servo.stm32i2c
+import time
 
 import hw_driver
 import i2c_reg
@@ -157,9 +160,29 @@
   def _has_reg(self, reg):
     return reg in self.REG_IDX
 
-  def _read_reg(self, name):
+  def _read_reg(self, name, timeout_retries=10):
     """Read architected register and return value."""
-    return self._i2c_obj._read_reg(self._get_reg_idx(name))
+    last_exception = None
+    for i in range(0, timeout_retries):
+      if i > 0:
+        sleep_ms = i ** 2
+        self._logger.warning('Read timed out, trying again in %d ms', sleep_ms)
+        time.sleep(sleep_ms / 1000.0)
+
+      try:
+        return self._i2c_obj._read_reg(self._get_reg_idx(name))
+      except IOError as e:
+        if e.errno == errno.ETIMEDOUT:
+          last_exception = e
+        else:
+          raise
+      except servo.stm32i2c.Si2cError as e:
+        last_exception = e
+
+    if last_exception:
+      raise last_exception
+    else:
+      raise ValueError('timeout_retries must be > 0')
 
   def _write_reg(self, name, value):
     """Write architected register."""