goofy_presenter: relay DHCP boot options

When DUT is behind presenter NAT, we need to relay the DHCP boot options
including 'sname' and 'file' so the DUT can netboot from the correct
server.  However, netboot install behind a cros device will not works
since the cros device does not have the 'nf_conntrack_tftp' module, so
it can not pass the tftp data properly.

Also move some advanced networking function into py/test/network.py to
fix dependency check.

BUG=chrome-os-partner:35391
TEST=Setup a DUT device befind a ubuntu running goofy_presenter, the DUT
should be able to perform netboot install.

Change-Id: I77f9af1166b187174c0712428e566fb5f442d04e
Reviewed-on: https://chromium-review.googlesource.com/262540
Reviewed-by: Hung-Te Lin <hungte@chromium.org>
Commit-Queue: Wei-Ning Huang <wnhuang@chromium.org>
Tested-by: Wei-Ning Huang <wnhuang@chromium.org>
diff --git a/py/goofy/dhcp_manager.py b/py/goofy/dhcp_manager.py
index fc0464d..66ddf67 100755
--- a/py/goofy/dhcp_manager.py
+++ b/py/goofy/dhcp_manager.py
@@ -29,6 +29,7 @@
     ip_start: The start of DHCP IP range.
     ip_end: The end of DHCP IP range.
     lease_time: How long in seconds before the DHCP lease expires.
+    bootp: A tuple (ip, filename, hostname) specifying the boot parameters.
     on_add: Optional callback function that's called when a client is issued
       a new IP address. The callback takes the IP address as the only argument.
     on_old: Similar to on_add, but called when a client is fed an IP address
@@ -54,6 +55,7 @@
                ip_start='192.168.0.10',
                ip_end='192.168.0.20',
                lease_time=3600,
+               bootp=None,
                on_add=None,
                on_old=None,
                on_del=None):
@@ -63,6 +65,7 @@
     self._ip_start = ip_start
     self._ip_end = ip_end
     self._lease_time = lease_time
+    self._bootp = bootp
     self._rpc_server = None
     self._process = None
     self._dhcp_action = {'add': on_add, 'old': on_old, 'del': on_del}
@@ -91,16 +94,19 @@
     # Make sure the interface is up
     net_utils.SetEthernetIp(self._my_ip, self._interface, self._netmask)
     # Start dnsmasq and have it call back to us on any DHCP event.
-    self._process = process_utils.Spawn(
-        ['dnsmasq', '--keep-in-foreground',
-         '--dhcp-range', dhcp_range,
-         '--interface', self._interface,
-         '--bind-interfaces',
-         '--port', str(dns_port),
-         '--dhcp-leasefile=%s' % lease_file,
-         '--pid-file=%s' % pid_file,
-         '--dhcp-script', callback_file_symlink],
-        sudo=True, log=True)
+    cmd = ['dnsmasq',
+           '--no-daemon',
+           '--dhcp-range', dhcp_range,
+           '--interface', self._interface,
+           '--port', str(dns_port),
+           '--no-dhcp-interface=%s' % net_utils.GetDefaultGatewayInterface(),
+           '--dhcp-leasefile=%s' % lease_file,
+           '--pid-file=%s' % pid_file,
+           '--dhcp-script', callback_file_symlink]
+    if self._bootp:
+      cmd.append('--dhcp-boot=%s,%s,%s' %
+                 (self._bootp[1], self._bootp[2], self._bootp[0]))
+    self._process = process_utils.Spawn(cmd, sudo=True, log=True)
     # Make sure the IP address is set on the interface
     net_utils.SetEthernetIp(self._my_ip, self._interface, self._netmask,
                             force=True)
diff --git a/py/goofy/link_manager.py b/py/goofy/link_manager.py
index f16f64d..3395ce0 100644
--- a/py/goofy/link_manager.py
+++ b/py/goofy/link_manager.py
@@ -21,6 +21,7 @@
 from cros.factory.system import service_manager
 from cros.factory.test import factory
 from cros.factory.test import utils
+from cros.factory.test import network
 from cros.factory.utils.jsonrpc_utils import JSONRPCServer
 from cros.factory.utils.jsonrpc_utils import TimeoutJSONRPCTransport
 from cros.factory.utils import net_utils
@@ -339,9 +340,14 @@
                          service_manager.Status.START, 15)
       # Give shill some time to run DHCP
       time.sleep(3)
+
+    # Get bootp parameters from gateway DHCP server
+    default_iface = net_utils.GetDefaultGatewayInterface()
+    bootp_params = network.GetDHCPBootParameters(default_iface)
+
     # OK, shill has done its job now. Let's see what interfaces are not managed.
     intf_blacklist = self._GetDHCPInterfaceBlacklist()
-    intfs = [intf for intf in net_utils.GetUnmanagedEthernetInterfaces()
+    intfs = [intf for intf in network.GetUnmanagedEthernetInterfaces()
              if intf not in intf_blacklist]
 
     for intf in intfs:
@@ -359,6 +365,7 @@
           ip_start=str(network_cidr.SelectIP(2)),
           ip_end=str(network_cidr.SelectIP(-3)),
           lease_time=3600,
+          bootp=bootp_params,
           on_add=self.OnDHCPEvent,
           on_old=self.OnDHCPEvent)
       dhcp_server.StartDHCP()
diff --git a/py/test/network.py b/py/test/network.py
index 59be744..f6155a9 100644
--- a/py/test/network.py
+++ b/py/test/network.py
@@ -2,21 +2,37 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""Advanced Networking-related utilities."""
+"""Advanced Networking-related utilities.
 
-import os
+Utilities with more complex functionalities and required interaction with other
+system components.
+"""
+
 import logging
+import os
+from multiprocessing import pool
+import tempfile
+import time
 
+import dpkt
 import netifaces
 import pexpect
 
 import factory_common  # pylint: disable=W0611
 from cros.factory.test import factory
 from cros.factory.test.utils import FormatExceptionOnly
+from cros.factory.utils import file_utils
 from cros.factory.utils import net_utils
+from cros.factory.utils import process_utils
 from cros.factory.utils import sync_utils
 from cros.factory.utils import type_utils
 
+try:
+  import dbus
+  HAS_DBUS = True
+except ImportError:
+  HAS_DBUS = False
+
 
 INSERT_ETHERNET_DONGLE_TIMEOUT = 30
 
@@ -66,8 +82,7 @@
       pexpect.spawn(
           'dhclient',
           ['-sf', DHCLIENT_SCRIPT, '-lf', DHCLIENT_LEASE, '-d',
-           '-v', '--no-pid', interface], +arguments,
-          timeout=timeout))
+           '-v', '--no-pid', interface] + arguments, timeout))
   try:
     dhcp_process.expect(expect_str)
   except:
@@ -103,6 +118,33 @@
   _SendDhclientCommand(['-r'], interface)
 
 
+def RenewDhcpLease(interface, timeout=3):
+  """Renews DHCP lease on a network interface.
+
+  Runs dhclient to obtain a new DHCP lease on the given network interface.
+
+  Args:
+    interface: The name of the network interface.
+    timeout: Timeout for waiting DHCPOFFERS in seconds.
+
+  Returns:
+    True if a new lease is obtained; otherwise, False.
+  """
+  with file_utils.UnopenedTemporaryFile() as conf_file:
+    with open(conf_file, "w") as f:
+      f.write("timeout %d;" % timeout)
+    p = process_utils.Spawn(['dhclient', '-1', '-cf', conf_file, interface])
+    # Allow one second for dhclient to gracefully exit
+    deadline = time.time() + timeout + 1
+    while p.poll() is None:
+      if time.time() > deadline:
+        # Well, dhclient is ignoring the timeout value. Kill it.
+        p.terminate()
+        return False
+      time.sleep(0.1)
+  return p.returncode == 0
+
+
 def PrepareNetwork(ip, force_new_ip=False, on_waiting=None):
   """High-level API to prepare networking.
 
@@ -143,3 +185,94 @@
     exception_string = FormatExceptionOnly()
     factory.console.error('Unable to setup network: %s', exception_string)
   factory.console.info('Network prepared. IP: %r', net_utils.GetEthernetIp())
+
+
+def GetUnmanagedEthernetInterfaces():
+  """Gets a list of unmanaged Ethernet interfaces.
+
+  This method returns a list of network interfaces on which no DHCP server
+  could be found.
+
+  On CrOS devices, shill should take care of managing this, so we simply
+  find Ethernet interfaces without IP addresses assigned. On non-CrOS devices,
+  we try to renew DHCP lease with dhclient on each interface.
+
+  Returns:
+    A list of interface names.
+  """
+  def IsShillRunning():
+    try:
+      shill_status = process_utils.Spawn(['status', 'shill'], read_stdout=True,
+                                         sudo=True)
+      return (shill_status.returncode == 0 and
+              'running' in shill_status.stdout_data)
+    except OSError:
+      return False
+
+  def IsShillUsingDHCP(intf):
+    if HAS_DBUS:
+      bus = dbus.SystemBus()
+      dev = bus.get_object("org.chromium.flimflam", "/device/%s" % intf)
+      dev_intf = dbus.Interface(dev, "org.chromium.flimflam.Device")
+      properties = dev_intf.GetProperties()
+      for config in properties['IPConfigs']:
+        if 'dhcp' in config:
+          return True
+      return False
+    else:
+      # We can't talk to shill without DBus, so let's just check for IP
+      # address.
+      return net_utils.GetEthernetIp(intf) is not None
+
+  if IsShillRunning():
+    # 'shill' running. Let's not mess with it. Just check whether shill got
+    # DHCP response on each interface.
+    return [intf for intf in net_utils.GetEthernetInterfaces() if
+            not IsShillUsingDHCP(intf)]
+  else:
+    # 'shill' not running. Use dhclient.
+    p = pool.ThreadPool(5)
+    def CheckManaged(interface):
+      if RenewDhcpLease(interface):
+        return None
+      else:
+        return interface
+    managed = p.map(CheckManaged, net_utils.GetEthernetInterfaces())
+    return [x for x in managed if x]
+
+
+def GetDHCPBootParameters(interface):
+  """Get DHCP Bootp parameters from interface.
+
+  Args:
+    interface: the target interface managed by some DHCP server
+
+  Returns:
+    A tuple (ip, filename, hostname) if bootp parameter is found, else None.
+  """
+  dhcp_filter = '((port 67 or port 68) and (udp[8:1] = 0x2))'
+  _, dump_file = tempfile.mkstemp()
+  p = process_utils.Spawn("tcpdump -i %s -c 1 -w %s '%s'" %
+                          (interface, dump_file, dhcp_filter), shell=True)
+
+  # Send two renew requests to make sure tcmpdump can capture the response.
+  for _ in range(2):
+    if not RenewDhcpLease(interface):
+      return RuntimeError('can not find DHCP server on %s' % interface)
+    time.sleep(0.5)
+
+  p.wait()
+
+  with open(dump_file, 'r') as f:
+    pcap = dpkt.pcap.Reader(f)
+    for _, buf in pcap:
+      eth = dpkt.ethernet.Ethernet(buf)
+      udp = eth.ip.data
+      dhcp = dpkt.dhcp.DHCP(udp.data)
+
+      if dhcp['siaddr'] != 0 and len(dhcp['file'].strip('\x00')):
+        ip = '.'.join([str(ord(x)) for x in
+                       ('%x' % dhcp['siaddr']).decode('hex')])
+        return (ip, dhcp['file'].strip('\x00'), dhcp['sname'].strip('\x00'))
+
+  return None
diff --git a/py/tools/deps.conf b/py/tools/deps.conf
index bd2ac9b..b680e2f 100644
--- a/py/tools/deps.conf
+++ b/py/tools/deps.conf
@@ -55,6 +55,7 @@
   known_deps:  # Well-known external dependencies.
    - dbus
    - _dbus_bindings
+   - dpkt
    - jsonrpclib
    - netifaces
    - numpy
diff --git a/py/utils/net_utils.py b/py/utils/net_utils.py
index d41631d..f9dfcb8 100644
--- a/py/utils/net_utils.py
+++ b/py/utils/net_utils.py
@@ -9,7 +9,6 @@
 import glob
 import httplib
 import logging
-from multiprocessing import pool
 import os
 import re
 import socket
@@ -18,13 +17,6 @@
 import time
 import xmlrpclib
 
-try:
-  import dbus
-  HAS_DBUS = True
-except ImportError:
-  HAS_DBUS = False
-
-
 import factory_common  # pylint: disable=W0611
 from cros.factory.utils import file_utils
 from cros.factory.utils import process_utils
@@ -81,7 +73,7 @@
     return self._ip
 
   def __eq__(self, obj):
-    return self._ip == obj._ip
+    return self._ip == obj._ip  # pylint: disable=W0212
 
 
 class CIDR(object):
@@ -500,33 +492,6 @@
   file_utils.WriteFile('/proc/sys/net/ipv4/ip_forward', '1')
 
 
-def RenewDhcpLease(interface, timeout=3):
-  """Renews DHCP lease on a network interface.
-
-  Runs dhclient to obtain a new DHCP lease on the given network interface.
-
-  Args:
-    interface: The name of the network interface.
-    timeout: Timeout for waiting DHCPOFFERS in seconds.
-
-  Returns:
-    True if a new lease is obtained; otherwise, False.
-  """
-  with file_utils.UnopenedTemporaryFile() as conf_file:
-    with open(conf_file, "w") as f:
-      f.write("timeout %d;" % timeout)
-    p = process_utils.Spawn(['dhclient', '-1', '-cf', conf_file, interface])
-    # Allow one second for dhclient to gracefully exit
-    deadline = time.time() + timeout + 1
-    while p.poll() is None:
-      if time.time() > deadline:
-        # Well, dhclient is ignoring the timeout value. Kill it.
-        p.terminate()
-        return False
-      time.sleep(0.1)
-  return p.returncode == 0
-
-
 def GetDefaultGatewayInterface():
   """Return the default gateway interface.
 
@@ -544,60 +509,6 @@
     raise RuntimeError('no default gateway found')
 
 
-def GetUnmanagedEthernetInterfaces():
-  """Gets a list of unmanaged Ethernet interfaces.
-
-  This method returns a list of network interfaces on which no DHCP server
-  could be found.
-
-  On CrOS devices, shill should take care of managing this, so we simply
-  find Ethernet interfaces without IP addresses assigned. On non-CrOS devices,
-  we try to renew DHCP lease with dhclient on each interface.
-
-  Returns:
-    A list of interface names.
-  """
-  def IsShillRunning():
-    try:
-      shill_status = process_utils.Spawn(['status', 'shill'], read_stdout=True,
-                                         sudo=True)
-      return (shill_status.returncode == 0 and
-              'running' in shill_status.stdout_data)
-    except OSError:
-      return False
-
-  def IsShillUsingDHCP(intf):
-    if HAS_DBUS:
-      bus = dbus.SystemBus()
-      dev = bus.get_object("org.chromium.flimflam", "/device/%s" % intf)
-      dev_intf = dbus.Interface(dev, "org.chromium.flimflam.Device")
-      properties = dev_intf.GetProperties()
-      for config in properties['IPConfigs']:
-        if 'dhcp' in config:
-          return True
-      return False
-    else:
-      # We can't talk to shill without DBus, so let's just check for IP
-      # address.
-      return GetEthernetIp(intf) is not None
-
-  if IsShillRunning():
-    # 'shill' running. Let's not mess with it. Just check whether shill got
-    # DHCP response on each interface.
-    return [intf for intf in GetEthernetInterfaces() if
-            not IsShillUsingDHCP(intf)]
-  else:
-    # 'shill' not running. Use dhclient.
-    p = pool.ThreadPool(5)
-    def CheckManaged(interface):
-      if RenewDhcpLease(interface):
-        return None
-      else:
-        return interface
-    managed = p.map(CheckManaged, GetEthernetInterfaces())
-    return [x for x in managed if x]
-
-
 def GetUnusedIPV4RangeCIDR(preferred_prefix_bits=24, exclude_ip_prefix=None):
   """Find unused IP ranges in IPV4 private address space.
 
diff --git a/sh/start_presenter.sh b/sh/start_presenter.sh
index c38a9cc..3f738ea 100755
--- a/sh/start_presenter.sh
+++ b/sh/start_presenter.sh
@@ -21,6 +21,7 @@
   has_python_package "numpy" || echo "python-numpy package"
   has_python_package "jsonrpclib" || echo "python-jsonrpclib package"
   has_python_package "ws4py" || echo "python-ws4py package"
+  has_python_package "dpkt" || echo "python-dpkt package"
 }
 
 MISSING_PREREQUISITES="$(check_missing_prerequisites)"