tap-bridge.cc

Go to the documentation of this file.

/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
/*
 * Copyright (c) 2009 University of Washington
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation;
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include "tap-bridge.h"
#include "tap-encode-decode.h"

#include "ns3/node.h"
#include "ns3/channel.h"
#include "ns3/packet.h"
#include "ns3/ethernet-header.h"
#include "ns3/llc-snap-header.h"
#include "ns3/log.h"
#include "ns3/abort.h"
#include "ns3/boolean.h"
#include "ns3/string.h"
#include "ns3/enum.h"
#include "ns3/ipv4.h"
#include "ns3/simulator.h"
#include "ns3/realtime-simulator-impl.h"
#include "ns3/unix-fd-reader.h"
#include "ns3/uinteger.h"

#include <sys/wait.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/ioctl.h>
#include <net/if.h>
#include <cerrno>
#include <limits>
#include <cstdlib>
#include <unistd.h>

namespace ns3 {

NS_LOG_COMPONENT_DEFINE ("TapBridge");

FdReader::Data TapBridgeFdReader::DoRead (void)
{
  NS_LOG_FUNCTION_NOARGS ();

  uint32_t bufferSize = 65536;
  uint8_t *buf = (uint8_t *)std::malloc (bufferSize);
  NS_ABORT_MSG_IF (buf == 0, "malloc() failed");

  NS_LOG_LOGIC ("Calling read on tap device fd " << m_fd);
  ssize_t len = read (m_fd, buf, bufferSize);
  if (len <= 0)
    {
      NS_LOG_INFO ("TapBridgeFdReader::DoRead(): done");
      std::free (buf);
      buf = 0;
      len = 0;
    }

  return FdReader::Data (buf, len);
}

#define TAP_MAGIC 95549

NS_OBJECT_ENSURE_REGISTERED (TapBridge);

TypeId
TapBridge::GetTypeId (void)
{
  static TypeId tid = TypeId ("ns3::TapBridge")
    .SetParent<NetDevice> ()
    .SetGroupName ("TapBridge")
    .AddConstructor<TapBridge> ()
    .AddAttribute ("Mtu", "The MAC-level Maximum Transmission Unit",
                   UintegerValue (0),
                   MakeUintegerAccessor (&TapBridge::SetMtu,
                                         &TapBridge::GetMtu),
                   MakeUintegerChecker<uint16_t> ())
    .AddAttribute ("DeviceName", 
                   "The name of the tap device to create.",
                   StringValue (""),
                   MakeStringAccessor (&TapBridge::m_tapDeviceName),
                   MakeStringChecker ())
    .AddAttribute ("Gateway", 
                   "The IP address of the default gateway to assign to the host machine, when in ConfigureLocal mode.",
                   Ipv4AddressValue ("255.255.255.255"),
                   MakeIpv4AddressAccessor (&TapBridge::m_tapGateway),
                   MakeIpv4AddressChecker ())
    .AddAttribute ("IpAddress", 
                   "The IP address to assign to the tap device, when in ConfigureLocal mode.  "
                   "This address will override the discovered IP address of the simulated device.",
                   Ipv4AddressValue ("255.255.255.255"),
                   MakeIpv4AddressAccessor (&TapBridge::m_tapIp),
                   MakeIpv4AddressChecker ())
    .AddAttribute ("MacAddress", 
                   "The MAC address to assign to the tap device, when in ConfigureLocal mode.  "
                   "This address will override the discovered MAC address of the simulated device.",
                   Mac48AddressValue (Mac48Address ("ff:ff:ff:ff:ff:ff")),
                   MakeMac48AddressAccessor (&TapBridge::m_tapMac),
                   MakeMac48AddressChecker ())
    .AddAttribute ("Netmask", 
                   "The network mask to assign to the tap device, when in ConfigureLocal mode.  "
                   "This address will override the discovered MAC address of the simulated device.",
                   Ipv4MaskValue ("255.255.255.255"),
                   MakeIpv4MaskAccessor (&TapBridge::m_tapNetmask),
                   MakeIpv4MaskChecker ())
    .AddAttribute ("Start", 
                   "The simulation time at which to spin up the tap device read thread.",
                   TimeValue (Seconds (0.)),
                   MakeTimeAccessor (&TapBridge::m_tStart),
                   MakeTimeChecker ())
    .AddAttribute ("Stop", 
                   "The simulation time at which to tear down the tap device read thread.",
                   TimeValue (Seconds (0.)),
                   MakeTimeAccessor (&TapBridge::m_tStop),
                   MakeTimeChecker ())
    .AddAttribute ("Mode", 
                   "The operating and configuration mode to use.",
                   EnumValue (USE_LOCAL),
                   MakeEnumAccessor (&TapBridge::SetMode),
                   MakeEnumChecker (CONFIGURE_LOCAL, "ConfigureLocal",
                                    USE_LOCAL, "UseLocal",
                                    USE_BRIDGE, "UseBridge"))
  ;
  return tid;
}

TapBridge::TapBridge ()
  : m_node (0),
    m_ifIndex (0),
    m_sock (-1),
    m_startEvent (),
    m_stopEvent (),
    m_fdReader (0),
    m_ns3AddressRewritten (false)
{
  NS_LOG_FUNCTION_NOARGS ();
  m_packetBuffer = new uint8_t[65536];
  Start (m_tStart);
}

TapBridge::~TapBridge()
{
  NS_LOG_FUNCTION_NOARGS ();

  StopTapDevice ();

  delete [] m_packetBuffer;
  m_packetBuffer = 0;

  m_bridgedDevice = 0;
}

void
TapBridge::DoDispose ()
{
  NS_LOG_FUNCTION_NOARGS ();
  NetDevice::DoDispose ();
}

void
TapBridge::Start (Time tStart)
{
  NS_LOG_FUNCTION (tStart);

  //
  // Cancel any pending start event and schedule a new one at some relative time in the future.
  //
  Simulator::Cancel (m_startEvent);
  m_startEvent = Simulator::Schedule (tStart, &TapBridge::StartTapDevice, this);
}

void
TapBridge::Stop (Time tStop)
{
  NS_LOG_FUNCTION (tStop);
  //
  // Cancel any pending stop event and schedule a new one at some relative time in the future.
  //
  Simulator::Cancel (m_stopEvent);
  m_startEvent = Simulator::Schedule (tStop, &TapBridge::StopTapDevice, this);
}

void
TapBridge::StartTapDevice (void)
{
  NS_LOG_FUNCTION_NOARGS ();

  NS_ABORT_MSG_IF (m_sock != -1, "TapBridge::StartTapDevice(): Tap is already started");

  //
  // A similar story exists for the node ID.  We can't just naively do a
  // GetNode ()->GetId () since GetNode is going to give us a Ptr<Node> which
  // is reference counted.  We need to stash away the node ID for use in the
  // read thread.
  //
  m_nodeId = GetNode ()->GetId ();

  //
  // Spin up the tap bridge and start receiving packets.
  //
  NS_LOG_LOGIC ("Creating tap device");

  //
  // Call out to a separate process running as suid root in order to get the 
  // tap device allocated and set up.  We do this to avoid having the entire 
  // simulation running as root.  If this method returns, we'll have a socket
  // waiting for us in m_sock that we can use to talk to the newly created 
  // tap device.
  //
  CreateTap ();

  // Declare the link up
  NotifyLinkUp ();

  //
  // Now spin up a read thread to read packets from the tap device.
  //
  NS_ABORT_MSG_IF (m_fdReader != 0,"TapBridge::StartTapDevice(): Receive thread is already running");
  NS_LOG_LOGIC ("Spinning up read thread");

  m_fdReader = Create<TapBridgeFdReader> ();
  m_fdReader->Start (m_sock, MakeCallback (&TapBridge::ReadCallback, this));
}

void
TapBridge::StopTapDevice (void)
{
  NS_LOG_FUNCTION_NOARGS ();

  if (m_fdReader != 0)
    {
      m_fdReader->Stop ();
      m_fdReader = 0;
    }

  if (m_sock != -1)
    {
      close (m_sock);
      m_sock = -1;
    }
}

void
TapBridge::CreateTap (void)
{
  NS_LOG_FUNCTION_NOARGS ();

  // 
  // The TapBridge has three distinct operating modes.  At this point, the
  // differences revolve around who is responsible for creating and configuring
  // the underlying network tap that we use.  In ConfigureLocal mode, the 
  // TapBridge has the responsibility for creating and configuring the TAP.
  //
  // In UseBridge or UseLocal modes, the user will provide us a configuration
  // and we have to adapt to it.  For example, in UseLocal mode, the user will
  // be configuring a tap device outside the scope of the ns-3 simulation and
  // will be expecting us to work with it.  The user will do something like:
  //
  //   sudo tunctl -t tap0
  //   sudo ifconfig tap0 hw ether 00:00:00:00:00:01
  //   sudo ifconfig tap0 10.1.1.1 netmask 255.255.255.0 up
  //
  // The user will then set the "Mode" Attribute of the TapBridge to "UseLocal"
  // and the "DeviceName" Attribute to "tap0" in this case.
  //
  // In ConfigureLocal mode, the user is asking the TapBridge to do the 
  // configuration and create a TAP with the provided "DeviceName" with which 
  // the user can later do what she wants.  We need to extract values for the
  // MAC address, IP address, net mask, etc, from the simualtion itself and 
  // use them to initialize corresponding values on the created tap device.
  //
  // In UseBridge mode, the user is asking us to use an existing tap device
  // has been included in an OS bridge.  She is asking us to take the simulated
  // net device and logically add it to the existing bridge.  We expect that
  // the user has done something like:
  //
  //   sudo brctl addbr mybridge
  //   sudo tunctl -t mytap
  //   sudo ifconfig mytap hw ether 00:00:00:00:00:01
  //   sudo ifconfig mytap 0.0.0.0 up
  //   sudo brctl addif mybridge mytap
  //   sudo brctl addif mybridge ...
  //   sudo ifconfig mybridge 10.1.1.1 netmask 255.255.255.0 up
  //
  // The bottom line at this point is that we want to either create or use a 
  // tap device on the host based on the verb part "Use" or "Configure" of the 
  // operating mode.  Unfortunately for us you have to have root privileges to
  // do either.  Instead of running the entire simulation as root, we decided 
  // to make a small program who's whole reason for being is to run as suid 
  // root and do what it takes to create the tap.  We're just going to pass 
  // off the configuration information to that program and let it deal with
  // the situation.
  //
  // We're going to fork and exec that program soon, but first we need to have 
  // a socket to talk to it with.  So we create a local interprocess (Unix) 
  // socket for that purpose.
  //
  int sock = socket (PF_UNIX, SOCK_DGRAM, 0);
  NS_ABORT_MSG_IF (sock == -1, "TapBridge::CreateTap(): Unix socket creation error, errno = " << std::strerror (errno));

  //
  // Bind to that socket and let the kernel allocate an endpoint
  //
  struct sockaddr_un un;
  memset (&un, 0, sizeof (un));
  un.sun_family = AF_UNIX;
  int status = bind (sock, (struct sockaddr*)&un, sizeof (sa_family_t));
  NS_ABORT_MSG_IF (status == -1, "TapBridge::CreateTap(): Could not bind(): errno = " << std::strerror (errno));
  NS_LOG_INFO ("Created Unix socket");
  NS_LOG_INFO ("sun_family = " << un.sun_family);
  NS_LOG_INFO ("sun_path = " << un.sun_path);

  //
  // We have a socket here, but we want to get it there -- to the program we're
  // going to exec.  What we'll do is to do a getsockname and then encode the
  // resulting address information as a string, and then send the string to the
  // program as an argument.  So we need to get the sock name.
  //
  socklen_t len = sizeof (un);
  status = getsockname (sock, (struct sockaddr*)&un, &len);
  NS_ABORT_MSG_IF (status == -1, "TapBridge::CreateTap(): Could not getsockname(): errno = " << std::strerror (errno));

  //
  // Now encode that socket name (family and path) as a string of hex digits
  //
  std::string path = TapBufferToString ((uint8_t *)&un, len);
  NS_LOG_INFO ("Encoded Unix socket as \"" << path << "\"");

  //
  // Tom Goff reports the possiblility of a deadlock when trying to acquire the
  // python GIL here.  He says that this might be due to trying to access Python
  // objects after fork() without calling PyOS_AfterFork() to properly reset 
  // Python state (including the GIL).  Originally these next three lines were
  // done after the fork, but were moved here to work around the deadlock.
  //
  Ptr<NetDevice> nd = GetBridgedNetDevice ();
  Ptr<Node> n = nd->GetNode ();
  Ptr<Ipv4> ipv4 = n->GetObject<Ipv4> ();

  //
  // Fork and exec the process to create our socket.  If we're us (the parent)
  // we wait for the child (the creator) to complete and read the socket it 
  // created and passed back using the ancillary data mechanism.
  //
  pid_t pid = ::fork ();
  if (pid == 0)
    {
      NS_LOG_DEBUG ("Child process");

      //
      // build a command line argument from the encoded endpoint string that 
      // the socket creation process will use to figure out how to respond to
      // the (now) parent process.  We're going to have to give this program
      // quite a bit of information.
      //
      // -d<device-name> The name of the tap device we want to create;
      // -g<gateway-address> The IP address to use as the default gateway;
      // -i<IP-address> The IP address to assign to the new tap device;
      // -m<MAC-address> The MAC-48 address to assign to the new tap device;
      // -n<network-mask> The network mask to assign to the new tap device;
      // -o<operating mode> The operating mode of the bridge (1=ConfigureLocal, 2=UseLocal, 3=UseBridge)
      // -p<path> the path to the unix socket described above.
      //
      // Example tap-creator -dnewdev -g1.2.3.2 -i1.2.3.1 -m08:00:2e:00:01:23 -n255.255.255.0 -o1 -pblah
      //
      // We want to get as much of this stuff automagically as possible.
      //
      // For CONFIGURE_LOCAL mode only:
      // <IP-address> is the IP address we are going to set in the newly 
      // created Tap device on the Linux host.  At the point in the simulation
      // where devices are coming up, we should have all of our IP addresses
      // assigned.  That means that we can find the IP address to assign to 
      // the new Tap device from the IP address associated with the bridged
      // net device.
      //

      bool wantIp = (m_mode == CONFIGURE_LOCAL);

      if (wantIp
          && (ipv4 == 0)
          && m_tapIp.IsBroadcast ()
          && m_tapNetmask.IsEqual (Ipv4Mask::GetOnes ()))
        {
          NS_FATAL_ERROR ("TapBridge::CreateTap(): Tap device IP configuration requested but neither IP address nor IP netmask is provided");
        }

      // Some stub values to make tap-creator happy
      Ipv4Address ipv4Address ("255.255.255.255");
      Ipv4Mask ipv4Mask ("255.255.255.255");

      if (ipv4 != 0)
        {
          uint32_t index = ipv4->GetInterfaceForDevice (nd);
          if (ipv4->GetNAddresses (index) > 1)
            {
              NS_LOG_WARN ("Underlying bridged NetDevice has multiple IP addresses; using first one.");
            }
          ipv4Address = ipv4->GetAddress (index, 0).GetLocal ();

          //
          // The net mask is sitting right there next to the ipv4 address.
          //
          ipv4Mask = ipv4->GetAddress (index, 0).GetMask ();
        }

      //
      // The MAC address should also already be assigned and waiting for us in
      // the bridged net device.
      //
      Address address = nd->GetAddress ();
      Mac48Address mac48Address = Mac48Address::ConvertFrom (address);

      //
      // The device-name is something we may want the system to make up in 
      // every case.  We also rely on it being configured via an Attribute 
      // through the helper.  By default, it is set to the empty string 
      // which tells the system to make up a device name such as "tap123".
      //
      std::ostringstream ossDeviceName;
      ossDeviceName << "-d" << m_tapDeviceName;

      //
      // The gateway-address is something we can't derive, so we rely on it
      // being configured via an Attribute through the helper.
      //
      std::ostringstream ossGateway;
      ossGateway << "-g" << m_tapGateway;

      //
      // For flexibility, we do allow a client to override any of the values
      // above via attributes, so only use our found values if the Attribute
      // is not equal to its default value (empty string or broadcast address). 
      //
      std::ostringstream ossIp;
      if (m_tapIp.IsBroadcast ())
        {
          ossIp << "-i" << ipv4Address;
        }
      else
        {
          ossIp << "-i" << m_tapIp;
        }

      std::ostringstream ossMac;
      if (m_tapMac.IsBroadcast ())
        {
          ossMac << "-m" << mac48Address;
        }
      else
        {
          ossMac << "-m" << m_tapMac;
        }

      std::ostringstream ossNetmask;
      if (m_tapNetmask.IsEqual (Ipv4Mask::GetOnes ()))
        {
          ossNetmask << "-n" << ipv4Mask;
        }
      else
        {
          ossNetmask << "-n" << m_tapNetmask;
        }

      std::ostringstream ossMode;
      ossMode << "-o";
      if (m_mode == CONFIGURE_LOCAL)
        {
          ossMode << "1";
        }
      else if (m_mode == USE_LOCAL)
        {
          ossMode << "2";
        }
      else
        {
          ossMode << "3";
        }

      std::ostringstream ossPath;
      ossPath << "-p" << path;
      //
      // Execute the socket creation process image.
      //
      status = ::execlp (TAP_CREATOR, 
                         TAP_CREATOR,                       // argv[0] (filename)
                         ossDeviceName.str ().c_str (),       // argv[1] (-d<device name>)
                         ossGateway.str ().c_str (),          // argv[2] (-g<gateway>)
                         ossIp.str ().c_str (),               // argv[3] (-i<IP address>)
                         ossMac.str ().c_str (),              // argv[4] (-m<MAC address>)
                         ossNetmask.str ().c_str (),          // argv[5] (-n<net mask>)
                         ossMode.str ().c_str (),             // argv[6] (-o<operating mode>)
                         ossPath.str ().c_str (),             // argv[7] (-p<path>)
                         (char *)NULL);

      //
      // If the execlp successfully completes, it never returns.  If it returns it failed or the OS is
      // broken.  In either case, we bail.
      //
      NS_FATAL_ERROR ("TapBridge::CreateTap(): Back from execlp(), errno = " << ::strerror (errno));
    }
  else
    {
      NS_LOG_DEBUG ("Parent process");
      //
      // We're the process running the emu net device.  We need to wait for the
      // socket creator process to finish its job.
      //
      int st;
      pid_t waited = waitpid (pid, &st, 0);
      NS_ABORT_MSG_IF (waited == -1, "TapBridge::CreateTap(): waitpid() fails, errno = " << std::strerror (errno));
      NS_ASSERT_MSG (pid == waited, "TapBridge::CreateTap(): pid mismatch");

      //
      // Check to see if the socket creator exited normally and then take a 
      // look at the exit code.  If it bailed, so should we.  If it didn't
      // even exit normally, we bail too.
      //
      if (WIFEXITED (st))
        {
          int exitStatus = WEXITSTATUS (st);
          NS_ABORT_MSG_IF (exitStatus != 0, 
                           "TapBridge::CreateTap(): socket creator exited normally with status " << exitStatus);
        }
      else 
        {
          NS_FATAL_ERROR ("TapBridge::CreateTap(): socket creator exited abnormally");
        }

      //
      // At this point, the socket creator has run successfully and should 
      // have created our tap device, initialized it with the information we
      // passed and sent it back to the socket address we provided.  A socket
      // (fd) we can use to talk to this tap device should be waiting on the 
      // Unix socket we set up to receive information back from the creator
      // program.  We've got to do a bunch of grunt work to get at it, though.
      //
      // The struct iovec below is part of a scatter-gather list.  It describes a
      // buffer.  In this case, it describes a buffer (an integer) that will
      // get the data that comes back from the socket creator process.  It will
      // be a magic number that we use as a consistency/sanity check.
      // 
      struct iovec iov;
      uint32_t magic;
      iov.iov_base = &magic;
      iov.iov_len = sizeof(magic);

      //
      // The CMSG macros you'll see below are used to create and access control 
      // messages (which is another name for ancillary data).  The ancillary 
      // data is made up of pairs of struct cmsghdr structures and associated
      // data arrays.
      //
      // First, we're going to allocate a buffer on the stack to receive our 
      // data array (that contains the socket).  Sometimes you'll see this called
      // an "ancillary element" but the msghdr uses the control message termimology
      // so we call it "control."
      //
      size_t msg_size = sizeof(int);
      char control[CMSG_SPACE (msg_size)];

      //
      // There is a msghdr that is used to minimize the number of parameters
      // passed to recvmsg (which we will use to receive our ancillary data).
      // This structure uses terminology corresponding to control messages, so
      // you'll see msg_control, which is the pointer to the ancillary data and 
      // controllen which is the size of the ancillary data array.
      //
      // So, initialize the message header that describes the ancillary/control
      // data we expect to receive and point it to buffer.
      //
      struct msghdr msg;
      msg.msg_name = 0;
      msg.msg_namelen = 0;
      msg.msg_iov = &iov;
      msg.msg_iovlen = 1;
      msg.msg_control = control;
      msg.msg_controllen = sizeof (control);
      msg.msg_flags = 0;

      //
      // Now we can actually receive the interesting bits from the tap
      // creator process.  Lots of pain to get four bytes.
      //
      ssize_t bytesRead = recvmsg (sock, &msg, 0);
      NS_ABORT_MSG_IF (bytesRead != sizeof(int), "TapBridge::CreateTap(): Wrong byte count from socket creator");

      //
      // There may be a number of message headers/ancillary data arrays coming in.
      // Let's look for the one with a type SCM_RIGHTS which indicates it's the
      // one we're interested in.
      //
      struct cmsghdr *cmsg;
      for (cmsg = CMSG_FIRSTHDR (&msg); cmsg != NULL; cmsg = CMSG_NXTHDR (&msg, cmsg))
        {
          if (cmsg->cmsg_level == SOL_SOCKET &&
              cmsg->cmsg_type == SCM_RIGHTS)
            {
              //
              // This is the type of message we want.  Check to see if the magic 
              // number is correct and then pull out the socket we care about if
              // it matches
              //
              if (magic == TAP_MAGIC)
                {
                  NS_LOG_INFO ("Got SCM_RIGHTS with correct magic " << magic);
                  int *rawSocket = (int*)CMSG_DATA (cmsg);
                  NS_LOG_INFO ("Got the socket from the socket creator = " << *rawSocket);
                  m_sock = *rawSocket;
                  break;
                }
              else
                {
                  NS_LOG_INFO ("Got SCM_RIGHTS, but with bad magic " << magic);
                }
            }
        }
      if (cmsg == NULL)
        {
          NS_FATAL_ERROR ("Did not get the raw socket from the socket creator");
        }

      if (m_mode == USE_LOCAL || m_mode == USE_BRIDGE)
        {
          //
          // Set the ns-3 device's mac address to the overlying container's
          // mac address
          //
          struct ifreq s;
          strncpy (s.ifr_name, m_tapDeviceName.c_str (), sizeof (s.ifr_name));

          NS_LOG_INFO ("Trying to get MacAddr of " << m_tapDeviceName);
          int ioctlResult = ioctl (sock, SIOCGIFHWADDR, &s);
          if (ioctlResult == 0)
            {
              Mac48Address learnedMac;
              learnedMac.CopyFrom ((uint8_t *)s.ifr_hwaddr.sa_data);
              NS_LOG_INFO ("Learned Tap device MacAddr is " << learnedMac << ": setting ns-3 device to use this address");
              m_bridgedDevice->SetAddress (learnedMac);
              m_ns3AddressRewritten = true;
            }

          if (!m_ns3AddressRewritten)
            {
              NS_LOG_INFO ("Cannot get MacAddr of Tap device: " << m_tapDeviceName << " while in USE_LOCAL/USE_BRIDGE mode: " << std::strerror (errno));
              NS_LOG_INFO ("Underlying ns-3 device will continue to use default address, what can lead to connectivity errors");
            }
        }
    }

  close (sock);
}

void
TapBridge::ReadCallback (uint8_t *buf, ssize_t len)
{
  NS_LOG_FUNCTION_NOARGS ();

  NS_ASSERT_MSG (buf != 0, "invalid buf argument");
  NS_ASSERT_MSG (len > 0, "invalid len argument");

  //
  // It's important to remember that we're in a completely different thread
  // than the simulator is running in.  We need to synchronize with that
  // other thread to get the packet up into ns-3.  What we will need to do
  // is to schedule a method to deal with the packet using the multithreaded
  // simulator we are most certainly running.  However, I just said it -- we
  // are talking about two threads here, so it is very, very dangerous to do
  // any kind of reference counting on a shared object.  Just don't do it.
  // So what we're going to do is pass the buffer allocated on the heap
  // into the ns-3 context thread where it will create the packet.
  //

  NS_LOG_INFO ("TapBridge::ReadCallback(): Received packet on node " << m_nodeId);
  NS_LOG_INFO ("TapBridge::ReadCallback(): Scheduling handler");
  Simulator::ScheduleWithContext (m_nodeId, Seconds (0.0), MakeEvent (&TapBridge::ForwardToBridgedDevice, this, buf, len));
}

void
TapBridge::ForwardToBridgedDevice (uint8_t *buf, ssize_t len)
{
  NS_LOG_FUNCTION (buf << len);

  //
  // There are three operating modes for the TapBridge
  //
  // CONFIGURE_LOCAL means that ns-3 will create and configure a tap device
  // and we are expected to use it.  The tap device and the ns-3 net device
  // will have the same MAC address by definition.  Thus Send and SendFrom
  // are equivalent in this case.  We use Send to allow all ns-3 devices to
  // participate in this mode.
  //
  // USE_LOCAL mode tells us that we have got to USE a pre-created tap device
  // that will have a different MAC address from the ns-3 net device.  We 
  // also enforce the requirement that there will only be one MAC address
  // bridged on the Linux side so we can use Send (instead of SendFrom) in
  // the linux to ns-3 direction.  Again, all ns-3 devices can participate
  // in this mode.
  //
  // USE_BRIDGE mode tells us that we are logically extending a Linux bridge
  // on which lies our tap device.  In this case there may be many linux
  // net devices on the other side of the bridge and so we must use SendFrom
  // to preserve the possibly many source addresses.  Thus, ns-3 devices 
  // must support SendFrom in order to be considered for USE_BRIDGE mode.
  //

  //
  // First, create a packet out of the byte buffer we received and free that
  // buffer.
  //
  Ptr<Packet> packet = Create<Packet> (reinterpret_cast<const uint8_t *> (buf), len);
  std::free (buf);
  buf = 0;

  //
  // Make sure the packet we received is reasonable enough for the rest of the 
  // system to handle and get it ready to be injected directly into an ns-3
  // device.  What should come back is a packet with the Ethernet header 
  // (and possibly an LLC header as well) stripped off.
  //
  Address src, dst;
  uint16_t type;

  NS_LOG_LOGIC ("Received packet from tap device");

  Ptr<Packet> p = Filter (packet, &src, &dst, &type);
  if (p == 0)
    {
      NS_LOG_LOGIC ("TapBridge::ForwardToBridgedDevice:  Discarding packet as unfit for ns-3 consumption");
      return;
    }

  NS_LOG_LOGIC ("Pkt source is " << src);
  NS_LOG_LOGIC ("Pkt destination is " << dst);
  NS_LOG_LOGIC ("Pkt LengthType is " << type);
  if (m_mode == USE_LOCAL)
    {
      //
      // Packets we are going to forward should not be from a broadcast src
      //
      NS_ASSERT_MSG (Mac48Address::ConvertFrom (src) != Mac48Address ("ff:ff:ff:ff:ff:ff"), 
                     "TapBridge::ForwardToBridgedDevice:  Source addr is broadcast");
      if (m_ns3AddressRewritten == false)
        {
          //
          // Set the ns-3 device's mac address to the overlying container's
          // mac address
          //
          Mac48Address learnedMac = Mac48Address::ConvertFrom (src);
          NS_LOG_LOGIC ("Learned MacAddr is " << learnedMac << ": setting ns-3 device to use this address");
          m_bridgedDevice->SetAddress (Mac48Address::ConvertFrom (learnedMac));
          m_ns3AddressRewritten = true;
        }
      // 
      // If we are operating in USE_LOCAL mode, we may be attached to an ns-3
      // device that does not support bridging (SupportsSendFrom returns false).
      // But, since the mac addresses are now aligned, we can call Send()
      //
      NS_LOG_LOGIC ("Forwarding packet to ns-3 device via Send()");
      m_bridgedDevice->Send (packet, dst, type);
      return;
    }

  //
  // If we are operating in USE_BRIDGE mode, we have the 
  // situation described below:
  //
  //  Other Device  <-bridge->  Tap Device  <-bridge-> ns3 device
  //   Mac Addr A               Mac Addr B             Mac Addr C
  //
  // In Linux, "Other Device" and "Tap Device" are bridged together.  By this
  // we mean that a user has sone something in Linux like:
  //
  //   brctl addbr mybridge
  //   brctl addif other-device
  //   brctl addif tap-device
  //
  // In USE_BRIDGE mode, we want to logically extend this Linux behavior to the 
  // simulated ns3 device and make it appear as if it is connected to the Linux
  // subnet.  As you may expect, this means that we need to act like a real
  // Linux bridge and take all packets that come from "Tap Device" and ask 
  // "ns3 Device" to send them down its directly connected network.  Just like 
  // in a normal everyday bridge we need to call SendFrom in order to preserve 
  //the original packet's from address.
  //
  // If we are operating in CONFIGURE_LOCAL mode, we simply simply take all packets
  // that come from "Tap Device" and ask "ns3 Device" to send them down its 
  // directly connected network.  A normal bridge would need to call SendFrom
  // in order to preserve the original from address, but in CONFIGURE_LOCAL mode
  // the tap device and the ns-3 device have the same MAC address by definition so 
  // we can call Send.
  //
  NS_LOG_LOGIC ("Forwarding packet");

  if (m_mode == USE_BRIDGE)
    {
      m_bridgedDevice->SendFrom (packet, src, dst, type);
    }
  else
    {
      NS_ASSERT_MSG (m_mode == CONFIGURE_LOCAL, "TapBridge::ForwardToBridgedDevice(): Internal error");
      m_bridgedDevice->Send (packet, dst, type);
    }
}

Ptr<Packet>
TapBridge::Filter (Ptr<Packet> p, Address *src, Address *dst, uint16_t *type)
{
  NS_LOG_FUNCTION (p);
  uint32_t pktSize;

  //
  // We have a candidate packet for injection into ns-3.  We expect that since
  // it came over a socket that provides Ethernet packets, it should be big 
  // enough to hold an EthernetHeader.  If it can't, we signify the packet 
  // should be filtered out by returning 0.
  //
  pktSize = p->GetSize ();
  EthernetHeader header (false);
  if (pktSize < header.GetSerializedSize ())
    {
      return 0;
    }

  uint32_t headerSize = p->PeekHeader (header);
  p->RemoveAtStart (headerSize);

  NS_LOG_LOGIC ("Pkt source is " << header.GetSource ());
  NS_LOG_LOGIC ("Pkt destination is " << header.GetDestination ());
  NS_LOG_LOGIC ("Pkt LengthType is " << header.GetLengthType ());

  //
  // If the length/type is less than 1500, it corresponds to a length 
  // interpretation packet.  In this case, it is an 802.3 packet and 
  // will also have an 802.2 LLC header.  If greater than 1500, we
  // find the protocol number (Ethernet type) directly.
  //
  if (header.GetLengthType () <= 1500)
    {
      *src = header.GetSource ();
      *dst = header.GetDestination ();

      pktSize = p->GetSize ();
      LlcSnapHeader llc;
      if (pktSize < llc.GetSerializedSize ())
        {
          return 0;
        }

      p->RemoveHeader (llc);
      *type = llc.GetType ();
    }
  else
    {
      *src = header.GetSource ();
      *dst = header.GetDestination ();
      *type = header.GetLengthType ();
    }

  //
  // What we give back is a packet without the Ethernet header (nor the 
  // possible llc/snap header) on it.  We think it is ready to send on
  // out the bridged net device.
  //
  return p;
}

Ptr<NetDevice>
TapBridge::GetBridgedNetDevice (void)
{
  NS_LOG_FUNCTION_NOARGS ();
  return m_bridgedDevice;
}

void 
TapBridge::SetBridgedNetDevice (Ptr<NetDevice> bridgedDevice)
{
  NS_LOG_FUNCTION (bridgedDevice);

  NS_ASSERT_MSG (m_node != 0, "TapBridge::SetBridgedDevice:  Bridge not installed in a node");
  NS_ASSERT_MSG (bridgedDevice != this, "TapBridge::SetBridgedDevice:  Cannot bridge to self");
  NS_ASSERT_MSG (m_bridgedDevice == 0, "TapBridge::SetBridgedDevice:  Already bridged");

  if (!Mac48Address::IsMatchingType (bridgedDevice->GetAddress ()))
    {
      NS_FATAL_ERROR ("TapBridge::SetBridgedDevice: Device does not support eui 48 addresses: cannot be added to bridge.");
    }

  if (m_mode == USE_BRIDGE && !bridgedDevice->SupportsSendFrom ())
    {
      NS_FATAL_ERROR ("TapBridge::SetBridgedDevice: Device does not support SendFrom: cannot be added to bridge.");
    }

  //
  // We need to disconnect the bridged device from the internet stack on our
  // node to ensure that only one stack responds to packets inbound over the
  // bridged device.  That one stack lives outside ns-3 so we just blatantly
  // steal the device callbacks.
  //
  // N.B This can be undone if someone does a RegisterProtocolHandler later 
  // on this node.
  //
  bridgedDevice->SetReceiveCallback (MakeCallback (&TapBridge::DiscardFromBridgedDevice, this));
  bridgedDevice->SetPromiscReceiveCallback (MakeCallback (&TapBridge::ReceiveFromBridgedDevice, this));
  m_bridgedDevice = bridgedDevice;
}

bool
TapBridge::DiscardFromBridgedDevice (Ptr<NetDevice> device, Ptr<const Packet> packet, uint16_t protocol, const Address &src)
{
  NS_LOG_FUNCTION (device << packet << protocol << src);
  NS_LOG_LOGIC ("Discarding packet stolen from bridged device " << device);
  return true;
}

bool
TapBridge::ReceiveFromBridgedDevice (
  Ptr<NetDevice> device, 
  Ptr<const Packet> packet, 
  uint16_t protocol,
  const Address &src, 
  const Address &dst, 
  PacketType packetType)
{
  NS_LOG_FUNCTION (device << packet << protocol << src << dst << packetType);
  NS_ASSERT_MSG (device == m_bridgedDevice, "TapBridge::SetBridgedDevice:  Received packet from unexpected device");
  NS_LOG_DEBUG ("Packet UID is " << packet->GetUid ());

  //
  // There are three operating modes for the TapBridge
  //
  // CONFIGURE_LOCAL means that ns-3 will create and configure a tap device
  // and we are expected to use it.  The tap device and the ns-3 net device
  // will have the same MAC address by definition.
  //
  // USE_LOCAL mode tells us that we have got to USE a pre-created tap device
  // that will have a different MAC address from the ns-3 net device.  In this
  // case we will be spoofing the MAC address of a received packet to match
  // the single allowed address on the Linux side.
  //
  // USE_BRIDGE mode tells us that we are logically extending a Linux bridge
  // on which lies our tap device.
  //

  if (m_mode == CONFIGURE_LOCAL && packetType == PACKET_OTHERHOST)
    {
      //
      // We hooked the promiscuous mode protocol handler so we could get the 
      // destination address of the actual packet.  This means we will be 
      // getting PACKET_OTHERHOST packets (not broadcast, not multicast, not 
      // unicast to the ns-3 net device, but to some other address).  In 
      // CONFIGURE_LOCAL mode we are not interested in these packets since they 
      // don't refer to the single MAC address shared by the ns-3 device and 
      // the TAP device.  If, however, we are in USE_LOCAL or USE_BRIDGE mode, 
      // we want to act like a bridge and forward these PACKET_OTHERHOST 
      // packets.
      //
      return true;
    }

  Mac48Address from = Mac48Address::ConvertFrom (src);
  Mac48Address to = Mac48Address::ConvertFrom (dst);

  Ptr<Packet> p = packet->Copy ();
  EthernetHeader header = EthernetHeader (false);
  header.SetSource (from);
  header.SetDestination (to);

  header.SetLengthType (protocol);
  p->AddHeader (header);

  NS_LOG_LOGIC ("Writing packet to Linux host");
  NS_LOG_LOGIC ("Pkt source is " << header.GetSource ());
  NS_LOG_LOGIC ("Pkt destination is " << header.GetDestination ());
  NS_LOG_LOGIC ("Pkt LengthType is " << header.GetLengthType ());
  NS_LOG_LOGIC ("Pkt size is " << p->GetSize ());

  NS_ASSERT_MSG (p->GetSize () <= 65536, "TapBridge::ReceiveFromBridgedDevice: Packet too big " << p->GetSize ());
  p->CopyData (m_packetBuffer, p->GetSize ());

  uint32_t bytesWritten = write (m_sock, m_packetBuffer, p->GetSize ());
  NS_ABORT_MSG_IF (bytesWritten != p->GetSize (), "TapBridge::ReceiveFromBridgedDevice(): Write error.");

  NS_LOG_LOGIC ("End of receive packet handling on node " << m_node->GetId ());
  return true;
}

void 
TapBridge::SetIfIndex (const uint32_t index)
{
  NS_LOG_FUNCTION_NOARGS ();
  m_ifIndex = index;
}

uint32_t 
TapBridge::GetIfIndex (void) const
{
  NS_LOG_FUNCTION_NOARGS ();
  return m_ifIndex;
}

Ptr<Channel> 
TapBridge::GetChannel (void) const
{
  NS_LOG_FUNCTION_NOARGS ();
  return 0;
}

void
TapBridge::SetAddress (Address address)
{
  NS_LOG_FUNCTION (address);
  m_address = Mac48Address::ConvertFrom (address);
}

Address 
TapBridge::GetAddress (void) const
{
  NS_LOG_FUNCTION_NOARGS ();
  return m_address;
}

void
TapBridge::SetMode (enum Mode mode)
{
  NS_LOG_FUNCTION (mode);
  m_mode = mode;
}

TapBridge::Mode
TapBridge::GetMode (void)
{
  NS_LOG_FUNCTION_NOARGS ();
  return m_mode;
}

bool 
TapBridge::SetMtu (const uint16_t mtu)
{
  NS_LOG_FUNCTION_NOARGS ();
  m_mtu = mtu;
  return true;
}

uint16_t 
TapBridge::GetMtu (void) const
{
  NS_LOG_FUNCTION_NOARGS ();
  return m_mtu;
}

void
TapBridge::NotifyLinkUp (void)
{
  NS_LOG_FUNCTION_NOARGS ();
  if (!m_linkUp)
    {
      m_linkUp = true;
      m_linkChangeCallbacks ();
    }
}

bool 
TapBridge::IsLinkUp (void) const
{
  NS_LOG_FUNCTION_NOARGS ();
  return m_linkUp;
}

void 
TapBridge::AddLinkChangeCallback (Callback<void> callback)
{
  NS_LOG_FUNCTION_NOARGS ();
  m_linkChangeCallbacks.ConnectWithoutContext (callback);
}

bool 
TapBridge::IsBroadcast (void) const
{
  NS_LOG_FUNCTION_NOARGS ();
  return true;
}

Address
TapBridge::GetBroadcast (void) const
{
  NS_LOG_FUNCTION_NOARGS ();
  return Mac48Address ("ff:ff:ff:ff:ff:ff");
}

bool
TapBridge::IsMulticast (void) const
{
  NS_LOG_FUNCTION_NOARGS ();
  return true;
}

Address
TapBridge::GetMulticast (Ipv4Address multicastGroup) const
{
  NS_LOG_FUNCTION (this << multicastGroup);
  Mac48Address multicast = Mac48Address::GetMulticast (multicastGroup);
  return multicast;
}

bool 
TapBridge::IsPointToPoint (void) const
{
  NS_LOG_FUNCTION_NOARGS ();
  return false;
}

bool 
TapBridge::IsBridge (void) const
{
  NS_LOG_FUNCTION_NOARGS ();
  //
  // Returning false from IsBridge in a device called TapBridge may seem odd
  // at first glance, but this test is for a device that bridges ns-3 devices
  // together.  The Tap bridge doesn't do that -- it bridges an ns-3 device to
  // a Linux device.  This is a completely different story.
  // 
  return false;
}

bool 
TapBridge::Send (Ptr<Packet> packet, const Address& dst, uint16_t protocol)
{
  NS_LOG_FUNCTION (packet << dst << protocol);
  NS_FATAL_ERROR ("TapBridge::Send: You may not call Send on a TapBridge directly");
  return false;
}

bool 
TapBridge::SendFrom (Ptr<Packet> packet, const Address& src, const Address& dst, uint16_t protocol)
{
  NS_LOG_FUNCTION (packet << src << dst << protocol);
  NS_FATAL_ERROR ("TapBridge::Send: You may not call SendFrom on a TapBridge directly");
  return false;
}

Ptr<Node> 
TapBridge::GetNode (void) const
{
  NS_LOG_FUNCTION_NOARGS ();
  return m_node;
}

void 
TapBridge::SetNode (Ptr<Node> node)
{
  NS_LOG_FUNCTION_NOARGS ();
  m_node = node;
}

bool 
TapBridge::NeedsArp (void) const
{
  NS_LOG_FUNCTION_NOARGS ();
  return true;
}

void 
TapBridge::SetReceiveCallback (NetDevice::ReceiveCallback cb)
{
  NS_LOG_FUNCTION_NOARGS ();
  m_rxCallback = cb;
}

void 
TapBridge::SetPromiscReceiveCallback (NetDevice::PromiscReceiveCallback cb)
{
  NS_LOG_FUNCTION_NOARGS ();
  m_promiscRxCallback = cb;
}

bool
TapBridge::SupportsSendFrom () const
{
  NS_LOG_FUNCTION_NOARGS ();
  return true;
}

Address TapBridge::GetMulticast (Ipv6Address addr) const
{
  NS_LOG_FUNCTION (this << addr);
  return Mac48Address::GetMulticast (addr);
}

} // namespace ns3