src/devices/emu/emu-net-device.cc
author Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
Tue, 17 Aug 2010 21:28:08 -0700
changeset 6580 ecb0f9a3849a
parent 6576 29512368dd2e
permissions -rw-r--r--
Call StopDevice() in EmuNetDevice::DoDispose()
     1 /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
     2 /*
     3  * Copyright (c) 2008 University of Washington
     4  *
     5  * This program is free software; you can redistribute it and/or modify
     6  * it under the terms of the GNU General Public License version 2 as
     7  * published by the Free Software Foundation;
     8  *
     9  * This program is distributed in the hope that it will be useful,
    10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    12  * GNU General Public License for more details.
    13  *
    14  * You should have received a copy of the GNU General Public License
    15  * along with this program; if not, write to the Free Software
    16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
    17  */
    18 
    19 #include "emu-net-device.h"
    20 #include "emu-encode-decode.h"
    21 
    22 #include "ns3/log.h"
    23 #include "ns3/queue.h"
    24 #include "ns3/simulator.h"
    25 #include "ns3/ethernet-header.h"
    26 #include "ns3/ethernet-trailer.h"
    27 #include "ns3/llc-snap-header.h"
    28 #include "ns3/boolean.h"
    29 #include "ns3/uinteger.h"
    30 #include "ns3/pointer.h"
    31 #include "ns3/string.h"
    32 #include "ns3/trace-source-accessor.h"
    33 #include "ns3/channel.h"
    34 #include "ns3/system-thread.h"
    35 #include "ns3/mac48-address.h"
    36 #include "ns3/enum.h"
    37 
    38 #include <sys/wait.h>
    39 #include <sys/stat.h>
    40 #include <sys/socket.h>
    41 #include <sys/un.h>
    42 #include <sys/ioctl.h>
    43 #include <net/ethernet.h>
    44 #include <net/if.h>
    45 #include <netinet/in.h>
    46 #include <netpacket/packet.h>
    47 #include <arpa/inet.h>
    48 #include <errno.h>
    49 #include <limits>
    50 #include <stdlib.h>
    51 #include <time.h>
    52 
    53 NS_LOG_COMPONENT_DEFINE ("EmuNetDevice");
    54 
    55 namespace ns3 {
    56 
    57 NS_OBJECT_ENSURE_REGISTERED (EmuNetDevice);
    58 
    59 #define EMU_MAGIC 65867
    60 
    61 TypeId 
    62 EmuNetDevice::GetTypeId (void)
    63 {
    64   static TypeId tid = TypeId ("ns3::EmuNetDevice")
    65     .SetParent<NetDevice> ()
    66     .AddConstructor<EmuNetDevice> ()
    67     .AddAttribute ("Mtu", "The MAC-level Maximum Transmission Unit",
    68                    UintegerValue (0), // arbitrary un-used value because no setter
    69                    MakeUintegerAccessor (&EmuNetDevice::GetMtu),
    70                    MakeUintegerChecker<uint16_t> ())                   
    71     .AddAttribute ("Address", 
    72                    "The ns-3 MAC address of this (virtual) device.",
    73                    Mac48AddressValue (Mac48Address ("ff:ff:ff:ff:ff:ff")),
    74                    MakeMac48AddressAccessor (&EmuNetDevice::m_address),
    75                    MakeMac48AddressChecker ())
    76     .AddAttribute ("DeviceName", 
    77                    "The name of the underlying real device (e.g. eth1).",
    78                    StringValue ("eth1"),
    79                    MakeStringAccessor (&EmuNetDevice::m_deviceName),
    80                    MakeStringChecker ())
    81     .AddAttribute ("Start", 
    82                    "The simulation time at which to spin up the device thread.",
    83                    TimeValue (Seconds (0.)),
    84                    MakeTimeAccessor (&EmuNetDevice::m_tStart),
    85                    MakeTimeChecker ())
    86     .AddAttribute ("Stop", 
    87                    "The simulation time at which to tear down the device thread.",
    88                    TimeValue (Seconds (0.)),
    89                    MakeTimeAccessor (&EmuNetDevice::m_tStop),
    90                    MakeTimeChecker ())
    91     .AddAttribute ("EncapsulationMode", 
    92                    "The link-layer encapsulation type to use.",
    93                    EnumValue (LLC),
    94                    MakeEnumAccessor (&EmuNetDevice::SetEncapsulationMode),
    95                    MakeEnumChecker (DIX, "Dix",
    96                                     LLC, "Llc"))
    97 
    98     //
    99     // Transmit queueing discipline for the device which includes its own set
   100     // of trace hooks.  Note that this is really going to run "on top of" the 
   101     // queueing discipline that will most likely be present in the devices of
   102     // the underlying operating system.
   103     //
   104     .AddAttribute ("TxQueue", 
   105                    "A queue to use as the transmit queue in the device.",
   106                    PointerValue (),
   107                    MakePointerAccessor (&EmuNetDevice::m_queue),
   108                    MakePointerChecker<Queue> ())
   109 
   110     .AddAttribute ("RxQueueSize", "Maximum size of the read queue.  "
   111                    "This value limits number of packets that have been read "
   112                    "from the network into a memory buffer but have not yet "
   113                    "been processed by the simulator.",
   114                    UintegerValue (1000),
   115                    MakeUintegerAccessor (&EmuNetDevice::m_maxPendingReads),
   116                    MakeUintegerChecker<uint32_t> ())
   117 
   118     //
   119     // Trace sources at the "top" of the net device, where packets transition
   120     // to/from higher layers.  These points do not really correspond to the 
   121     // MAC layer of the underlying operating system, but exist to provide 
   122     // a consitent tracing environment.  These trace hooks should really be
   123     // interpreted as the points at which a packet leaves the ns-3 environment
   124     // destined for the underlying operating system or vice-versa.
   125     //
   126     .AddTraceSource ("MacTx", 
   127                      "Trace source indicating a packet has arrived for transmission by this device",
   128                      MakeTraceSourceAccessor (&EmuNetDevice::m_macTxTrace))
   129     .AddTraceSource ("MacTxDrop", 
   130                      "Trace source indicating a packet has been dropped by the device before transmission",
   131                      MakeTraceSourceAccessor (&EmuNetDevice::m_macTxDropTrace))
   132     .AddTraceSource ("MacPromiscRx", 
   133                      "A packet has been received by this device, has been passed up from the physical layer "
   134                      "and is being forwarded up the local protocol stack.  This is a promiscuous trace,",
   135                      MakeTraceSourceAccessor (&EmuNetDevice::m_macPromiscRxTrace))
   136     .AddTraceSource ("MacRx", 
   137                      "A packet has been received by this device, has been passed up from the physical layer "
   138                      "and is being forwarded up the local protocol stack.  This is a non-promiscuous trace,",
   139                      MakeTraceSourceAccessor (&EmuNetDevice::m_macRxTrace))
   140 #if 0
   141     // Not currently implemented for this device
   142     .AddTraceSource ("MacRxDrop", 
   143                      "Trace source indicating a packet was dropped before being forwarded up the stack",
   144                      MakeTraceSourceAccessor (&EmuNetDevice::m_macRxDropTrace))
   145 #endif
   146     //
   147     // In normal ns-3 net devices, these trace souces correspond to the "bottom"
   148     // of the net device, where packets transition to/from the channel.  In 
   149     // the case of the emu device, there is no physical layer access -- all we
   150     // do is to send packets to another device that is really at a "real" MAC
   151     // level.  Since it could be misleading to call anything here PHY, we do not
   152     // implement these trace sources.
   153     //
   154 #if 0
   155     .AddTraceSource ("PhyTxBegin", 
   156                      "Trace source indicating a packet has begun transmitting over the channel",
   157                      MakeTraceSourceAccessor (&EmuNetDevice::m_phyTxBeginTrace))
   158     .AddTraceSource ("PhyTxEnd", 
   159                      "Trace source indicating a packet has been completely transmitted over the channel",
   160                      MakeTraceSourceAccessor (&EmuNetDevice::m_phyTxEndTrace))
   161     .AddTraceSource ("PhyTxDrop", 
   162                      "Trace source indicating a packet has been dropped by the device during transmission",
   163                      MakeTraceSourceAccessor (&EmuNetDevice::m_phyTxDropTrace))
   164     .AddTraceSource ("PhyRxBegin", 
   165                      "Trace source indicating a packet has begun being received by the device",
   166                      MakeTraceSourceAccessor (&EmuNetDevice::m_phyRxBeginTrace))
   167     .AddTraceSource ("PhyRxEnd", 
   168                      "Trace source indicating a packet has been completely received by the device",
   169                      MakeTraceSourceAccessor (&EmuNetDevice::m_phyRxEndTrace))
   170     .AddTraceSource ("PhyRxDrop", 
   171                      "Trace source indicating a packet has been dropped by the device during reception",
   172                      MakeTraceSourceAccessor (&EmuNetDevice::m_phyRxDropTrace))
   173 #endif
   174     //
   175     // Trace sources designed to simulate a packet sniffer facility (tcpdump). 
   176     //
   177     .AddTraceSource ("Sniffer", 
   178                      "Trace source simulating a non-promiscuous packet sniffer attached to the device",
   179                      MakeTraceSourceAccessor (&EmuNetDevice::m_snifferTrace))
   180     .AddTraceSource ("PromiscSniffer", 
   181                      "Trace source simulating a promiscuous packet sniffer attached to the device",
   182                      MakeTraceSourceAccessor (&EmuNetDevice::m_promiscSnifferTrace))
   183     ;
   184   return tid;
   185 }
   186 
   187 EmuNetDevice::EmuNetDevice () 
   188 : 
   189   m_startEvent (),
   190   m_stopEvent (),
   191   m_sock (-1),
   192   m_readThread (0),
   193   m_ifIndex (std::numeric_limits<uint32_t>::max ()),  // absurdly large value
   194   m_sll_ifindex (-1),
   195   m_isBroadcast (true),
   196   m_isMulticast (false),
   197   m_pendingReadCount (0)
   198 {
   199   NS_LOG_FUNCTION (this);
   200   m_packetBuffer = new uint8_t[65536];
   201   Start (m_tStart);
   202 }
   203 
   204 EmuNetDevice::~EmuNetDevice ()
   205 {
   206   delete [] m_packetBuffer;
   207   m_packetBuffer = 0;
   208 }
   209 
   210 void 
   211 EmuNetDevice::DoDispose()
   212 {
   213   NS_LOG_FUNCTION_NOARGS ();
   214   if (m_readThread != 0)
   215     {
   216       StopDevice ();
   217     }
   218   m_node = 0;
   219   NetDevice::DoDispose ();
   220 }
   221 
   222 void 
   223 EmuNetDevice::SetEncapsulationMode (enum EncapsulationMode mode)
   224 {
   225   NS_LOG_FUNCTION (mode);
   226   m_encapMode = mode;
   227   NS_LOG_LOGIC ("m_encapMode = " << m_encapMode);
   228 }
   229 
   230 EmuNetDevice::EncapsulationMode
   231 EmuNetDevice::GetEncapsulationMode (void) const
   232 {
   233   NS_LOG_FUNCTION_NOARGS ();
   234   return m_encapMode;
   235 }
   236 
   237 void
   238 EmuNetDevice::Start (Time tStart)
   239 {
   240   NS_LOG_FUNCTION (tStart);
   241 
   242   //
   243   // Cancel any pending start event and schedule a new one at some relative time in the future.
   244   //
   245   Simulator::Cancel (m_startEvent);
   246   m_startEvent = Simulator::Schedule (tStart, &EmuNetDevice::StartDevice, this);
   247 }
   248 
   249   void
   250 EmuNetDevice::Stop (Time tStop)
   251 {
   252   NS_LOG_FUNCTION (tStop);
   253   //
   254   // Cancel any pending stop event and schedule a new one at some relative time in the future.
   255   //
   256   Simulator::Cancel (m_stopEvent);
   257   m_startEvent = Simulator::Schedule (tStop, &EmuNetDevice::StopDevice, this);
   258 }
   259 
   260   void
   261 EmuNetDevice::StartDevice (void)
   262 {
   263   NS_LOG_FUNCTION_NOARGS ();
   264 
   265   //
   266   // Spin up the emu net device and start receiving packets.
   267   //
   268   if (m_sock != -1)
   269     {
   270       NS_FATAL_ERROR ("EmuNetDevice::StartDevice(): Device is already started");
   271     }
   272 
   273   //
   274   // We're going to need a pointer to the realtime simulator implementation.
   275   // It's important to remember that access to that implementation may happen 
   276   // in a completely different thread than the simulator is running in (we're 
   277   // going to spin up that thread below).  We are talking about multiple threads
   278   // here, so it is very, very dangerous to do any kind of reference couning on
   279   // a shared object that is unaware of what is happening.  What we are going to 
   280   // do to address that is to get a reference to the realtime simulator here 
   281   // where we are running in the context of a running simulator scheduler --
   282   // recall we did a Simulator::Schedule of this method above.  We get the
   283   // simulator implementation pointer in a single-threaded way and save the
   284   // underlying raw pointer for use by the (other) read thread.  We must not
   285   // free this pointer or we may delete the simulator out from under us an 
   286   // everyone else.  We assume that the simulator implementation cannot be 
   287   // replaced while the emu device is running and so will remain valid through
   288   // the time during which the read thread is running.
   289   //
   290   Ptr<RealtimeSimulatorImpl> impl = DynamicCast<RealtimeSimulatorImpl> (Simulator::GetImplementation ());
   291   m_rtImpl = GetPointer (impl);
   292 
   293   //
   294   // A similar story exists for the node ID.  We can't just naively do a
   295   // GetNode ()->GetId () since GetNode is going to give us a Ptr<Node> which
   296   // is reference counted.  We need to stash away the node ID for use in the
   297   // read thread.
   298   //
   299   m_nodeId = GetNode ()->GetId ();
   300 
   301   NS_LOG_LOGIC ("Creating socket");
   302 
   303   //
   304   // Call out to a separate process running as suid root in order to get a raw 
   305   // socket.  We do this to avoid having the entire simulation running as root.
   306   // If this method returns, we'll have a raw socket waiting for us in m_sock.
   307   //
   308   CreateSocket ();
   309 
   310   //
   311   // Figure out which interface index corresponds to the device name in the corresponding attribute.
   312   //
   313   struct ifreq ifr;
   314   bzero (&ifr, sizeof(ifr));
   315   strncpy ((char *)ifr.ifr_name, m_deviceName.c_str (), IFNAMSIZ);
   316 
   317   NS_LOG_LOGIC ("Getting interface index");
   318   int32_t rc = ioctl (m_sock, SIOCGIFINDEX, &ifr);
   319   if (rc == -1)
   320     {
   321       NS_FATAL_ERROR ("EmuNetDevice::StartDevice(): Can't get interface index");
   322     }
   323 
   324   //
   325   // Save the real interface index for later calls to sendto
   326   //
   327   m_sll_ifindex = ifr.ifr_ifindex;
   328 
   329   //
   330   // Bind the socket to the interface we just found.
   331   //
   332   struct sockaddr_ll ll;
   333   bzero (&ll, sizeof(ll));
   334 
   335   ll.sll_family = AF_PACKET;
   336   ll.sll_ifindex = m_sll_ifindex;
   337   ll.sll_protocol = htons(ETH_P_ALL); 
   338 
   339   NS_LOG_LOGIC ("Binding socket to interface");
   340 
   341   rc = bind (m_sock, (struct sockaddr *)&ll, sizeof (ll));
   342   if (rc == -1)
   343     {
   344       NS_FATAL_ERROR ("EmuNetDevice::StartDevice(): Can't bind to specified interface");
   345     }
   346 
   347   rc = ioctl(m_sock, SIOCGIFFLAGS, &ifr);
   348   if (rc == -1)
   349     {
   350       NS_FATAL_ERROR ("EmuNetDevice::StartDevice(): Can't get interface flags");
   351     }
   352   
   353   //
   354   // This device only works if the underlying interface is up in promiscuous 
   355   // mode.  We could have turned it on in the socket creator, but the situation
   356   // is that we expect these devices to be used in conjunction with virtual 
   357   // machines with connected host-only (simulated) networks, or in a testbed.
   358   // There is a lot of setup and configuration happening outside of this one 
   359   // issue, and we expect that configuration to include choosing a valid
   360   // interface (e.g, "ath1"), ensuring that the device supports promiscuous 
   361   // mode, and placing it in promiscuous mode.  We just make sure of the
   362   // end result.
   363   //
   364   if ((ifr.ifr_flags & IFF_PROMISC) == 0)
   365     {
   366       NS_FATAL_ERROR ("EmuNetDevice::StartDevice(): " << m_deviceName << " is not in promiscuous mode");
   367     }
   368   if ((ifr.ifr_flags & IFF_BROADCAST) != IFF_BROADCAST)
   369     {
   370       // We default m_isBroadcast to true but turn it off here if not
   371       // supported, because in the common case, overlying IP code will 
   372       // assert during configuration time if this is false, before this
   373       // method has a chance to set it during runtime
   374       m_isBroadcast = false;
   375     }
   376   if ((ifr.ifr_flags & IFF_MULTICAST) == IFF_MULTICAST)
   377     {
   378       // This one is OK to enable at runtime
   379       m_isMulticast = true;
   380     }
   381 
   382   //
   383   // Now spin up a read thread to read packets.
   384   //
   385   if (m_readThread != 0)
   386     {
   387       NS_FATAL_ERROR ("EmuNetDevice::StartDevice(): Receive thread is already running");
   388     }
   389 
   390   NS_LOG_LOGIC ("Spinning up read thread");
   391 
   392   m_readThread = Create<SystemThread> (MakeCallback (&EmuNetDevice::ReadThread, this));
   393   m_readThread->Start ();
   394 
   395   NotifyLinkUp ();
   396 }
   397 
   398 void
   399 EmuNetDevice::CreateSocket (void)
   400 {
   401   NS_LOG_FUNCTION_NOARGS ();
   402   //
   403   // We want to create a raw socket for our net device.  Unfortunately for us
   404   // you have to have root privileges to do that.  Instead of running the 
   405   // entire simulation as root, we decided to make a small program who's whole
   406   // reason for being is to run as suid root and create a raw socket.  We're
   407   // going to fork and exec that program soon, but we need to have a socket
   408   // to talk to it with.  So we create a local interprocess (Unix) socket 
   409   // for that purpose.
   410   //
   411   int sock = socket (PF_UNIX, SOCK_DGRAM, 0);
   412   if (sock == -1)
   413     {
   414       NS_FATAL_ERROR ("EmuNetDevice::CreateSocket(): Unix socket creation error, errno = " << strerror (errno));
   415     }
   416 
   417   //
   418   // Bind to that socket and let the kernel allocate an endpoint
   419   //
   420   struct sockaddr_un un;
   421   memset (&un, 0, sizeof (un));
   422   un.sun_family = AF_UNIX;
   423   int status = bind (sock, (struct sockaddr*)&un, sizeof (sa_family_t));
   424   if (status == -1)
   425     {
   426       NS_FATAL_ERROR ("EmuNetDevice::CreateSocket(): Could not bind(): errno = " << strerror (errno));
   427     }
   428 
   429   NS_LOG_INFO ("Created Unix socket");
   430   NS_LOG_INFO ("sun_family = " << un.sun_family);
   431   NS_LOG_INFO ("sun_path = " << un.sun_path);
   432 
   433   //
   434   // We have a socket here, but we want to get it there -- to the program we're
   435   // going to exec.  What we'll do is to do a getsockname and then encode the
   436   // resulting address information as a string, and then send the string to the
   437   // program as an argument.  So we need to get the sock name.
   438   //
   439   socklen_t len = sizeof (un);
   440   status = getsockname (sock, (struct sockaddr*)&un, &len);
   441   if (status == -1)
   442     {
   443       NS_FATAL_ERROR ("EmuNetDevice::CreateSocket(): Could not getsockname(): errno = " << strerror (errno));
   444     }
   445 
   446   //
   447   // Now encode that socket name (family and path) as a string of hex digits
   448   //
   449   std::string path = EmuBufferToString((uint8_t *)&un, len);
   450   NS_LOG_INFO ("Encoded Unix socket as \"" << path << "\"");
   451   //
   452   // Fork and exec the process to create our socket.  If we're us (the parent)
   453   // we wait for the child (the socket creator) to complete and read the 
   454   // socket it created using the ancillary data mechanism.
   455   //
   456   // Tom Goff reports the possiblility of a deadlock when trying to acquire the
   457   // python GIL here.  He says that this might be due to trying to access Python
   458   // objects after fork() without calling PyOS_AfterFork() to properly reset 
   459   // Python state (including the GIL).  There is no code to cause the problem
   460   // here in emu, but this was visible in similar code in tap-bridge.
   461   //
   462   pid_t pid = ::fork ();
   463   if (pid == 0)
   464     {
   465       NS_LOG_DEBUG ("Child process");
   466 
   467       //
   468       // build a command line argument from the encoded endpoint string that 
   469       // the socket creation process will use to figure out how to respond to
   470       // the (now) parent process.
   471       //
   472       std::ostringstream oss;
   473       oss << "-p" << path;
   474       NS_LOG_INFO ("Parameters set to \"" << oss.str () << "\"");
   475 
   476       //
   477       // Execute the socket creation process image.
   478       //
   479       status = ::execlp ("emu-sock-creator", 
   480                         "emu-sock-creator",                             // argv[0] (filename)
   481                         oss.str ().c_str (),                            // argv[1] (-p<path?
   482                         (char *)NULL);
   483 
   484       //
   485       // If the execlp successfully completes, it never returns.  If it returns it failed or the OS is
   486       // broken.  In either case, we bail.
   487       //
   488       NS_FATAL_ERROR ("EmuNetDevice::CreateSocket(): Back from execlp(), errno = " << ::strerror (errno));
   489     }
   490   else
   491     {
   492       NS_LOG_DEBUG ("Parent process");
   493       //
   494       // We're the process running the emu net device.  We need to wait for the
   495       // socket creator process to finish its job.
   496       //
   497       int st;
   498       pid_t waited = waitpid (pid, &st, 0);
   499       if (waited == -1)
   500 	{
   501 	  NS_FATAL_ERROR ("EmuNetDevice::CreateSocket(): waitpid() fails, errno = " << strerror (errno));
   502 	}
   503       NS_ASSERT_MSG (pid == waited, "EmuNetDevice::CreateSocket(): pid mismatch");
   504 
   505       //
   506       // Check to see if the socket creator exited normally and then take a 
   507       // look at the exit code.  If it bailed, so should we.  If it didn't
   508       // even exit normally, we bail too.
   509       //
   510       if (WIFEXITED (st))
   511 	{
   512           int exitStatus = WEXITSTATUS (st);
   513           if (exitStatus != 0)
   514             {
   515               NS_FATAL_ERROR ("EmuNetDevice::CreateSocket(): socket creator exited normally with status " << exitStatus);
   516             }
   517 	}
   518       else 
   519 	{
   520           NS_FATAL_ERROR ("EmuNetDevice::CreateSocket(): socket creator exited abnormally");
   521 	}
   522 
   523       //
   524       // At this point, the socket creator has run successfully and should 
   525       // have created our raw socket and sent it back to the socket address
   526       // we provided.  Our socket should be waiting on the Unix socket.  We've
   527       // got to do a bunch of grunto work to get at it, though.
   528       //
   529       // The struct iovec below is part of a scatter-gather list.  It describes a
   530       // buffer.  In this case, it describes a buffer (an integer) that will
   531       // get the data that comes back from the socket creator process.  It will
   532       // be a magic number that we use as a consistency/sanity check.
   533       // 
   534       struct iovec iov;
   535       uint32_t magic;
   536       iov.iov_base = &magic;
   537       iov.iov_len = sizeof(magic);
   538 
   539       //
   540       // The CMSG macros you'll see below are used to create and access control 
   541       // messages (which is another name for ancillary data).  The ancillary 
   542       // data is made up of pairs of struct cmsghdr structures and associated
   543       // data arrays.
   544       //
   545       // First, we're going to allocate a buffer on the stack to receive our 
   546       // data array (that contains the socket).  Sometimes you'll see this called
   547       // an "ancillary element" but the msghdr uses the control message termimology
   548       // so we call it "control."
   549       //
   550       size_t msg_size = sizeof(int);
   551       char control[CMSG_SPACE(msg_size)];
   552 
   553       //
   554       // There is a msghdr that is used to minimize the number of parameters
   555       // passed to recvmsg (which we will use to receive our ancillary data).  
   556       // This structure uses terminology corresponding to control messages, so
   557       // you'll see msg_control, which is the pointer to the ancillary data and 
   558       // controllen which is the size of the ancillary data array.
   559       //
   560       // So, initialize the message header that describes the ancillary/control
   561       // data we expect to receive and point it to buffer.
   562       //
   563       struct msghdr msg;
   564       msg.msg_name = 0;
   565       msg.msg_namelen = 0;
   566       msg.msg_iov = &iov;
   567       msg.msg_iovlen = 1;
   568       msg.msg_control = control;
   569       msg.msg_controllen = sizeof (control);
   570       msg.msg_flags = 0;
   571 
   572       //
   573       // Now we can actually receive the interesting bits from the socket
   574       // creator process.
   575       //
   576       ssize_t bytesRead = recvmsg (sock, &msg, 0);
   577       if (bytesRead != sizeof(int))
   578 	{
   579           NS_FATAL_ERROR ("EmuNetDevice::CreateSocket(): Wrong byte count from socket creator");
   580 	}
   581 
   582       //
   583       // There may be a number of message headers/ancillary data arrays coming in.
   584       // Let's look for the one with a type SCM_RIGHTS which indicates it' the
   585       // one we're interested in.
   586       //
   587       struct cmsghdr *cmsg;
   588       for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) 
   589 	{
   590 	  if (cmsg->cmsg_level == SOL_SOCKET &&
   591 	      cmsg->cmsg_type == SCM_RIGHTS)
   592 	    {
   593               //
   594               // This is the type of message we want.  Check to see if the magic 
   595               // number is correct and then pull out the socket we care about if
   596               // it matches
   597               //
   598               if (magic == EMU_MAGIC)
   599                 {
   600                   NS_LOG_INFO ("Got SCM_RIGHTS with correct magic " << magic);
   601                   int *rawSocket = (int*)CMSG_DATA (cmsg);
   602                   NS_LOG_INFO ("Got the socket from the socket creator = " << *rawSocket);
   603                   m_sock = *rawSocket;
   604                   return;
   605                 }
   606               else
   607                 {
   608                   NS_LOG_INFO ("Got SCM_RIGHTS, but with bad magic " << magic);                  
   609                 }
   610 	    }
   611 	}
   612       NS_FATAL_ERROR ("Did not get the raw socket from the socket creator");
   613     }
   614 }
   615 
   616 void
   617 EmuNetDevice::StopDevice (void)
   618 {
   619   NS_LOG_FUNCTION_NOARGS ();
   620 
   621   close (m_sock);
   622   m_sock = -1;
   623 
   624   NS_ASSERT_MSG (m_readThread != 0, "EmuNetDevice::StopDevice(): Receive thread is not running");
   625 
   626   NS_LOG_LOGIC ("Joining read thread");
   627   m_readThread->Join ();
   628   m_readThread = 0;
   629 }
   630 
   631 void
   632 EmuNetDevice::ForwardUp (uint8_t *buf, uint32_t len)
   633 {
   634   NS_LOG_FUNCTION (buf << len);
   635 
   636   //
   637   // Create a packet out of the buffer we received and free that buffer.
   638   //
   639   Ptr<Packet> packet = Create<Packet> (reinterpret_cast<const uint8_t *> (buf), len);
   640   free (buf);
   641   buf = 0;
   642 
   643   {
   644     CriticalSection cs (m_pendingReadMutex);
   645     //std::cerr << std::endl << "EmuNetDevice main thread: m_pendingReadCount is " << m_pendingReadCount << std::endl;
   646     --m_pendingReadCount;
   647   }
   648 
   649   //
   650   // Trace sinks will expect complete packets, not packets without some of the
   651   // headers.
   652   //
   653   Ptr<Packet> originalPacket = packet->Copy ();
   654 
   655   EthernetHeader header (false);
   656 
   657   //
   658   // This device could be running in an environment where completely unexpected
   659   // kinds of packets are flying around, so we need to harden things a bit and
   660   // filter out packets we think are completely bogus, so we always check to see
   661   // that the packet is long enough to contain the header we want to remove.
   662   //
   663   if (packet->GetSize() < header.GetSerializedSize())
   664     {
   665       m_phyRxDropTrace (originalPacket);
   666       return;
   667     }
   668 
   669   packet->RemoveHeader (header);
   670 
   671   NS_LOG_LOGIC ("Pkt source is " << header.GetSource ());
   672   NS_LOG_LOGIC ("Pkt destination is " << header.GetDestination ());
   673 
   674   uint16_t protocol;
   675   
   676   switch (m_encapMode)
   677     {
   678     case LLC:
   679       //
   680       // If the length/type is less than 1500, it corresponds to a length 
   681       // interpretation packet.  In this case, it is an 802.3 packet and 
   682       // will also have an 802.2 LLC header.  If greater than 1500, we
   683       // find the protocol number (Ethernet type) directly.
   684       //
   685       if (header.GetLengthType () <= 1500)
   686         {
   687           LlcSnapHeader llc;
   688           //
   689           // Check to see that the packet is long enough to possibly contain the
   690           // header we want to remove before just naively calling.
   691           //
   692           if (packet->GetSize() < llc.GetSerializedSize())
   693             {
   694               m_phyRxDropTrace (originalPacket);
   695               return;
   696             }
   697 
   698           packet->RemoveHeader (llc);
   699           protocol = llc.GetType ();
   700         }
   701       else
   702         {
   703           protocol = header.GetLengthType ();
   704         }
   705       break;
   706 
   707     case DIX:
   708       protocol = header.GetLengthType ();
   709       break;
   710 
   711     default:
   712       NS_FATAL_ERROR ("invalid encapsulation mode");
   713       protocol = 0; /* quiet compiler */
   714     }
   715 
   716   PacketType packetType;
   717 
   718   if (header.GetDestination ().IsBroadcast ())
   719     {
   720       packetType = NS3_PACKET_BROADCAST;
   721     }
   722   else if (header.GetDestination ().IsGroup ())
   723     {
   724       packetType = NS3_PACKET_MULTICAST;          
   725     }
   726   else if (header.GetDestination () == m_address)
   727     {
   728       packetType = NS3_PACKET_HOST;
   729     }
   730   else
   731     {
   732       packetType = NS3_PACKET_OTHERHOST;
   733     }
   734 
   735   // 
   736   // For all kinds of packetType we receive, we hit the promiscuous sniffer
   737   // hook and pass a copy up to the promiscuous callback.  Pass a copy to 
   738   // make sure that nobody messes with our packet.
   739   //
   740   m_promiscSnifferTrace (originalPacket);
   741 
   742   if (!m_promiscRxCallback.IsNull ())
   743     {
   744       m_macPromiscRxTrace (originalPacket);
   745       m_promiscRxCallback (this, packet, protocol, header.GetSource (), header.GetDestination (), packetType);
   746     }
   747 
   748   //
   749   // If this packet is not destined for some other host, it must be for us
   750   // as either a broadcast, multicast or unicast.  We need to hit the mac
   751   // packet received trace hook and forward the packet up the stack.
   752   //
   753   if (packetType != NS3_PACKET_OTHERHOST)
   754     {
   755       m_snifferTrace (originalPacket);
   756       m_macRxTrace (originalPacket);
   757       m_rxCallback (this, packet, protocol, header.GetSource ());
   758     }
   759 }
   760 
   761 void
   762 EmuNetDevice::ReadThread (void)
   763 {
   764   NS_LOG_FUNCTION_NOARGS ();
   765 
   766   // It's important to remember that we're in a completely different thread than the simulator is running in.
   767   // We are talking about multiple threads here, so it is very, very dangerous to do any kind of reference couning
   768   // on a shared object.
   769   //
   770 
   771   int32_t len = -1;
   772   struct sockaddr_ll addr;
   773   socklen_t addrSize = sizeof (addr);
   774 
   775   for (;;) 
   776     {
   777       //
   778       // Too many pending reads at the same time leads to excessive memory allocations.  This counter prevents it.
   779       // 
   780       bool skip = false;
   781       
   782       {
   783         CriticalSection cs (m_pendingReadMutex);            
   784         //std::cerr << std::endl << "EmuNetDevice read thread: m_pendingReadCount is " << m_pendingReadCount << std::endl;
   785         if (m_pendingReadCount >= m_maxPendingReads)
   786           {
   787             skip = true;
   788           }
   789         else
   790           {
   791             ++m_pendingReadCount;
   792           }
   793       }
   794 
   795       if (skip)  
   796         {           
   797           struct timespec time = { 0, 100000000L }; // 100 ms
   798           nanosleep (&time, NULL);
   799           continue;
   800         }
   801 
   802       //
   803       // to avoid any issues with a shared reference counted packet, we allocate a buffer on the heap and pass that
   804       // buffer into the ns-3 context thread where it will create the packet, copy the buffer and then free it.
   805       //
   806       uint32_t bufferSize = 65536;
   807       uint8_t *buf = (uint8_t *)malloc (bufferSize);
   808       if (buf == 0)
   809         {
   810           NS_FATAL_ERROR ("EmuNetDevice::ReadThread(): malloc packet buffer failed");
   811         }
   812 
   813       NS_LOG_LOGIC ("Calling recvfrom");
   814       len = recvfrom (m_sock, buf, bufferSize, 0, (struct sockaddr *)&addr, &addrSize);
   815 
   816       if (len == -1)
   817         {
   818           free (buf);
   819           buf = 0;
   820           return;
   821         }
   822 
   823       NS_LOG_INFO ("EmuNetDevice::EmuNetDevice(): Received packet on node " << m_nodeId);
   824       NS_LOG_INFO ("EmuNetDevice::ReadThread(): Scheduling handler");
   825       NS_ASSERT_MSG (m_rtImpl, "EmuNetDevice::ReadThread(): Realtime simulator implementation pointer not set");
   826       m_rtImpl->ScheduleRealtimeNowWithContext (m_nodeId, MakeEvent (&EmuNetDevice::ForwardUp, this, buf, len));
   827       buf = 0;
   828     }
   829 }
   830 
   831 bool 
   832 EmuNetDevice::Send (Ptr<Packet> packet, const Address &dest, uint16_t protocolNumber)
   833 {
   834   NS_LOG_FUNCTION (packet << dest << protocolNumber);
   835   //
   836   // The immediate questions here are how are we going to encapsulate packets and what do we use as the MAC source and 
   837   // destination (hardware) addresses?
   838   //
   839   // If we return false from EmuNetDevice::NeedsArp, the ArpIpv4Interface will pass the broadcast address as the 
   840   // hardware (Ethernet) destination by default.  If we return true from EmuNetDevice::NeedsArp, then the hardware
   841   // destination is actually meaningful, but we'll have an ns-3 ARP running on this device.  There can also be an ARP
   842   // running on the underlying OS so we have to be very careful, both about multiple ARPs and also about TCP, UDP, etc.
   843   //
   844   // We are operating in promiscuous mode on the receive side (all ns-3 net devices are required to implement the 
   845   // promiscuous callback in a meaningful way), so we have an option regarding the hardware addresses.  We don't actually have
   846   // to use the real hardware addresses and IP addresses of the underlying system.  We can completely use MAC-spoofing to
   847   // fake out the OS by using the ns-3 assigned MAC address (and also the ns-3 assigned IP addresses).  Ns-3 starts its 
   848   // MAC address allocation using the OUI (vendor-code) 00:00:00 which is unassigned to any organization and is a globally
   849   // administered address, so there shouldn't be any collisions with real hardware.
   850   //
   851   // So what we do is we return true from EmuNetDevice::NeedsArp which tells ns-3 to use its own ARP.  We spoof the 
   852   // MAC address of the device and use promiscuous mode to receive traffic destined to that address.
   853   //
   854   return SendFrom (packet, m_address, dest, protocolNumber);
   855 }
   856 
   857 bool 
   858 EmuNetDevice::SendFrom (Ptr<Packet> packet, const Address &src, const Address &dest, uint16_t protocolNumber)
   859 {
   860   NS_LOG_FUNCTION (packet << src << dest << protocolNumber);
   861   NS_LOG_LOGIC ("packet =" << packet);
   862   NS_LOG_LOGIC ("UID is " << packet->GetUid () << ")");
   863 
   864   if (IsLinkUp () == false)
   865     {
   866       m_macTxDropTrace (packet);
   867       return false;
   868     }
   869 
   870   Mac48Address destination = Mac48Address::ConvertFrom (dest);
   871   Mac48Address source = Mac48Address::ConvertFrom (src);
   872 
   873   NS_LOG_LOGIC ("Transmit packet with UID " << packet->GetUid ());
   874   NS_LOG_LOGIC ("Transmit packet from " << source);
   875   NS_LOG_LOGIC ("Transmit packet to " << destination);
   876 
   877   EthernetHeader header (false);
   878   header.SetSource (source);
   879   header.SetDestination (destination);
   880 
   881   switch (m_encapMode)
   882     {
   883     case LLC:
   884       {
   885         LlcSnapHeader llc;
   886         llc.SetType (protocolNumber);
   887         packet->AddHeader (llc);
   888 
   889         header.SetLengthType (packet->GetSize ());
   890       }
   891       break;
   892       
   893     case DIX:
   894       header.SetLengthType (protocolNumber);
   895       break;
   896       
   897     default:
   898       NS_FATAL_ERROR ("invalid encapsulation mode");
   899     }
   900   
   901   packet->AddHeader (header);
   902 
   903   //
   904   // there's not much meaning associated with the different layers in this
   905   // device, so don't be surprised when they're all stacked together in 
   906   // essentially one place.  We do this for trace consistency across devices.
   907   //
   908   m_macTxTrace (packet);
   909 
   910   // 
   911   // Enqueue and dequeue the packet to hit the queue tracing hooks.
   912   //
   913   m_queue->Enqueue (packet);
   914   packet = m_queue->Dequeue ();
   915   NS_ASSERT_MSG (packet, "EmuNetDevice::SendFrom(): packet zero from queue");
   916 
   917   m_promiscSnifferTrace (packet);
   918   m_snifferTrace (packet);
   919 
   920   struct sockaddr_ll ll;
   921   bzero (&ll, sizeof (ll));
   922 
   923   ll.sll_family = AF_PACKET;
   924   ll.sll_ifindex = m_sll_ifindex;
   925   ll.sll_protocol = htons(ETH_P_ALL); 
   926 
   927   NS_LOG_LOGIC ("calling sendto");
   928 
   929   NS_ASSERT_MSG (packet->GetSize () <= 65536, "EmuNetDevice::SendFrom(): Packet too big " << packet->GetSize ());
   930   packet->CopyData (m_packetBuffer, packet->GetSize ());
   931 
   932   int32_t rc = sendto (m_sock, m_packetBuffer, packet->GetSize (), 0, reinterpret_cast<struct sockaddr *> (&ll), sizeof (ll));
   933   NS_LOG_LOGIC ("sendto returns " << rc);
   934 
   935   return rc == -1 ? false : true;
   936 }
   937 
   938 void 
   939 EmuNetDevice::SetDataRate(DataRate bps)
   940 {
   941   NS_LOG_FUNCTION (this << bps);
   942   NS_FATAL_ERROR ("EmuNetDevice::SetDataRate():  Unable."); 
   943 }
   944 
   945 void
   946 EmuNetDevice::SetQueue (Ptr<Queue> q)
   947 {
   948   NS_LOG_FUNCTION (this << q);
   949   m_queue = q;
   950 }
   951 
   952 Ptr<Queue> 
   953 EmuNetDevice::GetQueue(void) const 
   954 { 
   955   NS_LOG_FUNCTION_NOARGS ();
   956   return m_queue;
   957 }
   958 
   959 void
   960 EmuNetDevice::NotifyLinkUp (void)
   961 {
   962   m_linkUp = true;
   963   m_linkChangeCallbacks ();
   964 }
   965 
   966 void 
   967 EmuNetDevice::SetIfIndex(const uint32_t index)
   968 {
   969   m_ifIndex = index;
   970 }
   971 
   972 uint32_t 
   973 EmuNetDevice::GetIfIndex(void) const
   974 {
   975   return m_ifIndex;
   976 }
   977 
   978 Ptr<Channel> 
   979 EmuNetDevice::GetChannel (void) const
   980 {
   981   NS_FATAL_ERROR ("EmuNetDevice::GetChannel():  Unable."); 
   982   return 0;
   983 }
   984 
   985 void 
   986 EmuNetDevice::SetAddress (Address address)
   987 {
   988   NS_LOG_FUNCTION (address);
   989   m_address = Mac48Address::ConvertFrom (address);
   990 }
   991 
   992 Address 
   993 EmuNetDevice::GetAddress (void) const
   994 {
   995   NS_LOG_FUNCTION_NOARGS ();
   996   return m_address;
   997 }
   998 
   999 bool 
  1000 EmuNetDevice::SetMtu (const uint16_t mtu)
  1001 {
  1002   NS_FATAL_ERROR ("EmuNetDevice::SetMtu():  Unable."); 
  1003   return false;
  1004 }
  1005 
  1006 uint16_t 
  1007 EmuNetDevice::GetMtu (void) const
  1008 {
  1009   struct ifreq ifr;
  1010   bzero (&ifr, sizeof (ifr));
  1011   strcpy(ifr.ifr_name, m_deviceName.c_str ());
  1012 
  1013   int32_t fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
  1014 
  1015 
  1016   int32_t rc = ioctl(fd, SIOCGIFMTU, &ifr);
  1017   if (rc == -1)
  1018     {
  1019       NS_FATAL_ERROR ("EmuNetDevice::GetMtu(): Can't ioctl SIOCGIFMTU");
  1020     }
  1021 
  1022   close (fd);
  1023 
  1024   return ifr.ifr_mtu;
  1025 }
  1026 
  1027 bool 
  1028 EmuNetDevice::IsLinkUp (void) const
  1029 {
  1030   return m_linkUp;
  1031 }
  1032 
  1033 void 
  1034 EmuNetDevice::AddLinkChangeCallback (Callback<void> callback)
  1035 {
  1036   m_linkChangeCallbacks.ConnectWithoutContext (callback);
  1037 }
  1038 
  1039 bool 
  1040 EmuNetDevice::IsBroadcast (void) const
  1041 {
  1042   return m_isBroadcast;
  1043 }
  1044 
  1045 Address
  1046 EmuNetDevice::GetBroadcast (void) const
  1047 {
  1048   return Mac48Address ("ff:ff:ff:ff:ff:ff");
  1049 }
  1050 
  1051 bool 
  1052 EmuNetDevice::IsMulticast (void) const
  1053 {
  1054   return m_isMulticast;
  1055 }
  1056 
  1057   Address 
  1058 EmuNetDevice::GetMulticast (Ipv4Address multicastGroup) const
  1059 {
  1060   NS_LOG_FUNCTION (multicastGroup);
  1061 
  1062   Mac48Address ad = Mac48Address::GetMulticast (multicastGroup);
  1063 
  1064   //
  1065   // Implicit conversion (operator Address ()) is defined for Mac48Address, so
  1066   // use it by just returning the EUI-48 address which is automagically converted
  1067   // to an Address.
  1068   //
  1069   NS_LOG_LOGIC ("multicast address is " << ad);
  1070 
  1071   return ad;
  1072 }
  1073 
  1074 Address
  1075 EmuNetDevice::GetMulticast (Ipv6Address addr) const
  1076 {
  1077   NS_LOG_FUNCTION(this << addr);
  1078 
  1079   Mac48Address ad = Mac48Address::GetMulticast (addr);
  1080   NS_LOG_LOGIC("MAC IPv6 multicast address is " << ad);
  1081 
  1082   return ad;
  1083 }
  1084 
  1085 bool 
  1086 EmuNetDevice::IsPointToPoint (void) const
  1087 {
  1088   return false;
  1089 }
  1090 
  1091 bool 
  1092 EmuNetDevice::IsBridge (void) const
  1093 {
  1094   return false;
  1095 }
  1096 
  1097 void
  1098 EmuNetDevice::SetPromiscReceiveCallback (PromiscReceiveCallback cb)
  1099 {
  1100   m_promiscRxCallback = cb;
  1101 }
  1102 
  1103   bool 
  1104 EmuNetDevice::SupportsSendFrom () const
  1105 {
  1106   NS_LOG_FUNCTION_NOARGS ();
  1107   return true;
  1108 }
  1109 
  1110 
  1111 Ptr<Node> 
  1112 EmuNetDevice::GetNode (void) const
  1113 {
  1114   return m_node;
  1115 }
  1116 
  1117 void 
  1118 EmuNetDevice::SetNode (Ptr<Node> node)
  1119 {
  1120   m_node = node;
  1121 }
  1122 
  1123 bool 
  1124 EmuNetDevice::NeedsArp (void) const
  1125 {
  1126   return true;
  1127 }
  1128 
  1129 void 
  1130 EmuNetDevice::SetReceiveCallback (NetDevice::ReceiveCallback cb)
  1131 {
  1132   m_rxCallback = cb;
  1133 }
  1134 
  1135 } // namespace ns3