A Discrete-Event Network Simulator
API
dpdk-net-device.cc
Go to the documentation of this file.
1 /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
2 /*
3  * Copyright (c) 2019 NITK Surathkal
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License version 2 as
7  * published by the Free Software Foundation;
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17  *
18  * Author: Harsh Patel <thadodaharsh10@gmail.com>
19  * Hrishikesh Hiraskar <hrishihiraskar@gmail.com>
20  * Mohit P. Tahiliani <tahiliani@nitk.edu.in>
21  */
22 
23 #include "dpdk-net-device.h"
24 
25 #include "ns3/log.h"
26 #include "ns3/net-device-queue-interface.h"
27 #include "ns3/simulator.h"
28 #include "ns3/uinteger.h"
29 
30 #include <sys/ioctl.h>
31 #include <sys/mman.h>
32 #include <sys/signal.h>
33 #include <unistd.h>
34 
35 #include <poll.h>
36 
37 #include <rte_eal.h>
38 #include <rte_ethdev.h>
39 #include <rte_common.h>
40 #include <rte_mempool.h>
41 #include <rte_mbuf.h>
42 #include <rte_malloc.h>
43 #include <rte_cycles.h>
44 #include <rte_port.h>
45 
46 namespace ns3 {
47 
48 NS_LOG_COMPONENT_DEFINE ("DpdkNetDevice");
49 
50 NS_OBJECT_ENSURE_REGISTERED (DpdkNetDevice);
51 
52 volatile bool DpdkNetDevice::m_forceQuit = false;
53 
54 TypeId
56 {
57  static TypeId tid = TypeId ("ns3::DpdkNetDevice")
59  .SetGroupName ("FdNetDevice")
60  .AddConstructor<DpdkNetDevice> ()
61  .AddAttribute ("TxTimeout",
62  "The time to wait before transmitting burst from Tx buffer.",
63  TimeValue (MicroSeconds (2000)),
65  MakeTimeChecker ())
66  .AddAttribute ("MaxRxBurst",
67  "Size of Rx Burst.",
68  UintegerValue (64),
70  MakeUintegerChecker<uint32_t> ())
71  .AddAttribute ("MaxTxBurst",
72  "Size of Tx Burst.",
73  UintegerValue (64),
75  MakeUintegerChecker<uint32_t> ())
76  .AddAttribute ("MempoolCacheSize",
77  "Size of mempool cache.",
78  UintegerValue (256),
80  MakeUintegerChecker<uint32_t> ())
81  .AddAttribute ("NbRxDesc",
82  "Number of Rx descriptors.",
83  UintegerValue (1024),
85  MakeUintegerChecker<uint16_t> ())
86  .AddAttribute ("NbTxDesc",
87  "Number of Tx descriptors.",
88  UintegerValue (1024),
90  MakeUintegerChecker<uint16_t> ())
91  ;
92  return tid;
93 }
94 
96  : m_mempool (NULL)
97 {
98  NS_LOG_FUNCTION (this);
99 }
100 
102 {
103  NS_LOG_FUNCTION (this);
105  m_forceQuit = true;
106 
107  rte_eal_wait_lcore (1);
108  rte_eth_dev_stop (m_portId);
109  rte_eth_dev_close (m_portId);
110 }
111 
112 void
113 DpdkNetDevice::SetDeviceName (std::string deviceName)
114 {
115  NS_LOG_FUNCTION (this);
116 
117  m_deviceName = deviceName;
118 }
119 
120 void
122 {
123  NS_LOG_FUNCTION (this);
124 
125  #define CHECK_INTERVAL 100 /* 100ms */
126  #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
127  uint8_t count, allPortsUp, printFlag = 0;
128  struct rte_eth_link link;
129 
130  for (count = 0; count <= MAX_CHECK_TIME; count++)
131  {
132 
133  allPortsUp = 1;
134 
135  if (m_forceQuit)
136  {
137  return;
138  }
139  if ((1 << m_portId) == 0)
140  {
141  continue;
142  }
143  memset (&link, 0, sizeof(link));
144  rte_eth_link_get (m_portId, &link);
145  /* print link status if flag set */
146  if (printFlag == 1)
147  {
148  if (link.link_status)
149  {
150  continue;
151  }
152  else
153  {
154  printf ("Port %d Link Down\n", m_portId);
155  }
156  continue;
157  }
158  /* clear allPortsUp flag if any link down */
159  if (link.link_status == ETH_LINK_DOWN)
160  {
161  allPortsUp = 0;
162  break;
163  }
164 
165  /* after finally printing all link status, get out */
166  if (printFlag == 1)
167  {
168  break;
169  }
170 
171  if (allPortsUp == 0)
172  {
173  fflush (stdout);
174  rte_delay_ms (CHECK_INTERVAL);
175  }
176 
177  /* set the printFlag if all ports up or timeout */
178  if (allPortsUp == 1 || count == (MAX_CHECK_TIME - 1))
179  {
180  printFlag = 1;
181  }
182  }
183 }
184 
185 void
187 {
188  if (signum == SIGINT || signum == SIGTERM)
189  {
190  printf ("\n\nSignal %d received, preparing to exit...\n",
191  signum);
192  m_forceQuit = true;
193  }
194 }
195 
196 void
198 {
199  int queueId = 0;
200  rte_eth_tx_buffer_flush (m_portId, queueId, m_txBuffer);
201 }
202 
203 void
205 {
206  int queueId = 0;
207  m_rxBuffer->length = rte_eth_rx_burst (m_portId,
208  queueId,
209  m_rxBuffer->pkts,
211 
212  for (uint16_t i = 0; i < m_rxBuffer->length; i++)
213  {
214  struct rte_mbuf *pkt = NULL;
215  pkt = m_rxBuffer->pkts[i];
216 
217  if (!pkt)
218  {
219  continue;
220  }
221 
222  uint8_t * buf = rte_pktmbuf_mtod (pkt, uint8_t *);
223  size_t length = pkt->data_len;
224  FdNetDevice::ReceiveCallback (buf,length);
225  }
226 
227  m_rxBuffer->length = 0;
228 }
229 
230 int
232 {
233  DpdkNetDevice *dpdkNetDevice = (DpdkNetDevice*) arg;
234  unsigned lcoreId;
235  lcoreId = rte_lcore_id ();
236  if (lcoreId != 1)
237  {
238  return 0;
239  }
240 
241  while (!m_forceQuit)
242  {
243  dpdkNetDevice->HandleRx ();
244  }
245 
246  return 0;
247 }
248 
249 bool
251 {
252  // Refer https://mails.dpdk.org/archives/users/2018-December/003822.html
253  return true;
254 }
255 
256 void
257 DpdkNetDevice::InitDpdk (int argc, char** argv, std::string dpdkDriver)
258 {
259  NS_LOG_FUNCTION (this << argc << argv);
260 
261  NS_LOG_INFO ("Binding device to DPDK");
262  std::string command;
263  command.append ("dpdk-devbind.py --force ");
264  command.append ("--bind=");
265  command.append (dpdkDriver.c_str ());
266  command.append (" ");
267  command.append (m_deviceName.c_str ());
268  printf ("Executing: %s\n", command.c_str ());
269  if (system (command.c_str ()))
270  {
271  rte_exit (EXIT_FAILURE, "Execution failed - bye\n");
272  }
273 
274  // wait for the device to bind to Dpdk
275  sleep (5); /* 5 seconds */
276 
277  NS_LOG_INFO ("Initialize DPDK EAL");
278  int ret = rte_eal_init (argc, argv);
279  if (ret < 0)
280  {
281  rte_exit (EXIT_FAILURE, "Invalid EAL arguments\n");
282  }
283 
284  m_forceQuit = false;
285  signal (SIGINT, SignalHandler);
286  signal (SIGTERM, SignalHandler);
287 
288  unsigned nbPorts = rte_eth_dev_count_avail ();
289  if (nbPorts == 0)
290  {
291  rte_exit (EXIT_FAILURE, "No Ethernet ports - bye\n");
292  }
293 
294  NS_LOG_INFO ("Get port id of the device");
295  if (rte_eth_dev_get_port_by_name (m_deviceName.c_str (), &m_portId) != 0)
296  {
297  rte_exit (EXIT_FAILURE, "Cannot get port id - bye\n");
298  }
299 
300  // Set number of logical cores to 2
301  unsigned int nbLcores = 2;
302 
303  unsigned int nbMbufs = RTE_MAX (nbPorts * (m_nbRxDesc + m_nbTxDesc + m_maxRxPktBurst +
305  nbLcores * m_mempoolCacheSize),
306  8192U);
307 
308  NS_LOG_INFO ("Create the mbuf pool");
309  m_mempool = rte_pktmbuf_pool_create ("mbuf_pool", nbMbufs,
311  RTE_MBUF_DEFAULT_BUF_SIZE,
312  rte_socket_id ());
313 
314  if (m_mempool == NULL)
315  {
316  rte_exit (EXIT_FAILURE, "Cannot init mbuf pool\n");
317  }
318 
319  NS_LOG_INFO ("Initialize port");
320  static struct rte_eth_conf portConf = {};
321  portConf.rxmode = {};
322  portConf.rxmode.split_hdr_size = 0;
323  portConf.txmode = {};
324  portConf.txmode.mq_mode = ETH_MQ_TX_NONE;
325 
326  struct rte_eth_rxconf reqConf;
327  struct rte_eth_txconf txqConf;
328  struct rte_eth_conf localPortConf = portConf;
329  struct rte_eth_dev_info devInfo;
330 
331  fflush (stdout);
332  rte_eth_dev_info_get (m_portId, &devInfo);
333  if (devInfo.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
334  {
335  localPortConf.txmode.offloads |=
336  DEV_TX_OFFLOAD_MBUF_FAST_FREE;
337  }
338  ret = rte_eth_dev_configure (m_portId, 1, 1, &localPortConf);
339  if (ret < 0)
340  {
341  rte_exit (EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n",
342  ret, m_portId);
343  }
344 
345  ret = rte_eth_dev_adjust_nb_rx_tx_desc (m_portId, &m_nbRxDesc, &m_nbTxDesc);
346  if (ret < 0)
347  {
348  rte_exit (EXIT_FAILURE,
349  "Cannot adjust number of descriptors: err=%d, port=%u\n",
350  ret, m_portId);
351  }
352 
353  NS_LOG_INFO ("Initialize one Rx queue");
354  fflush (stdout);
355  reqConf = devInfo.default_rxconf;
356  reqConf.offloads = localPortConf.rxmode.offloads;
357  ret = rte_eth_rx_queue_setup (m_portId, 0, m_nbRxDesc,
358  rte_eth_dev_socket_id (m_portId),
359  &reqConf,
360  m_mempool);
361  if (ret < 0)
362  {
363  rte_exit (EXIT_FAILURE, "rte_eth_rx_queue_setup:err=%d, port=%u\n",
364  ret, m_portId);
365  }
366 
367  NS_LOG_INFO ("Initialize one Tx queue per port");
368  fflush (stdout);
369  txqConf = devInfo.default_txconf;
370  txqConf.offloads = localPortConf.txmode.offloads;
371  ret = rte_eth_tx_queue_setup (m_portId, 0, m_nbTxDesc,
372  rte_eth_dev_socket_id (m_portId),
373  &txqConf);
374  if (ret < 0)
375  {
376  rte_exit (EXIT_FAILURE, "rte_eth_tx_queue_setup:err=%d, port=%u\n",
377  ret, m_portId);
378  }
379 
380  NS_LOG_INFO ("Initialize Tx buffers");
381  m_txBuffer = (rte_eth_dev_tx_buffer*)
382  rte_zmalloc_socket ("tx_buffer",
383  RTE_ETH_TX_BUFFER_SIZE (m_maxTxPktBurst), 0,
384  rte_eth_dev_socket_id (m_portId));
385  NS_LOG_INFO ("Initialize Rx buffers");
386  m_rxBuffer = (rte_eth_dev_tx_buffer*)
387  rte_zmalloc_socket ("rx_buffer",
388  RTE_ETH_TX_BUFFER_SIZE (m_maxRxPktBurst), 0,
389  rte_eth_dev_socket_id (m_portId));
390  if (m_txBuffer == NULL || m_rxBuffer == NULL)
391  {
392  rte_exit (EXIT_FAILURE, "Cannot allocate buffer for rx/tx on port %u\n",
393  m_portId);
394  }
395 
396  rte_eth_tx_buffer_init (m_txBuffer, m_maxTxPktBurst);
397  rte_eth_tx_buffer_init (m_rxBuffer, m_maxRxPktBurst);
398 
399  NS_LOG_INFO ("Start the device");
400  ret = rte_eth_dev_start (m_portId);
401  if (ret < 0)
402  {
403  rte_exit (EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
404  ret, m_portId);
405  }
406 
407  rte_eth_promiscuous_enable (m_portId);
408 
410 
411  NS_LOG_INFO ("Launching core threads");
412  rte_eal_mp_remote_launch (LaunchCore, this, CALL_MASTER);
413 }
414 
415 uint8_t*
417 {
418  struct rte_mbuf *pkt = rte_pktmbuf_alloc (m_mempool);
419  if (!pkt)
420  {
421  return NULL;
422  }
423  uint8_t *buf = rte_pktmbuf_mtod (pkt, uint8_t *);
424  return buf;
425 }
426 
427 void
429 {
430  struct rte_mbuf *pkt;
431 
432  if (!buf)
433  {
434  return;
435  }
436  pkt = (struct rte_mbuf *)
437  RTE_PTR_SUB ( buf,
438  sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM);
439 
440  rte_pktmbuf_free (pkt);
441 }
442 
443 ssize_t
444 DpdkNetDevice::Write (uint8_t *buffer, size_t length)
445 {
446  struct rte_mbuf ** pkt = new struct rte_mbuf*[1];
447  int queueId = 0;
448 
449  if (buffer == NULL || m_txBuffer->length == m_maxTxPktBurst)
450  {
451  NS_LOG_ERROR ("Error allocating mbuf" << buffer);
452  return -1;
453  }
454 
455  pkt[0] = (struct rte_mbuf *)
456  RTE_PTR_SUB ( buffer,
457  sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM);
458 
459  pkt[0]->pkt_len = length;
460  pkt[0]->data_len = length;
461  rte_eth_tx_buffer (m_portId, queueId, m_txBuffer, pkt[0]);
462 
463  if (m_txBuffer->length == 1)
464  {
465  // If this is a first packet in buffer, schedule a tx.
468  }
469 
470  return length;
471 }
472 
473 void
475 {
477 
478  while (!m_pendingQueue.empty ())
479  {
480  std::pair<uint8_t *, ssize_t> next = m_pendingQueue.front ();
481  m_pendingQueue.pop ();
482 
483  FreeBuffer (next.first);
484  }
485 }
486 
487 } // namespace ns3
ns3::TypeId
a unique identifier for an interface.
Definition: type-id.h:59
NS_LOG_COMPONENT_DEFINE
#define NS_LOG_COMPONENT_DEFINE(name)
Define a Log component with a specific name.
Definition: log.h:205
ns3::DpdkNetDevice::DoFinishStoppingDevice
void DoFinishStoppingDevice(void)
Complete additional actions, if any, to tear down the device.
Definition: dpdk-net-device.cc:474
ns3::DpdkNetDevice::m_rxBuffer
struct rte_eth_dev_tx_buffer * m_rxBuffer
Buffer to handle burst reception.
Definition: dpdk-net-device.h:172
NS_OBJECT_ENSURE_REGISTERED
#define NS_OBJECT_ENSURE_REGISTERED(type)
Register an Object subclass with the TypeId system.
Definition: object-base.h:45
CHECK_INTERVAL
#define CHECK_INTERVAL
ns3::MakeTimeChecker
Ptr< const AttributeChecker > MakeTimeChecker(const Time min, const Time max)
Helper to make a Time checker with bounded range.
Definition: time.cc:533
ns3::DpdkNetDevice::LaunchCore
static int LaunchCore(void *arg)
A function to handle rx & tx operations.
Definition: dpdk-net-device.cc:231
ns3::DpdkNetDevice::m_deviceName
std::string m_deviceName
The device name;.
Definition: dpdk-net-device.h:150
ns3::DpdkNetDevice::AllocateBuffer
virtual uint8_t * AllocateBuffer(size_t len)
Allocate packet buffer.
Definition: dpdk-net-device.cc:416
ns3
Every class exported by the ns3 library is enclosed in the ns3 namespace.
ns3::MicroSeconds
Time MicroSeconds(uint64_t value)
Construct a Time in the indicated unit.
Definition: nstime.h:1305
ns3::DpdkNetDevice::IsLinkUp
bool IsLinkUp(void) const
Check the status of the link.
Definition: dpdk-net-device.cc:250
ns3::DpdkNetDevice::InitDpdk
void InitDpdk(int argc, char **argv, std::string dpdkDriver)
Initialize Dpdk.
Definition: dpdk-net-device.cc:257
ns3::DpdkNetDevice::m_maxRxPktBurst
uint32_t m_maxRxPktBurst
Size of Rx burst.
Definition: dpdk-net-device.h:187
ns3::FdNetDevice::m_pendingQueue
std::queue< std::pair< uint8_t *, ssize_t > > m_pendingQueue
Number of packets that were received and scheduled for read but not yet read.
Definition: fd-net-device.h:241
ns3::Simulator::Schedule
static EventId Schedule(Time const &delay, FUNC f, Ts &&... args)
Schedule an event to expire after delay.
Definition: simulator.h:556
ns3::DpdkNetDevice::DpdkNetDevice
DpdkNetDevice()
Constructor for the DpdkNetDevice.
Definition: dpdk-net-device.cc:95
ns3::TypeId::SetParent
TypeId SetParent(TypeId tid)
Set the parent TypeId.
Definition: type-id.cc:923
ns3::DpdkNetDevice::m_mempoolCacheSize
uint32_t m_mempoolCacheSize
Mempool cache size.
Definition: dpdk-net-device.h:197
ns3::DpdkNetDevice::CheckAllPortsLinkStatus
void CheckAllPortsLinkStatus(void)
Check the link status of all ports in up to 9s and print them finally.
Definition: dpdk-net-device.cc:121
ns3::NetDevice::ReceiveCallback
Callback< bool, Ptr< NetDevice >, Ptr< const Packet >, uint16_t, const Address & > ReceiveCallback
Definition: net-device.h:318
ns3::Simulator::Cancel
static void Cancel(const EventId &id)
Set the cancel bit on this event: the event's associated function will not be invoked when it expires...
Definition: simulator.cc:268
NS_LOG_INFO
#define NS_LOG_INFO(msg)
Use NS_LOG to output a message of level LOG_INFO.
Definition: log.h:281
ns3::CriticalSection
A class which provides a simple way to implement a Critical Section.
Definition: system-mutex.h:119
ns3::DpdkNetDevice::m_portId
uint16_t m_portId
The port number of the device to be used.
Definition: dpdk-net-device.h:145
ns3::DpdkNetDevice::HandleTx
void HandleTx()
Transmit packets in burst from the tx_buffer to the nic.
Definition: dpdk-net-device.cc:197
ns3::DpdkNetDevice::m_mempool
struct rte_mempool * m_mempool
Packet memory pool.
Definition: dpdk-net-device.h:162
ns3::DpdkNetDevice::HandleRx
void HandleRx()
Receive packets in burst from the nic to the rx_buffer.
Definition: dpdk-net-device.cc:204
ns3::DpdkNetDevice::~DpdkNetDevice
~DpdkNetDevice()
Destructor for the DpdkNetDevice.
Definition: dpdk-net-device.cc:101
ns3::DpdkNetDevice::GetTypeId
static TypeId GetTypeId(void)
Get the type ID.
Definition: dpdk-net-device.cc:55
ns3::DpdkNetDevice
a NetDevice to read/write network traffic from/into a Dpdk enabled port.
Definition: dpdk-net-device.h:49
NS_LOG_ERROR
#define NS_LOG_ERROR(msg)
Use NS_LOG to output a message of level LOG_ERROR.
Definition: log.h:257
ns3::FdNetDevice
a NetDevice to read/write network traffic from/into a file descriptor.
Definition: fd-net-device.h:86
ns3::DpdkNetDevice::m_txEvent
EventId m_txEvent
Event for stale packet transmission.
Definition: dpdk-net-device.h:177
ns3::DpdkNetDevice::SetDeviceName
void SetDeviceName(std::string deviceName)
Set device name.
Definition: dpdk-net-device.cc:113
ns3::FdNetDevice::m_pendingReadMutex
SystemMutex m_pendingReadMutex
Mutex to increase pending read counter.
Definition: fd-net-device.h:236
ns3::DpdkNetDevice::m_maxTxPktBurst
uint32_t m_maxTxPktBurst
Size of Tx burst.
Definition: dpdk-net-device.h:192
ns3::DpdkNetDevice::Write
ssize_t Write(uint8_t *buffer, size_t length)
Write packet data to device.
Definition: dpdk-net-device.cc:444
ns3::TimeValue
AttributeValue implementation for Time.
Definition: nstime.h:1353
NS_LOG_FUNCTION
#define NS_LOG_FUNCTION(parameters)
If log level LOG_FUNCTION is enabled, this macro will output all input parameters separated by ",...
Definition: log-macros-enabled.h:244
ns3::DpdkNetDevice::m_nbRxDesc
uint16_t m_nbRxDesc
Number of Rx descriptors.
Definition: dpdk-net-device.h:202
ns3::UintegerValue
Hold an unsigned integer type.
Definition: uinteger.h:44
ns3::DpdkNetDevice::m_nbTxDesc
uint16_t m_nbTxDesc
Number of Tx descriptors.
Definition: dpdk-net-device.h:207
ns3::DpdkNetDevice::FreeBuffer
virtual void FreeBuffer(uint8_t *buf)
Free the given packet buffer.
Definition: dpdk-net-device.cc:428
ns3::MakeUintegerAccessor
Ptr< const AttributeAccessor > MakeUintegerAccessor(T1 a1)
Create an AttributeAccessor for a class data member, or a lone class get functor or set method.
Definition: uinteger.h:45
ns3::DpdkNetDevice::m_forceQuit
static volatile bool m_forceQuit
Condition variable for Dpdk to stop.
Definition: dpdk-net-device.h:157
MAX_CHECK_TIME
#define MAX_CHECK_TIME
dpdk-net-device.h
ns3::DpdkNetDevice::SignalHandler
static void SignalHandler(int signum)
A signal handler for SIGINT and SIGTERM signals.
Definition: dpdk-net-device.cc:186
ns3::MakeTimeAccessor
Ptr< const AttributeAccessor > MakeTimeAccessor(T1 a1)
Create an AttributeAccessor for a class data member, or a lone class get functor or set method.
Definition: nstime.h:1354
ns3::DpdkNetDevice::m_txBuffer
struct rte_eth_dev_tx_buffer * m_txBuffer
Buffer to handle burst transmission.
Definition: dpdk-net-device.h:167
ns3::DpdkNetDevice::m_txTimeout
Time m_txTimeout
The time to wait before transmitting burst from Tx buffer.
Definition: dpdk-net-device.h:182