A Discrete-Event Network Simulator
API
simple-distributed-mpi-comm.cc
Go to the documentation of this file.
1/*
2 * Copyright 2018. Lawrence Livermore National Security, LLC.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation;
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16 *
17 * Author: Steven Smith <smith84@llnl.gov>
18 */
19
57#include "mpi-test-fixtures.h"
58#include "mpi.h"
59
60#include "ns3/core-module.h"
61#include "ns3/internet-stack-helper.h"
62#include "ns3/ipv4-address-helper.h"
63#include "ns3/ipv4-global-routing-helper.h"
64#include "ns3/ipv4-list-routing-helper.h"
65#include "ns3/ipv4-static-routing-helper.h"
66#include "ns3/mpi-interface.h"
67#include "ns3/network-module.h"
68#include "ns3/nix-vector-helper.h"
69#include "ns3/on-off-helper.h"
70#include "ns3/packet-sink-helper.h"
71#include "ns3/packet-sink.h"
72#include "ns3/point-to-point-helper.h"
73
74using namespace ns3;
75
76NS_LOG_COMPONENT_DEFINE("SimpleDistributedMpiComm");
77
83const int NS_COLOR = 1;
84const int NOT_NS_COLOR = NS_COLOR + 1;
85
94void
95ReportRank(int color, MPI_Comm splitComm)
96{
97 int otherId = 0;
98 int otherSize = 1;
99
100 MPI_Comm_rank(splitComm, &otherId);
101 MPI_Comm_size(splitComm, &otherSize);
102
103 if (color == NS_COLOR)
104 {
105 RANK0COUT("ns-3 rank: ");
106 }
107 else
108 {
109 RANK0COUT("Other rank: ");
110 }
111
112 RANK0COUTAPPEND("in MPI_COMM_WORLD: " << SinkTracer::GetWorldRank() << ":"
113 << SinkTracer::GetWorldSize() << ", in splitComm: "
114 << otherId << ":" << otherSize << std::endl);
115
116} // ReportRank()
117
118int
119main(int argc, char* argv[])
120{
121 bool nix = true;
122 bool nullmsg = false;
123 bool tracing = false;
124 bool init = false;
125 bool verbose = false;
126 bool testing = false;
127
128 // Parse command line
129 CommandLine cmd(__FILE__);
130 cmd.AddValue("nix", "Enable the use of nix-vector or global routing", nix);
131 cmd.AddValue("nullmsg",
132 "Enable the use of null-message synchronization (instead of granted time window)",
133 nullmsg);
134 cmd.AddValue("tracing", "Enable pcap tracing", tracing);
135 cmd.AddValue("init", "ns-3 should initialize MPI by calling MPI_Init", init);
136 cmd.AddValue("verbose", "verbose output", verbose);
137 cmd.AddValue("test", "Enable regression test output", testing);
138 cmd.Parse(argc, argv);
139
140 // Defer reporting the configuration until we know the communicator
141
142 // Distributed simulation setup; by default use granted time window algorithm.
143 if (nullmsg)
144 {
145 GlobalValue::Bind("SimulatorImplementationType",
146 StringValue("ns3::NullMessageSimulatorImpl"));
147 }
148 else
149 {
150 GlobalValue::Bind("SimulatorImplementationType",
151 StringValue("ns3::DistributedSimulatorImpl"));
152 }
153
154 // MPI_Init
155
156 if (init)
157 {
158 // Initialize MPI directly
159 MPI_Init(&argc, &argv);
160 }
161 else
162 {
163 // Let ns-3 call MPI_Init and MPI_Finalize
164 MpiInterface::Enable(&argc, &argv);
165 }
166
167 SinkTracer::Init();
168
169 auto worldSize = SinkTracer::GetWorldSize();
170 auto worldRank = SinkTracer::GetWorldRank();
171
172 if ((!init) && (worldSize != 2))
173 {
174 RANK0COUT("This simulation requires exactly 2 logical processors if --init is not set."
175 << std::endl);
176 return 1;
177 }
178
179 if (worldSize < 2)
180 {
181 RANK0COUT("This simulation requires 2 or more logical processors." << std::endl);
182 return 1;
183 }
184
185 // Set up the MPI communicator for ns-3
186 // Condition ns-3 Communicator
187 // a. worldSize = 2 copy of MPI_COMM_WORLD
188 // b. worldSize > 2 communicator of ranks 1-2
189
190 // Flag to record that we created a communicator so we can free it at the end.
191 bool freeComm = false;
192 // The new communicator, if we create one
193 MPI_Comm splitComm = MPI_COMM_WORLD;
194 // The list of ranks assigned to ns-3
195 std::string ns3Ranks;
196 // Tag for whether this rank should go into a new communicator
197 int color = MPI_UNDEFINED;
198
199 if (worldSize == 2)
200 {
201 std::stringstream ss;
202 color = NS_COLOR;
203 ss << "MPI_COMM_WORLD (" << worldSize << " ranks)";
204 ns3Ranks = ss.str();
205 splitComm = MPI_COMM_WORLD;
206 freeComm = false;
207 }
208 else
209 {
210 // worldSize > 2 communicator of ranks 1-2
211
212 // Put ranks 1-2 in the new communicator
213 if (worldRank == 1 || worldRank == 2)
214 {
215 color = NS_COLOR;
216 }
217 else
218 {
219 color = NOT_NS_COLOR;
220 }
221 std::stringstream ss;
222 ss << "Split [1-2] (out of " << worldSize << " ranks) from MPI_COMM_WORLD";
223 ns3Ranks = ss.str();
224
225 // Now create the new communicator
226 MPI_Comm_split(MPI_COMM_WORLD, color, worldRank, &splitComm);
227 freeComm = true;
228 }
229
230 if (init)
231 {
232 MpiInterface::Enable(splitComm);
233 }
234
235 // Report the configuration from rank 0 only
236 RANK0COUT(cmd.GetName() << "\n");
237 RANK0COUT("\n");
238 RANK0COUT("Configuration:\n");
239 RANK0COUT("Routing: " << (nix ? "nix-vector" : "global") << "\n");
240 RANK0COUT("Synchronization: " << (nullmsg ? "null-message" : "granted time window (YAWNS)")
241 << "\n");
242 RANK0COUT("MPI_Init called: "
243 << (init ? "explicitly by this program" : "implicitly by ns3::MpiInterface::Enable()")
244 << "\n");
245 RANK0COUT("ns-3 Communicator: " << ns3Ranks << "\n");
246 RANK0COUT("PCAP tracing: " << (tracing ? "" : "not") << " enabled\n");
247 RANK0COUT("\n");
248 RANK0COUT("Rank assignments:" << std::endl);
249
250 if (worldRank == 0)
251 {
252 ReportRank(color, splitComm);
253 }
254
255 if (verbose)
256 {
257 // Circulate a token to have each rank report in turn
258 int token;
259
260 if (worldRank == 0)
261 {
262 token = 1;
263 }
264 else
265 {
266 MPI_Recv(&token, 1, MPI_INT, worldRank - 1, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
267 ReportRank(color, splitComm);
268 }
269
270 MPI_Send(&token, 1, MPI_INT, (worldRank + 1) % worldSize, 0, MPI_COMM_WORLD);
271
272 if (worldRank == 0)
273 {
274 MPI_Recv(&token, 1, MPI_INT, worldSize - 1, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
275 }
276 } // circulate token to report rank
277
278 RANK0COUT(std::endl);
279
280 if (color != NS_COLOR)
281 {
282 // Do other work outside the ns-3 communicator
283
284 // In real use of a separate communicator from ns-3
285 // the other tasks would be running another simulator
286 // or other desired work here..
287
288 // Our work is done, just wait for everyone else to finish.
289
290 MpiInterface::Disable();
291
292 if (init)
293 {
294 MPI_Finalize();
295 }
296
297 return 0;
298 }
299
300 // The code below here is essentially the same as simple-distributed.cc
301 // --------------------------------------------------------------------
302
303 // We use a trace instead of relying on NS_LOG
304
305 if (verbose)
306 {
307 LogComponentEnable("PacketSink", LOG_LEVEL_INFO);
308 }
309
310 uint32_t systemId = MpiInterface::GetSystemId();
311 uint32_t systemCount = MpiInterface::GetSize();
312
313 // Check for valid distributed parameters.
314 // Both this script and simple-distributed.cc will work
315 // with arbitrary numbers of ranks, as long as there are at least 2.
316 if (systemCount < 2)
317 {
318 RANK0COUT("This simulation requires at least 2 logical processors." << std::endl);
319 return 1;
320 }
321
322 // Some default values
323 Config::SetDefault("ns3::OnOffApplication::PacketSize", UintegerValue(512));
324 Config::SetDefault("ns3::OnOffApplication::DataRate", StringValue("1Mbps"));
325 Config::SetDefault("ns3::OnOffApplication::MaxBytes", UintegerValue(512));
326
327 // Create leaf nodes on left with system id 0
328 NodeContainer leftLeafNodes;
329 leftLeafNodes.Create(4, 0);
330
331 // Create router nodes. Left router
332 // with system id 0, right router with
333 // system id 1
334 NodeContainer routerNodes;
335 Ptr<Node> routerNode1 = CreateObject<Node>(0);
336 Ptr<Node> routerNode2 = CreateObject<Node>(1);
337 routerNodes.Add(routerNode1);
338 routerNodes.Add(routerNode2);
339
340 // Create leaf nodes on left with system id 1
341 NodeContainer rightLeafNodes;
342 rightLeafNodes.Create(4, 1);
343
344 PointToPointHelper routerLink;
345 routerLink.SetDeviceAttribute("DataRate", StringValue("5Mbps"));
346 routerLink.SetChannelAttribute("Delay", StringValue("5ms"));
347
348 PointToPointHelper leafLink;
349 leafLink.SetDeviceAttribute("DataRate", StringValue("1Mbps"));
350 leafLink.SetChannelAttribute("Delay", StringValue("2ms"));
351
352 // Add link connecting routers
353 NetDeviceContainer routerDevices;
354 routerDevices = routerLink.Install(routerNodes);
355
356 // Add links for left side leaf nodes to left router
357 NetDeviceContainer leftRouterDevices;
358 NetDeviceContainer leftLeafDevices;
359 for (uint32_t i = 0; i < 4; ++i)
360 {
361 NetDeviceContainer temp = leafLink.Install(leftLeafNodes.Get(i), routerNodes.Get(0));
362 leftLeafDevices.Add(temp.Get(0));
363 leftRouterDevices.Add(temp.Get(1));
364 }
365
366 // Add links for right side leaf nodes to right router
367 NetDeviceContainer rightRouterDevices;
368 NetDeviceContainer rightLeafDevices;
369 for (uint32_t i = 0; i < 4; ++i)
370 {
371 NetDeviceContainer temp = leafLink.Install(rightLeafNodes.Get(i), routerNodes.Get(1));
372 rightLeafDevices.Add(temp.Get(0));
373 rightRouterDevices.Add(temp.Get(1));
374 }
375
377 Ipv4NixVectorHelper nixRouting;
378 Ipv4StaticRoutingHelper staticRouting;
379
381 list.Add(staticRouting, 0);
382 list.Add(nixRouting, 10);
383
384 if (nix)
385 {
386 stack.SetRoutingHelper(list); // has effect on the next Install ()
387 }
388
389 stack.InstallAll();
390
391 Ipv4InterfaceContainer routerInterfaces;
392 Ipv4InterfaceContainer leftLeafInterfaces;
393 Ipv4InterfaceContainer leftRouterInterfaces;
394 Ipv4InterfaceContainer rightLeafInterfaces;
395 Ipv4InterfaceContainer rightRouterInterfaces;
396
397 Ipv4AddressHelper leftAddress;
398 leftAddress.SetBase("10.1.1.0", "255.255.255.0");
399
400 Ipv4AddressHelper routerAddress;
401 routerAddress.SetBase("10.2.1.0", "255.255.255.0");
402
403 Ipv4AddressHelper rightAddress;
404 rightAddress.SetBase("10.3.1.0", "255.255.255.0");
405
406 // Router-to-Router interfaces
407 routerInterfaces = routerAddress.Assign(routerDevices);
408
409 // Left interfaces
410 for (uint32_t i = 0; i < 4; ++i)
411 {
413 ndc.Add(leftLeafDevices.Get(i));
414 ndc.Add(leftRouterDevices.Get(i));
415 Ipv4InterfaceContainer ifc = leftAddress.Assign(ndc);
416 leftLeafInterfaces.Add(ifc.Get(0));
417 leftRouterInterfaces.Add(ifc.Get(1));
418 leftAddress.NewNetwork();
419 }
420
421 // Right interfaces
422 for (uint32_t i = 0; i < 4; ++i)
423 {
425 ndc.Add(rightLeafDevices.Get(i));
426 ndc.Add(rightRouterDevices.Get(i));
427 Ipv4InterfaceContainer ifc = rightAddress.Assign(ndc);
428 rightLeafInterfaces.Add(ifc.Get(0));
429 rightRouterInterfaces.Add(ifc.Get(1));
430 rightAddress.NewNetwork();
431 }
432
433 if (!nix)
434 {
435 Ipv4GlobalRoutingHelper::PopulateRoutingTables();
436 }
437
438 if (tracing == true)
439 {
440 if (systemId == 0)
441 {
442 routerLink.EnablePcap("router-left", routerDevices, true);
443 leafLink.EnablePcap("leaf-left", leftLeafDevices, true);
444 }
445
446 if (systemId == 1)
447 {
448 routerLink.EnablePcap("router-right", routerDevices, true);
449 leafLink.EnablePcap("leaf-right", rightLeafDevices, true);
450 }
451 }
452
453 // Create a packet sink on the right leafs to receive packets from left leafs
454 uint16_t port = 50000;
455 if (systemId == 1)
456 {
457 Address sinkLocalAddress(InetSocketAddress(Ipv4Address::GetAny(), port));
458 PacketSinkHelper sinkHelper("ns3::UdpSocketFactory", sinkLocalAddress);
459 ApplicationContainer sinkApp;
460 for (uint32_t i = 0; i < 4; ++i)
461 {
462 auto apps = sinkHelper.Install(rightLeafNodes.Get(i));
463 auto sink = DynamicCast<PacketSink>(apps.Get(0));
464 NS_ASSERT_MSG(sink, "Couldn't get PacketSink application.");
465 if (testing)
466 {
467 sink->TraceConnectWithoutContext("RxWithAddresses",
468 MakeCallback(&SinkTracer::SinkTrace));
469 }
470 sinkApp.Add(apps);
471 }
472 sinkApp.Start(Seconds(1.0));
473 sinkApp.Stop(Seconds(5));
474 }
475
476 // Create the OnOff applications to send
477 if (systemId == 0)
478 {
479 OnOffHelper clientHelper("ns3::UdpSocketFactory", Address());
480 clientHelper.SetAttribute("OnTime", StringValue("ns3::ConstantRandomVariable[Constant=1]"));
481 clientHelper.SetAttribute("OffTime",
482 StringValue("ns3::ConstantRandomVariable[Constant=0]"));
483
485 for (uint32_t i = 0; i < 4; ++i)
486 {
487 AddressValue remoteAddress(InetSocketAddress(rightLeafInterfaces.GetAddress(i), port));
488 clientHelper.SetAttribute("Remote", remoteAddress);
489 clientApps.Add(clientHelper.Install(leftLeafNodes.Get(i)));
490 }
491 clientApps.Start(Seconds(1.0));
492 clientApps.Stop(Seconds(5));
493 }
494
495 RANK0COUT(std::endl);
496
497 Simulator::Stop(Seconds(5));
498 Simulator::Run();
499 Simulator::Destroy();
500
501 // --------------------------------------------------------------------
502 // Conditional cleanup based on whether we built a communicator
503 // and called MPI_Init
504
505 if (freeComm)
506 {
507 MPI_Comm_free(&splitComm);
508 }
509
510 if (testing)
511 {
512 SinkTracer::Verify(4);
513 }
514
515 // Clean up the ns-3 MPI execution environment
516 // This will call MPI_Finalize if MpiInterface::Initialize was called
517 MpiInterface::Disable();
518
519 if (init)
520 {
521 // We called MPI_Init, so we have to call MPI_Finalize
522 MPI_Finalize();
523 }
524
525 return 0;
526}
a polymophic address class
Definition: address.h:92
AttributeValue implementation for Address.
holds a vector of ns3::Application pointers.
void Start(Time start)
Arrange for all of the Applications in this container to Start() at the Time given as a parameter.
void Add(ApplicationContainer other)
Append the contents of another ApplicationContainer to the end of this container.
void Stop(Time stop)
Arrange for all of the Applications in this container to Stop() at the Time given as a parameter.
Parse command-line arguments.
Definition: command-line.h:232
an Inet address class
aggregate IP/TCP/UDP functionality to existing Nodes.
A helper class to make life easier while doing simple IPv4 address assignment in scripts.
void SetBase(Ipv4Address network, Ipv4Mask mask, Ipv4Address base="0.0.0.1")
Set the base network number, network mask and base address.
Ipv4Address NewNetwork()
Increment the network number and reset the IP address counter to the base value provided in the SetBa...
Ipv4InterfaceContainer Assign(const NetDeviceContainer &c)
Assign IP addresses to the net devices specified in the container based on the current network prefix...
holds a vector of std::pair of Ptr<Ipv4> and interface index.
std::pair< Ptr< Ipv4 >, uint32_t > Get(uint32_t i) const
Get the std::pair of an Ptr<Ipv4> and interface stored at the location specified by the index.
void Add(const Ipv4InterfaceContainer &other)
Concatenate the entries in the other container with ours.
Ipv4Address GetAddress(uint32_t i, uint32_t j=0) const
Helper class that adds ns3::Ipv4ListRouting objects.
Helper class that adds ns3::Ipv4StaticRouting objects.
holds a vector of ns3::NetDevice pointers
void Add(NetDeviceContainer other)
Append the contents of another NetDeviceContainer to the end of this container.
Ptr< NetDevice > Get(uint32_t i) const
Get the Ptr<NetDevice> stored in this container at a given index.
Helper class that adds Nix-vector routing to nodes.
keep track of a set of node pointers.
void Create(uint32_t n)
Create n nodes and append pointers to them to the end of this NodeContainer.
void Add(const NodeContainer &nc)
Append the contents of another NodeContainer to the end of this container.
Ptr< Node > Get(uint32_t i) const
Get the Ptr<Node> stored in this container at a given index.
bool TraceConnectWithoutContext(std::string name, const CallbackBase &cb)
Connect a TraceSource to a Callback without a context.
Definition: object-base.cc:369
A helper to make it easier to instantiate an ns3::OnOffApplication on a set of nodes.
Definition: on-off-helper.h:44
A helper to make it easier to instantiate an ns3::PacketSinkApplication on a set of nodes.
void EnablePcap(std::string prefix, Ptr< NetDevice > nd, bool promiscuous=false, bool explicitFilename=false)
Enable pcap output the indicated net device.
Build a set of PointToPointNetDevice objects.
void SetDeviceAttribute(std::string name, const AttributeValue &value)
Set an attribute value to be propagated to each NetDevice created by the helper.
void SetChannelAttribute(std::string name, const AttributeValue &value)
Set an attribute value to be propagated to each Channel created by the helper.
NetDeviceContainer Install(NodeContainer c)
Hold variables of type string.
Definition: string.h:42
Hold an unsigned integer type.
Definition: uinteger.h:45
uint16_t port
Definition: dsdv-manet.cc:45
#define NS_ASSERT_MSG(condition, message)
At runtime, in debugging builds, if this condition is not true, the program prints the message to out...
Definition: assert.h:86
void SetDefault(std::string name, const AttributeValue &value)
Definition: config.cc:891
#define NS_LOG_COMPONENT_DEFINE(name)
Define a Log component with a specific name.
Definition: log.h:202
#define RANK0COUT(x)
Write to std::cout only from rank 0.
#define RANK0COUTAPPEND(x)
Append to std::cout only from rank 0.
Time Seconds(double value)
Construct a Time in the indicated unit.
Definition: nstime.h:1338
Common methods for MPI examples.
clientApps
Definition: first.py:58
stack
Definition: first.py:37
Every class exported by the ns3 library is enclosed in the ns3 namespace.
Callback< R, Args... > MakeCallback(R(T::*memPtr)(Args...), OBJ objPtr)
Build Callbacks for class method members which take varying numbers of arguments and potentially retu...
Definition: callback.h:691
@ LOG_LEVEL_INFO
LOG_INFO and above.
Definition: log.h:107
uint32_t GetSize(Ptr< const Packet > packet, const WifiMacHeader *hdr, bool isAmpdu)
Return the total size of the packet after WifiMacHeader and FCS trailer have been added.
Definition: wifi-utils.cc:132
void LogComponentEnable(const char *name, enum LogLevel level)
Enable the logging output associated with that log component.
Definition: log.cc:358
cmd
Definition: second.py:33
#define list
bool verbose
void ReportRank(int color, MPI_Comm splitComm)
Report my rank, in both MPI_COMM_WORLD and the split communicator.
const int NS_COLOR
Tag for whether this rank should go into a new communicator ns-3 ranks will have color == 1.
const int NOT_NS_COLOR
Tag for whether this rank should go into a new communicator ns-3 ranks will have color == 1.
bool tracing
Flag to enable/disable generation of tracing files.
Ptr< PacketSink > sink
Pointer to the packet sink application.
Definition: wifi-tcp.cc:55