fqcodel-l4s-example.cc

Go to the documentation of this file.

/*
 * Copyright (c) 2020 NITK Surathkal
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation;
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 * Authors: Bhaskar Kataria <bhaskar.k7920@gmail.com>
 *          Tom Henderson <tomhend@u.washington.edu>
 *          Mohit P. Tahiliani <tahiliani@nitk.edu.in>
 *          Vivek Jain <jain.vivek.anand@gmail.com>
 *          Ankit Deepak <adadeepak8@gmail.com>
 * This script is written using Tom Henderson's L4S evaluation available at
 * https://gitlab.com/tomhend/modules/l4s-evaluation
 */
 
// The 9 configurations below test BIC and DCTCP under various scenarios.
// Scenarios are numbered 1-9.  By default, scenario number 0 (i.e., no
// scenario) is configured, which means that the user is free to set
// any of the parameters freely; if scenarios 1 through 9 are selected,
// the scenario parameters are fixed.
//
// The configuration of the scenarios starts from basic TCP BIC without ECN
// with the base RTT of 80ms and then in the next scenario, ECN is enabled,
// and gradually, the complexity of the scenario increases and the last scenario
// consists of 2 flows one with BIC and other with DCTCP and finally tests
// The performance of the L4S mode of FqCoDel queue disc.
 
#include "ns3/applications-module.h"
#include "ns3/core-module.h"
#include "ns3/flow-monitor-helper.h"
#include "ns3/internet-apps-module.h"
#include "ns3/internet-module.h"
#include "ns3/network-module.h"
#include "ns3/point-to-point-module.h"
#include "ns3/traffic-control-module.h"
 
using namespace ns3;
 
NS_LOG_COMPONENT_DEFINE("FqCoDelL4SExample");
 
uint32_t checkTimes;
double avgQueueDiscSize;
 
uint32_t g_n0BytesReceived = 0;
uint32_t g_n1BytesReceived = 0;
uint32_t g_marksObserved = 0;
uint32_t g_dropsObserved = 0;
 
void
TraceN0Cwnd(std::ofstream* ofStream, uint32_t oldCwnd, uint32_t newCwnd)
{
    // TCP segment size is configured below to be 1448 bytes
    // so that we can report cwnd in units of segments
    *ofStream << Simulator::Now().GetSeconds() << " " << static_cast<double>(newCwnd) / 1448
              << std::endl;
}
 
void
TraceN1Cwnd(std::ofstream* ofStream, uint32_t oldCwnd, uint32_t newCwnd)
{
    // TCP segment size is configured below to be 1448 bytes
    // so that we can report cwnd in units of segments
    *ofStream << Simulator::Now().GetSeconds() << " " << static_cast<double>(newCwnd) / 1448
              << std::endl;
}
 
void
TraceN0Rtt(std::ofstream* ofStream, Time oldRtt, Time newRtt)
{
    *ofStream << Simulator::Now().GetSeconds() << " " << newRtt.GetSeconds() * 1000 << std::endl;
}
 
void
TraceN1Rtt(std::ofstream* ofStream, Time oldRtt, Time newRtt)
{
    *ofStream << Simulator::Now().GetSeconds() << " " << newRtt.GetSeconds() * 1000 << std::endl;
}
 
void
TracePingRtt(std::ofstream* ofStream, Time rtt)
{
    *ofStream << Simulator::Now().GetSeconds() << " " << rtt.GetSeconds() * 1000 << std::endl;
}
 
void
TraceN0Rx(Ptr<const Packet> packet, const Address& address)
{
    g_n0BytesReceived += packet->GetSize();
}
 
void
TraceN1Rx(Ptr<const Packet> packet, const Address& address)
{
    g_n1BytesReceived += packet->GetSize();
}
 
void
TraceDrop(std::ofstream* ofStream, Ptr<const QueueDiscItem> item)
{
    *ofStream << Simulator::Now().GetSeconds() << " " << std::hex << item->Hash() << std::endl;
    g_dropsObserved++;
}
 
void
TraceMark(std::ofstream* ofStream, Ptr<const QueueDiscItem> item, const char* reason)
{
    *ofStream << Simulator::Now().GetSeconds() << " " << std::hex << item->Hash() << std::endl;
    g_marksObserved++;
}
 
void
TraceQueueLength(std::ofstream* ofStream, DataRate linkRate, uint32_t oldVal, uint32_t newVal)
{
    // output in units of ms
    *ofStream << Simulator::Now().GetSeconds() << " " << std::fixed
              << static_cast<double>(newVal * 8) / (linkRate.GetBitRate() / 1000) << std::endl;
}
 
void
TraceDropsFrequency(std::ofstream* ofStream, Time dropsSamplingInterval)
{
    *ofStream << Simulator::Now().GetSeconds() << " " << g_dropsObserved << std::endl;
    g_dropsObserved = 0;
    Simulator::Schedule(dropsSamplingInterval,
                        &TraceDropsFrequency,
                        ofStream,
                        dropsSamplingInterval);
}
 
void
TraceMarksFrequency(std::ofstream* ofStream, Time marksSamplingInterval)
{
    *ofStream << Simulator::Now().GetSeconds() << " " << g_marksObserved << std::endl;
    g_marksObserved = 0;
    Simulator::Schedule(marksSamplingInterval,
                        &TraceMarksFrequency,
                        ofStream,
                        marksSamplingInterval);
}
 
void
TraceN0Throughput(std::ofstream* ofStream, Time throughputInterval)
{
    *ofStream << Simulator::Now().GetSeconds() << " "
              << g_n0BytesReceived * 8 / throughputInterval.GetSeconds() / 1e6 << std::endl;
    g_n0BytesReceived = 0;
    Simulator::Schedule(throughputInterval, &TraceN0Throughput, ofStream, throughputInterval);
}
 
void
TraceN1Throughput(std::ofstream* ofStream, Time throughputInterval)
{
    *ofStream << Simulator::Now().GetSeconds() << " "
              << g_n1BytesReceived * 8 / throughputInterval.GetSeconds() / 1e6 << std::endl;
    g_n1BytesReceived = 0;
    Simulator::Schedule(throughputInterval, &TraceN1Throughput, ofStream, throughputInterval);
}
 
void
ScheduleN0TcpCwndTraceConnection(std::ofstream* ofStream)
{
    Config::ConnectWithoutContext("/NodeList/1/$ns3::TcpL4Protocol/SocketList/0/CongestionWindow",
                                  MakeBoundCallback(&TraceN0Cwnd, ofStream));
}
 
void
ScheduleN0TcpRttTraceConnection(std::ofstream* ofStream)
{
    Config::ConnectWithoutContext("/NodeList/1/$ns3::TcpL4Protocol/SocketList/0/RTT",
                                  MakeBoundCallback(&TraceN0Rtt, ofStream));
}
 
void
ScheduleN0PacketSinkConnection()
{
    Config::ConnectWithoutContext("/NodeList/6/ApplicationList/*/$ns3::PacketSink/Rx",
                                  MakeCallback(&TraceN0Rx));
}
 
void
ScheduleN1TcpCwndTraceConnection(std::ofstream* ofStream)
{
    Config::ConnectWithoutContext("/NodeList/2/$ns3::TcpL4Protocol/SocketList/0/CongestionWindow",
                                  MakeBoundCallback(&TraceN1Cwnd, ofStream));
}
 
void
ScheduleN1TcpRttTraceConnection(std::ofstream* ofStream)
{
    Config::ConnectWithoutContext("/NodeList/2/$ns3::TcpL4Protocol/SocketList/0/RTT",
                                  MakeBoundCallback(&TraceN1Rtt, ofStream));
}
 
void
ScheduleN1PacketSinkConnection()
{
    Config::ConnectWithoutContext("/NodeList/7/ApplicationList/*/$ns3::PacketSink/Rx",
                                  MakeCallback(&TraceN1Rx));
}
 
static void
PacketDequeue(std::ofstream* n0OfStream, std::ofstream* n1OfStream, Ptr<const QueueDiscItem> item)
{
    Ptr<Packet> p = item->GetPacket();
    Ptr<const Ipv4QueueDiscItem> iqdi =
        Ptr<const Ipv4QueueDiscItem>(dynamic_cast<const Ipv4QueueDiscItem*>(PeekPointer(item)));
    Ipv4Address address = iqdi->GetHeader().GetDestination();
    Time qDelay = Simulator::Now() - item->GetTimeStamp();
    if (address == "192.168.2.2")
    {
        *n0OfStream << Simulator::Now().GetSeconds() << " " << qDelay.GetMicroSeconds() / 1000.0
                    << std::endl;
    }
    else if (address == "192.168.3.2")
    {
        *n1OfStream << Simulator::Now().GetSeconds() << " " << qDelay.GetMicroSeconds() / 1000.0
                    << std::endl;
    }
}
 
int
main(int argc, char* argv[])
{
    // variables not configured at command line               //
    Time stopTime = Seconds(70);
    Time baseRtt = MilliSeconds(80);
    uint32_t pingSize = 100; // bytes
    Time pingInterval = MilliSeconds(100);
    Time marksSamplingInterval = MilliSeconds(100);
    Time throughputSamplingInterval = MilliSeconds(200);
    DataRate bottleneckRate("100Mbps");
 
    std::string dir = "results/FqCoDel-L4S/";
    std::string dirToSave = "mkdir -p " + dir;
    if (system(dirToSave.c_str()) == -1)
    {
        exit(1);
    }
 
    std::string pingTraceFile = dir + "ping.dat";
    std::string n0TcpRttTraceFile = dir + "n0-tcp-rtt.dat";
    std::string n0TcpCwndTraceFile = dir + "n0-tcp-cwnd.dat";
    std::string n0TcpThroughputTraceFile = dir + "n0-tcp-throughput.dat";
    std::string n1TcpRttTraceFile = dir + "n1-tcp-rtt.dat";
    std::string n1TcpCwndTraceFile = dir + "n1-tcp-cwnd.dat";
    std::string n1TcpThroughputTraceFile = dir + "n1-tcp-throughput.dat";
    std::string dropTraceFile = dir + "drops.dat";
    std::string dropsFrequencyTraceFile = dir + "drops-frequency.dat";
    std::string lengthTraceFile = dir + "length.dat";
    std::string markTraceFile = dir + "mark.dat";
    std::string marksFrequencyTraceFile = dir + "marks-frequency.dat";
    std::string queueDelayN0TraceFile = dir + "queue-delay-n0.dat";
    std::string queueDelayN1TraceFile = dir + "queue-delay-n1.dat";
 
    // variables configured at command line                   //
    bool enablePcap = false;
    bool useCeThreshold = false;
    Time ceThreshold = MilliSeconds(1);
    std::string n0TcpType = "bic";
    std::string n1TcpType = "";
    bool enableN1Tcp = false;
    bool useEcn = true;
    std::string queueType = "fq";
    std::string linkDataRate = "1Gbps";
    uint32_t scenarioNum = 0;
 
    // Override ns-3 defaults                                 //
    Config::SetDefault("ns3::TcpSocket::SegmentSize", UintegerValue(1448));
    // Increase default buffer sizes to improve throughput over long delay paths
    Config::SetDefault("ns3::TcpSocket::SndBufSize", UintegerValue(8192000));
    Config::SetDefault("ns3::TcpSocket::RcvBufSize", UintegerValue(8192000));
    Config::SetDefault("ns3::TcpSocket::InitialCwnd", UintegerValue(10));
    Config::SetDefault("ns3::TcpL4Protocol::RecoveryType",
                       TypeIdValue(TcpPrrRecovery::GetTypeId()));
 
    // command-line argument parsing                          //
    CommandLine cmd;
    cmd.AddValue("n0TcpType", "n0 TCP type (bic, dctcp, or reno)", n0TcpType);
    cmd.AddValue("n1TcpType", "n1 TCP type (bic, dctcp, or reno)", n1TcpType);
    cmd.AddValue("scenarioNum",
                 "Scenario number from the scenarios avalaible in the file (1-9)",
                 scenarioNum);
    cmd.AddValue("bottleneckQueueType", "n2 queue type (fq or codel)", queueType);
    cmd.AddValue("baseRtt", "base RTT", baseRtt);
    cmd.AddValue("useCeThreshold", "use CE Threshold", useCeThreshold);
    cmd.AddValue("useEcn", "use ECN", useEcn);
    cmd.AddValue("ceThreshold", "CoDel CE threshold", ceThreshold);
    cmd.AddValue("bottleneckRate", "data rate of bottleneck", bottleneckRate);
    cmd.AddValue("linkRate", "data rate of edge link", linkDataRate);
    cmd.AddValue("stopTime", "simulation stop time", stopTime);
    cmd.AddValue("enablePcap", "enable Pcap", enablePcap);
    cmd.AddValue("pingTraceFile", "filename for ping tracing", pingTraceFile);
    cmd.AddValue("n0TcpRttTraceFile", "filename for n0 rtt tracing", n0TcpRttTraceFile);
    cmd.AddValue("n0TcpCwndTraceFile", "filename for n0 cwnd tracing", n0TcpCwndTraceFile);
    cmd.AddValue("n0TcpThroughputTraceFile",
                 "filename for n0 throughput tracing",
                 n0TcpThroughputTraceFile);
    cmd.AddValue("n1TcpRttTraceFile", "filename for n1 rtt tracing", n1TcpRttTraceFile);
    cmd.AddValue("n1TcpCwndTraceFile", "filename for n1 cwnd tracing", n1TcpCwndTraceFile);
    cmd.AddValue("n1TcpThroughputTraceFile",
                 "filename for n1 throughput tracing",
                 n1TcpThroughputTraceFile);
    cmd.AddValue("dropTraceFile", "filename for n2 drops tracing", dropTraceFile);
    cmd.AddValue("dropsFrequencyTraceFile",
                 "filename for n2 drop frequency tracing",
                 dropsFrequencyTraceFile);
    cmd.AddValue("lengthTraceFile", "filename for n2 queue length tracing", lengthTraceFile);
    cmd.AddValue("markTraceFile", "filename for n2 mark tracing", markTraceFile);
    cmd.AddValue("marksFrequencyTraceFile",
                 "filename for n2 mark frequency tracing",
                 marksFrequencyTraceFile);
    cmd.AddValue("queueDelayN0TraceFile",
                 "filename for n0 queue delay tracing",
                 queueDelayN0TraceFile);
    cmd.AddValue("queueDelayN1TraceFile",
                 "filename for n1 queue delay tracing",
                 queueDelayN1TraceFile);
    cmd.Parse(argc, argv);
    Time oneWayDelay = baseRtt / 2;
    TypeId n0TcpTypeId;
    TypeId n1TcpTypeId;
    TypeId queueTypeId;
    if (!scenarioNum)
    {
        if (useEcn)
        {
            Config::SetDefault("ns3::TcpSocketBase::UseEcn", StringValue("On"));
        }
 
        if (n0TcpType == "reno")
        {
            n0TcpTypeId = TcpNewReno::GetTypeId();
        }
        else if (n0TcpType == "bic")
        {
            n0TcpTypeId = TcpBic::GetTypeId();
        }
        else if (n0TcpType == "dctcp")
        {
            n0TcpTypeId = TcpDctcp::GetTypeId();
        }
        else
        {
            NS_FATAL_ERROR("Fatal error:  tcp unsupported");
        }
 
        if (n1TcpType == "reno")
        {
            enableN1Tcp = true;
            n1TcpTypeId = TcpNewReno::GetTypeId();
        }
        else if (n1TcpType == "bic")
        {
            enableN1Tcp = true;
            n1TcpTypeId = TcpBic::GetTypeId();
        }
        else if (n1TcpType == "dctcp")
        {
            enableN1Tcp = true;
            n1TcpTypeId = TypeId::LookupByName("ns3::TcpDctcp");
        }
        else if (n1TcpType == "")
        {
            NS_LOG_DEBUG("No N1 TCP selected");
        }
        else
        {
            NS_FATAL_ERROR("Fatal error:  tcp unsupported");
        }
 
        if (queueType == "fq")
        {
            queueTypeId = FqCoDelQueueDisc::GetTypeId();
        }
        else if (queueType == "codel")
        {
            queueTypeId = CoDelQueueDisc::GetTypeId();
        }
        else
        {
            NS_FATAL_ERROR("Fatal error:  queueType unsupported");
        }
        if (useCeThreshold)
        {
            Config::SetDefault("ns3::FqCoDelQueueDisc::CeThreshold", TimeValue(ceThreshold));
        }
    }
    else if (scenarioNum == 1 || scenarioNum == 2 || scenarioNum == 5 || scenarioNum == 6)
    {
        if (scenarioNum == 2 || scenarioNum == 6)
        {
            Config::SetDefault("ns3::TcpSocketBase::UseEcn", StringValue("On"));
        }
        n0TcpTypeId = TcpBic::GetTypeId();
        if (scenarioNum == 5 || scenarioNum == 6)
        {
            enableN1Tcp = true;
            n1TcpTypeId = TcpBic::GetTypeId();
        }
        queueTypeId = FqCoDelQueueDisc::GetTypeId();
    }
    else if (scenarioNum == 3 || scenarioNum == 4 || scenarioNum == 7 || scenarioNum == 8 ||
             scenarioNum == 9)
    {
        Config::SetDefault("ns3::TcpSocketBase::UseEcn", StringValue("On"));
        n0TcpTypeId = TcpDctcp::GetTypeId();
        queueTypeId = FqCoDelQueueDisc::GetTypeId();
        oneWayDelay = MicroSeconds(500);
        Config::SetDefault("ns3::FqCoDelQueueDisc::CeThreshold", TimeValue(MilliSeconds(1)));
        if (scenarioNum == 9)
        {
            n0TcpTypeId = TcpBic::GetTypeId();
            // For TCP Bic base RTT is 80 and base RTT for dctcp is set to 1 while setting delay for
            // p2p devices
            oneWayDelay = MilliSeconds(40);
        }
        if (scenarioNum == 4 || scenarioNum == 8 || scenarioNum == 9)
        {
            Config::SetDefault("ns3::FqCoDelQueueDisc::UseL4s", BooleanValue(true));
            Config::SetDefault("ns3::TcpDctcp::UseEct0", BooleanValue(false));
        }
        if (scenarioNum == 7 || scenarioNum == 8 || scenarioNum == 9)
        {
            enableN1Tcp = true;
            n1TcpTypeId = TcpDctcp::GetTypeId();
        }
    }
    else
    {
        NS_FATAL_ERROR("Fatal error:  scenario unavailble");
    }
 
    std::ofstream pingOfStream;
    pingOfStream.open(pingTraceFile, std::ofstream::out);
    std::ofstream n0TcpRttOfStream;
    n0TcpRttOfStream.open(n0TcpRttTraceFile, std::ofstream::out);
    std::ofstream n0TcpCwndOfStream;
    n0TcpCwndOfStream.open(n0TcpCwndTraceFile, std::ofstream::out);
    std::ofstream n0TcpThroughputOfStream;
    n0TcpThroughputOfStream.open(n0TcpThroughputTraceFile, std::ofstream::out);
    std::ofstream n1TcpRttOfStream;
    n1TcpRttOfStream.open(n1TcpRttTraceFile, std::ofstream::out);
    std::ofstream n1TcpCwndOfStream;
    n1TcpCwndOfStream.open(n1TcpCwndTraceFile, std::ofstream::out);
    std::ofstream n1TcpThroughputOfStream;
    n1TcpThroughputOfStream.open(n1TcpThroughputTraceFile, std::ofstream::out);
 
    // Queue disc files
    std::ofstream dropOfStream;
    dropOfStream.open(dropTraceFile, std::ofstream::out);
    std::ofstream markOfStream;
    markOfStream.open(markTraceFile, std::ofstream::out);
    std::ofstream dropsFrequencyOfStream;
    dropsFrequencyOfStream.open(dropsFrequencyTraceFile, std::ofstream::out);
    std::ofstream marksFrequencyOfStream;
    marksFrequencyOfStream.open(marksFrequencyTraceFile, std::ofstream::out);
    std::ofstream lengthOfStream;
    lengthOfStream.open(lengthTraceFile, std::ofstream::out);
    std::ofstream queueDelayN0OfStream;
    queueDelayN0OfStream.open(queueDelayN0TraceFile, std::ofstream::out);
    std::ofstream queueDelayN1OfStream;
    queueDelayN1OfStream.open(queueDelayN1TraceFile, std::ofstream::out);
 
    // scenario setup                                         //
    Ptr<Node> pingServer = CreateObject<Node>();
    Ptr<Node> n0Server = CreateObject<Node>();
    Ptr<Node> n1Server = CreateObject<Node>();
    Ptr<Node> n2 = CreateObject<Node>();
    Ptr<Node> n3 = CreateObject<Node>();
    Ptr<Node> pingClient = CreateObject<Node>();
    Ptr<Node> n4Client = CreateObject<Node>();
    Ptr<Node> n5Client = CreateObject<Node>();
 
    // Device containers
    NetDeviceContainer pingServerDevices;
    NetDeviceContainer n0ServerDevices;
    NetDeviceContainer n1ServerDevices;
    NetDeviceContainer n2n3Devices;
    NetDeviceContainer pingClientDevices;
    NetDeviceContainer n4ClientDevices;
    NetDeviceContainer n5ClientDevices;
 
    PointToPointHelper p2p;
    p2p.SetQueue("ns3::DropTailQueue", "MaxSize", QueueSizeValue(QueueSize("3p")));
    p2p.SetDeviceAttribute("DataRate", DataRateValue(DataRate(linkDataRate)));
    // Add delay only on the server links
    p2p.SetChannelAttribute("Delay", TimeValue(oneWayDelay));
    pingServerDevices = p2p.Install(n2, pingServer);
    n0ServerDevices = p2p.Install(n2, n0Server);
 
    // In scenario 9, base RTT of n1server (dctcp) is 1ms
    if (scenarioNum == 9)
    {
        p2p.SetChannelAttribute("Delay", TimeValue(MicroSeconds(500)));
    }
    n1ServerDevices = p2p.Install(n2, n1Server);
    p2p.SetChannelAttribute("Delay", TimeValue(MicroSeconds(1)));
    n2n3Devices = p2p.Install(n2, n3);
    pingClientDevices = p2p.Install(n3, pingClient);
    n4ClientDevices = p2p.Install(n3, n4Client);
    n5ClientDevices = p2p.Install(n3, n5Client);
    Ptr<PointToPointNetDevice> p = n2n3Devices.Get(0)->GetObject<PointToPointNetDevice>();
    p->SetAttribute("DataRate", DataRateValue(bottleneckRate));
 
    InternetStackHelper stackHelper;
    stackHelper.InstallAll();
 
    // Set the per-node TCP type here
    Ptr<TcpL4Protocol> proto;
    proto = n4Client->GetObject<TcpL4Protocol>();
    proto->SetAttribute("SocketType", TypeIdValue(n0TcpTypeId));
    proto = n0Server->GetObject<TcpL4Protocol>();
    proto->SetAttribute("SocketType", TypeIdValue(n0TcpTypeId));
    if (enableN1Tcp)
    {
        proto = n5Client->GetObject<TcpL4Protocol>();
        proto->SetAttribute("SocketType", TypeIdValue(n1TcpTypeId));
        proto = n1Server->GetObject<TcpL4Protocol>();
        proto->SetAttribute("SocketType", TypeIdValue(n1TcpTypeId));
    }
 
    // InternetStackHelper will install a base TrafficControLayer on the node,
    // but the Ipv4AddressHelper below will install the default FqCoDelQueueDisc
    // on all single device nodes.  The below code overrides the configuration
    // that is normally done by the Ipv4AddressHelper::Install() method by
    // instead explicitly configuring the queue discs we want on each device.
    TrafficControlHelper tchFq;
    tchFq.SetRootQueueDisc("ns3::FqCoDelQueueDisc");
    tchFq.SetQueueLimits("ns3::DynamicQueueLimits", "HoldTime", StringValue("1ms"));
    tchFq.Install(pingServerDevices);
    tchFq.Install(n0ServerDevices);
    tchFq.Install(n1ServerDevices);
    tchFq.Install(n2n3Devices.Get(1)); // n2 queue for bottleneck link
    tchFq.Install(pingClientDevices);
    tchFq.Install(n4ClientDevices);
    tchFq.Install(n5ClientDevices);
    TrafficControlHelper tchN2;
    tchN2.SetRootQueueDisc(queueTypeId.GetName());
    tchN2.SetQueueLimits("ns3::DynamicQueueLimits", "HoldTime", StringValue("1000ms"));
    tchN2.Install(n2n3Devices.Get(0));
 
    Ipv4AddressHelper ipv4;
    ipv4.SetBase("10.1.1.0", "255.255.255.0");
    Ipv4InterfaceContainer pingServerIfaces = ipv4.Assign(pingServerDevices);
    ipv4.SetBase("10.1.2.0", "255.255.255.0");
    Ipv4InterfaceContainer n0ServerIfaces = ipv4.Assign(n0ServerDevices);
    ipv4.SetBase("10.1.3.0", "255.255.255.0");
    Ipv4InterfaceContainer secondServerIfaces = ipv4.Assign(n1ServerDevices);
    ipv4.SetBase("172.16.1.0", "255.255.255.0");
    Ipv4InterfaceContainer n2n3Ifaces = ipv4.Assign(n2n3Devices);
    ipv4.SetBase("192.168.1.0", "255.255.255.0");
    Ipv4InterfaceContainer pingClientIfaces = ipv4.Assign(pingClientDevices);
    ipv4.SetBase("192.168.2.0", "255.255.255.0");
    Ipv4InterfaceContainer n4ClientIfaces = ipv4.Assign(n4ClientDevices);
    ipv4.SetBase("192.168.3.0", "255.255.255.0");
    Ipv4InterfaceContainer n5ClientIfaces = ipv4.Assign(n5ClientDevices);
 
    Ipv4GlobalRoutingHelper::PopulateRoutingTables();
 
    // application setup                                      //
 
    V4PingHelper pingHelper("192.168.1.2");
    pingHelper.SetAttribute("Interval", TimeValue(pingInterval));
    pingHelper.SetAttribute("Size", UintegerValue(pingSize));
    ApplicationContainer pingContainer = pingHelper.Install(pingServer);
    Ptr<V4Ping> v4Ping = pingContainer.Get(0)->GetObject<V4Ping>();
    v4Ping->TraceConnectWithoutContext("Rtt", MakeBoundCallback(&TracePingRtt, &pingOfStream));
    pingContainer.Start(Seconds(1));
    pingContainer.Stop(stopTime - Seconds(1));
 
    BulkSendHelper tcp("ns3::TcpSocketFactory", Address());
    // set to large value:  e.g. 1000 Mb/s for 60 seconds = 7500000000 bytes
    tcp.SetAttribute("MaxBytes", UintegerValue(7500000000));
    // Configure n4/n0 TCP client/server pair
    uint16_t n4Port = 5000;
    ApplicationContainer n0App;
    InetSocketAddress n0DestAddress(n4ClientIfaces.GetAddress(1), n4Port);
    tcp.SetAttribute("Remote", AddressValue(n0DestAddress));
    n0App = tcp.Install(n0Server);
    n0App.Start(Seconds(5));
    n0App.Stop(stopTime - Seconds(1));
 
    Address n4SinkAddress(InetSocketAddress(Ipv4Address::GetAny(), n4Port));
    PacketSinkHelper n4SinkHelper("ns3::TcpSocketFactory", n4SinkAddress);
    ApplicationContainer n4SinkApp;
    n4SinkApp = n4SinkHelper.Install(n4Client);
    n4SinkApp.Start(Seconds(5));
    n4SinkApp.Stop(stopTime - MilliSeconds(500));
 
    // Configure second TCP client/server pair
    if (enableN1Tcp)
    {
        uint16_t n5Port = 5000;
        ApplicationContainer secondApp;
        InetSocketAddress n1DestAddress(n5ClientIfaces.GetAddress(1), n5Port);
        tcp.SetAttribute("Remote", AddressValue(n1DestAddress));
        secondApp = tcp.Install(n1Server);
        secondApp.Start(Seconds(15));
        secondApp.Stop(stopTime - Seconds(1));
 
        Address n5SinkAddress(InetSocketAddress(Ipv4Address::GetAny(), n5Port));
        PacketSinkHelper n5SinkHelper("ns3::TcpSocketFactory", n5SinkAddress);
        ApplicationContainer n5SinkApp;
        n5SinkApp = n5SinkHelper.Install(n5Client);
        n5SinkApp.Start(Seconds(15));
        n5SinkApp.Stop(stopTime - MilliSeconds(500));
    }
 
    // Setup traces that can be hooked now
    Ptr<TrafficControlLayer> tc;
    Ptr<QueueDisc> qd;
    tc = n2n3Devices.Get(0)->GetNode()->GetObject<TrafficControlLayer>();
    qd = tc->GetRootQueueDiscOnDevice(n2n3Devices.Get(0));
    qd->TraceConnectWithoutContext("Drop", MakeBoundCallback(&TraceDrop, &dropOfStream));
    qd->TraceConnectWithoutContext("Mark", MakeBoundCallback(&TraceMark, &markOfStream));
    qd->TraceConnectWithoutContext(
        "BytesInQueue",
        MakeBoundCallback(&TraceQueueLength, &lengthOfStream, bottleneckRate));
    qd->TraceConnectWithoutContext(
        "Dequeue",
        MakeBoundCallback(&PacketDequeue, &queueDelayN0OfStream, &queueDelayN1OfStream));
 
    // Setup scheduled traces; TCP traces must be hooked after socket creation
    Simulator::Schedule(Seconds(5) + MilliSeconds(100),
                        &ScheduleN0TcpRttTraceConnection,
                        &n0TcpRttOfStream);
    Simulator::Schedule(Seconds(5) + MilliSeconds(100),
                        &ScheduleN0TcpCwndTraceConnection,
                        &n0TcpCwndOfStream);
    Simulator::Schedule(Seconds(5) + MilliSeconds(100), &ScheduleN0PacketSinkConnection);
    Simulator::Schedule(throughputSamplingInterval,
                        &TraceN0Throughput,
                        &n0TcpThroughputOfStream,
                        throughputSamplingInterval);
    // Setup scheduled traces; TCP traces must be hooked after socket creation
    if (enableN1Tcp)
    {
        Simulator::Schedule(Seconds(15) + MilliSeconds(100),
                            &ScheduleN1TcpRttTraceConnection,
                            &n1TcpRttOfStream);
        Simulator::Schedule(Seconds(15) + MilliSeconds(100),
                            &ScheduleN1TcpCwndTraceConnection,
                            &n1TcpCwndOfStream);
        Simulator::Schedule(Seconds(15) + MilliSeconds(100), &ScheduleN1PacketSinkConnection);
    }
    Simulator::Schedule(throughputSamplingInterval,
                        &TraceN1Throughput,
                        &n1TcpThroughputOfStream,
                        throughputSamplingInterval);
    Simulator::Schedule(marksSamplingInterval,
                        &TraceMarksFrequency,
                        &marksFrequencyOfStream,
                        marksSamplingInterval);
    Simulator::Schedule(marksSamplingInterval,
                        &TraceDropsFrequency,
                        &dropsFrequencyOfStream,
                        marksSamplingInterval);
 
    if (enablePcap)
    {
        p2p.EnablePcapAll("FqCoDel-L4S-example", false);
    }
 
    Simulator::Stop(stopTime);
    Simulator::Run();
 
    pingOfStream.close();
    n0TcpCwndOfStream.close();
    n0TcpRttOfStream.close();
    n0TcpThroughputOfStream.close();
    n1TcpCwndOfStream.close();
    n1TcpRttOfStream.close();
    n1TcpThroughputOfStream.close();
    dropOfStream.close();
    markOfStream.close();
    dropsFrequencyOfStream.close();
    marksFrequencyOfStream.close();
    lengthOfStream.close();
    queueDelayN0OfStream.close();
    queueDelayN1OfStream.close();
}