src/dev/net/dist_iface.hh - amd/gem5 - Git at Google

 /*
  * Copyright (c) 2015-2016 ARM Limited
  * All rights reserved
  *
  * The license below extends only to copyright in the software and shall
  * not be construed as granting a license to any other intellectual
  * property including but not limited to intellectual property relating
  * to a hardware implementation of the functionality of the software
  * licensed hereunder.  You may use the software subject to the license
  * terms below provided that you ensure that this notice is replicated
  * unmodified and in its entirety in all distributions of the software,
  * modified or unmodified, in source code or in binary form.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met: redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer;
  * redistributions in binary form must reproduce the above copyright
  * notice, this list of conditions and the following disclaimer in the
  * documentation and/or other materials provided with the distribution;
  * neither the name of the copyright holders nor the names of its
  * contributors may be used to endorse or promote products derived from
  * this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * Authors: Gabor Dozsa
  */

 /* @file
  * The interface class for dist gem5 simulations.
  *
  * dist-gem5 is an extension to gem5 to enable parallel simulation of a
  * distributed system (e.g. simulation of a pool of machines
  * connected by Ethernet links). A dist gem5 run consists of seperate gem5
  * processes running in parallel. Each gem5 process executes
  * the simulation of a component of the simulated distributed system.
  * (An example component can be a dist-core board with an Ethernet NIC.)
  * The DistIface class below provides services to transfer data and
  * control messages among the gem5 processes. The main such services are
  * as follows.
  *
  * 1. Send a data packet coming from a simulated Ethernet link. The packet
  * will be transferred to (all) the target(s) gem5 processes. The send
  * operation is always performed by the simulation thread, i.e. the gem5
  * thread that is processing the event queue associated with the simulated
  * Ethernet link.
  *
  * 2. Spawn a receiver thread to process messages coming in from the
  * from other gem5 processes. Each simulated Ethernet link has its own
  * associated receiver thread. The receiver thread saves the incoming packet
  * and schedule an appropriate receive event in the event queue.
  *
  * 3. Schedule a global barrier event periodically to keep the gem5
  * processes in sync.
  * Periodic barrier event to keep peer gem5 processes in sync. The basic idea
  * is that no gem5 process can go ahead further than the simulated link
  * transmission delay to ensure that a corresponding receive event can always
  * be scheduled for any message coming in from a peer gem5 process.
  *
  *
  *
  * This interface is an abstract class. It can work with various low level
  * send/receive service implementations (e.g. TCP/IP, MPI,...). A TCP
  * stream socket version is implemented in src/dev/net/tcp_iface.[hh,cc].
  */
 #ifndef __DEV_DIST_IFACE_HH__
 #define __DEV_DIST_IFACE_HH__

 #include <array>
 #include <mutex>
 #include <queue>
 #include <thread>
 #include <utility>

 #include "base/logging.hh"
 #include "dev/net/dist_packet.hh"
 #include "dev/net/etherpkt.hh"
 #include "sim/core.hh"
 #include "sim/drain.hh"
 #include "sim/global_event.hh"
 #include "sim/serialize.hh"

 class EventManager;
 class System;
 class ThreadContext;

 /**
  * The interface class to talk to peer gem5 processes.
  */
 class DistIface : public Drainable, public Serializable
 {
   public:
     typedef DistHeaderPkt::Header Header;

   protected:
     typedef DistHeaderPkt::MsgType MsgType;
     typedef DistHeaderPkt::ReqType ReqType;

   private:
     class SyncEvent;
     /** @class Sync
      * This class implements global sync operations among gem5 peer processes.
      *
      * @note This class is used as a singleton object (shared by all DistIface
      * objects).
      */
     class Sync : public Serializable
     {
       protected:
         /**
          * The lock to protect access to the Sync object.
          */
         std::mutex lock;
         /**
          * Condition variable for the simulation thread to wait on
          * until all receiver threads completes the current global
          * synchronisation.
          */
         std::condition_variable cv;
         /**
          * Number of receiver threads that not yet completed the current global
          * synchronisation.
          */
         unsigned waitNum;
         /**
          * Flag is set if exit is permitted upon sync completion
          */
         bool doExit;
         /**
          * Flag is set if taking a ckpt is permitted upon sync completion
          */
         bool doCkpt;
         /**
          * Flag is set if sync is to stop upon sync completion
          */
         bool doStopSync;
         /**
          * The repeat value for the next periodic sync
          */
         Tick nextRepeat;
         /**
          * Tick for the next periodic sync (if the event is not scheduled yet)
          */
         Tick nextAt;
         /**
          *  Flag is set if the sync is aborted (e.g. due to connection lost)
          */
         bool isAbort;

         friend class SyncEvent;

       public:
         /**
          * Initialize periodic sync params.
          *
          * @param start Start tick for dist synchronisation
          * @param repeat Frequency of dist synchronisation
          *
          */
         void init(Tick start, Tick repeat);
         /**
          *  Core method to perform a full dist sync.
          *
          * @return true if the sync completes, false if it gets aborted
          */
         virtual bool run(bool same_tick) = 0;
         /**
          * Callback when the receiver thread gets a sync ack message.
          *
          * @return false if the receiver thread needs to stop (e.g.
          * simulation is to exit)
          */
         virtual bool progress(Tick send_tick,
                               Tick next_repeat,
                               ReqType do_ckpt,
                               ReqType do_exit,
                               ReqType do_stop_sync) = 0;
         /**
          * Abort processing an on-going sync event (in case of an error, e.g.
          * lost connection to a peer gem5)
          */
         void abort();

         virtual void requestCkpt(ReqType req) = 0;
         virtual void requestExit(ReqType req) = 0;
         virtual void requestStopSync(ReqType req) = 0;

         void drainComplete();

         virtual void serialize(CheckpointOut &cp) const override = 0;
         virtual void unserialize(CheckpointIn &cp) override = 0;
     };

     class SyncNode: public Sync
     {
       private:
         /**
          * Exit requested
          */
         ReqType needExit;
         /**
          * Ckpt requested
          */
         ReqType needCkpt;
         /**
          * Sync stop requested
          */
         ReqType needStopSync;

       public:

         SyncNode();
         ~SyncNode() {}
         bool run(bool same_tick) override;
         bool progress(Tick max_req_tick,
                       Tick next_repeat,
                       ReqType do_ckpt,
                       ReqType do_exit,
                       ReqType do_stop_sync) override;

         void requestCkpt(ReqType req) override;
         void requestExit(ReqType req) override;
         void requestStopSync(ReqType req) override;

         void serialize(CheckpointOut &cp) const override;
         void unserialize(CheckpointIn &cp) override;
     };

     class SyncSwitch: public Sync
     {
       private:
         /**
          * Counter for recording exit requests
          */
         unsigned numExitReq;
         /**
          * Counter for recording ckpt requests
          */
         unsigned numCkptReq;
         /**
          * Counter for recording stop sync requests
          */
         unsigned numStopSyncReq;
         /**
          *  Number of connected simulated nodes
          */
         unsigned numNodes;

       public:
         SyncSwitch(int num_nodes);
         ~SyncSwitch() {}

         bool run(bool same_tick) override;
         bool progress(Tick max_req_tick,
                       Tick next_repeat,
                       ReqType do_ckpt,
                       ReqType do_exit,
                       ReqType do_stop_sync) override;

         void requestCkpt(ReqType) override {
             panic("Switch requested checkpoint");
         }
         void requestExit(ReqType) override {
             panic("Switch requested exit");
         }
         void requestStopSync(ReqType) override {
             panic("Switch requested stop sync");
         }

         void serialize(CheckpointOut &cp) const override;
         void unserialize(CheckpointIn &cp) override;
     };

     /**
      * The global event to schedule periodic dist sync. It is used as a
      * singleton object.
      *
      * The periodic synchronisation works as follows.
      * 1. A SyncEvent is scheduled as a global event when startup() is
      * called.
      * 2. The process() method of the SyncEvent initiates a new barrier
      * for each simulated Ethernet link.
      * 3. Simulation thread(s) then waits until all receiver threads
      * complete the ongoing barrier. The global sync event is done.
      */
     class SyncEvent : public GlobalSyncEvent
     {
       private:
         /**
          * Flag to set when the system is draining
          */
         bool _draining;
       public:
         /**
          * Only the firstly instantiated DistIface object will
          * call this constructor.
          */
         SyncEvent() : GlobalSyncEvent(Sim_Exit_Pri, 0), _draining(false) {}

         ~SyncEvent() {}
         /**
          * Schedule the first periodic sync event.
          */
         void start();
         /**
          * This is a global event so process() will only be called by
          * exactly one simulation thread. (See further comments in the .cc
          * file.)
          */
         void process() override;

         bool draining() const { return _draining; }
         void draining(bool fl) { _draining = fl; }
     };
     /**
      * Class to encapsulate information about data packets received.

      * @note The main purpose of the class to take care of scheduling receive
      * done events for the simulated network link and store incoming packets
      * until they can be received by the simulated network link.
      */
     class RecvScheduler : public Serializable
     {
       private:
         /**
          * Received packet descriptor. This information is used by the receive
          * thread to schedule receive events and by the simulation thread to
          * process those events.
          */
         struct Desc : public Serializable
         {
             EthPacketPtr packet;
             Tick sendTick;
             Tick sendDelay;

             Desc() : sendTick(0), sendDelay(0) {}
             Desc(EthPacketPtr p, Tick s, Tick d) :
                 packet(p), sendTick(s), sendDelay(d) {}
             Desc(const Desc &d) :
                 packet(d.packet), sendTick(d.sendTick), sendDelay(d.sendDelay) {}

             void serialize(CheckpointOut &cp) const override;
             void unserialize(CheckpointIn &cp) override;
         };
         /**
          * The queue to store the receive descriptors.
          */
         std::queue<Desc> descQueue;
         /**
          * The tick when the most recent receive event was processed.
          *
          * @note This information is necessary to simulate possible receiver
          * link contention when calculating the receive tick for the next
          * incoming data packet (see the calcReceiveTick() method)
          */
         Tick prevRecvTick;
         /**
          * The receive done event for the simulated Ethernet link.
          *
          * @note This object is constructed by the simulated network link. We
          * schedule this object for each incoming data packet.
          */
         Event *recvDone;
         /**
          * The link delay in ticks for the simulated Ethernet link.
          *
          * @note This value is used for calculating the receive ticks for
          * incoming data packets.
          */
         Tick linkDelay;
         /**
          * The event manager associated with the simulated Ethernet link.
          *
          * @note It is used to access the event queue for scheduling receive
          * done events for the link.
          */
         EventManager *eventManager;
         /**
          * Calculate the tick to schedule the next receive done event.
          *
          * @param send_tick The tick the packet was sent.
          * @param send_delay The simulated delay at the sender side.
          * @param prev_recv_tick Tick when the last receive event was
          * processed.
          *
          * @note This method tries to take into account possible receiver link
          * contention and adjust receive tick for the incoming packets
          * accordingly.
          */
         Tick calcReceiveTick(Tick send_tick,
                              Tick send_delay,
                              Tick prev_recv_tick);

         /**
          * Flag to set if receive ticks for pending packets need to be
          * recalculated due to changed link latencies at a resume
          */
         bool ckptRestore;

       public:
         /**
          * Scheduler for the incoming data packets.
          *
          * @param em The event manager associated with the simulated Ethernet
          * link.
          */
         RecvScheduler(EventManager *em) :
             prevRecvTick(0), recvDone(nullptr), linkDelay(0),
             eventManager(em), ckptRestore(false) {}

         /**
          *  Initialize network link parameters.
          *
          * @note This method is called from the receiver thread (see
          * recvThreadFunc()).
          */
         void init(Event *recv_done, Tick link_delay);
         /**
          * Fetch the next packet that is to be received by the simulated network
          * link.
          *
          * @note This method is called from the process() method of the receive
          * done event associated with the network link.
          */
         EthPacketPtr popPacket();
         /**
          * Push a newly arrived packet into the desc queue.
          */
         void pushPacket(EthPacketPtr new_packet,
                         Tick send_tick,
                         Tick send_delay);

         void serialize(CheckpointOut &cp) const override;
         void unserialize(CheckpointIn &cp) override;
         /**
          * Adjust receive ticks for pending packets when restoring from a
          * checkpoint
          *
          * @note Link speed and delay parameters may change at resume.
          */
         void resumeRecvTicks();
     };
     /**
      * Tick to schedule the first dist sync event.
      * This is just as optimization : we do not need any dist sync
      * event until the simulated NIC is brought up by the OS.
      */
     Tick syncStart;
     /**
      * Frequency of dist sync events in ticks.
      */
     Tick syncRepeat;
     /**
      * Receiver thread pointer.
      * Each DistIface object must have exactly one receiver thread.
      */
     std::thread *recvThread;
     /**
      * Meta information about data packets received.
      */
     RecvScheduler recvScheduler;
     /**
      * Use pseudoOp to start synchronization.
      */
     bool syncStartOnPseudoOp;

   protected:
     /**
      * The rank of this process among the gem5 peers.
      */
     unsigned rank;
     /**
      * The number of gem5 processes comprising this dist simulation.
      */
     unsigned size;
     /**
      * Number of DistIface objects (i.e. dist links in this gem5 process)
      */
     static unsigned distIfaceNum;
     /**
      * Unique id for the dist link
      */
     unsigned distIfaceId;

     bool isMaster;

   private:
     /**
      * Number of receiver threads (in this gem5 process)
      */
     static unsigned recvThreadsNum;
     /**
      * The singleton Sync object to perform dist synchronisation.
      */
     static Sync *sync;
     /**
      * The singleton SyncEvent object to schedule periodic dist sync.
      */
     static SyncEvent *syncEvent;
     /**
      * The very first DistIface object created becomes the master. We need
      * a master to co-ordinate the global synchronisation.
      */
     static DistIface *master;
     /**
      * System pointer used to wakeup sleeping threads when stopping sync.
      */
     static System *sys;
     /**
      * Is this node a switch?
      */
      static bool isSwitch;

   private:
     /**
      * Send out a data packet to the remote end.
      * @param header Meta info about the packet (which needs to be transferred
      * to the destination alongside the packet).
      * @param packet Pointer to the packet to send.
      */
     virtual void sendPacket(const Header &header, const EthPacketPtr &packet) = 0;
     /**
      * Send out a control command to the remote end.
      * @param header Meta info describing the command (e.g. sync request)
      */
     virtual void sendCmd(const Header &header) = 0;
     /**
      * Receive a header (i.e. meta info describing a data packet or a control command)
      * from the remote end.
      * @param header The meta info structure to store the incoming header.
      */
     virtual bool recvHeader(Header &header) = 0;
     /**
      * Receive a packet from the remote end.
      * @param header Meta info about the incoming packet (obtanied by a previous
      * call to the recvHedaer() method).
      * @param Pointer to packet received.
      */
     virtual void recvPacket(const Header &header, EthPacketPtr &packet) = 0;
     /**
      * Init hook for the underlaying transport
      */
     virtual void initTransport() = 0;
     /**
      * spawn the receiver thread.
      * @param recv_done The receive done event associated with the simulated
      * Ethernet link.
      * @param link_delay The link delay for the simulated Ethernet link.
      */
     void spawnRecvThread(const Event *recv_done, Tick link_delay);
     /**
      * The function executed by a receiver thread.
      */
     void recvThreadFunc(Event *recv_done, Tick link_delay);

   public:

     /**
      * ctor
      * @param dist_rank Rank of this gem5 process within the dist run
      * @param sync_start Start tick for dist synchronisation
      * @param sync_repeat Frequency for dist synchronisation
      * @param em The event manager associated with the simulated Ethernet link
      */
     DistIface(unsigned dist_rank,
               unsigned dist_size,
               Tick sync_start,
               Tick sync_repeat,
               EventManager *em,
               bool use_pseudo_op,
               bool is_switch,
               int num_nodes);

     virtual ~DistIface();
     /**
      * Send out an Ethernet packet.
      * @param pkt The Ethernet packet to send.
      * @param send_delay The delay in ticks for the send completion event.
      */
     void packetOut(EthPacketPtr pkt, Tick send_delay);
     /**
      * Fetch the packet scheduled to be received next by the simulated
      * network link.
      *
      * @note This method is called within the process() method of the link
      * receive done event. It also schedules the next receive event if the
      * receive queue is not empty.
      */
     EthPacketPtr packetIn() { return recvScheduler.popPacket(); }

     DrainState drain() override;
     void drainResume() override;
     void init(const Event *e, Tick link_delay);
     void startup();

     void serialize(CheckpointOut &cp) const override;
     void unserialize(CheckpointIn &cp) override;
     /**
      * Initiate the exit from the simulation.
      * @param delay Delay param from the m5 exit command. If Delay is zero
      * then a collaborative exit is requested (i.e. all nodes have to call
      * this method before the distributed simulation can exit). If Delay is
      * not zero then exit is requested asap (and it will happen at the next
      * sync tick).
      * @return False if we are in distributed mode (i.e. exit can happen only
      * at sync), True otherwise.
      */
     static bool readyToExit(Tick delay);
     /**
      * Initiate taking a checkpoint
      * @param delay Delay param from the m5 checkpoint command. If Delay is
      * zero then a collaborative checkpoint is requested (i.e. all nodes have
      * to call this method before the checkpoint can be taken). If Delay is
      * not zero then a checkpoint is requested asap (and it will happen at the
      * next sync tick).
      * @return False if we are in dist mode (i.e. exit can happen only at
      * sync), True otherwise.
      */
     static bool readyToCkpt(Tick delay, Tick period);
     /**
      * Getter for the dist rank param.
      */
     static uint64_t rankParam();
     /**
      * Getter for the dist size param.
      */
     static uint64_t sizeParam();
     /**
      * Trigger the master to start/stop synchronization.
      */
     static void toggleSync(ThreadContext *tc);
  };

 #endif
	/*
	* Copyright (c) 2015-2016 ARM Limited
	* All rights reserved
	*
	* The license below extends only to copyright in the software and shall
	* not be construed as granting a license to any other intellectual
	* property including but not limited to intellectual property relating
	* to a hardware implementation of the functionality of the software
	* licensed hereunder. You may use the software subject to the license
	* terms below provided that you ensure that this notice is replicated
	* unmodified and in its entirety in all distributions of the software,
	* modified or unmodified, in source code or in binary form.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions are
	* met: redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer;
	* redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution;
	* neither the name of the copyright holders nor the names of its
	* contributors may be used to endorse or promote products derived from
	* this software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*
	* Authors: Gabor Dozsa
	*/

	/* @file
	* The interface class for dist gem5 simulations.
	*
	* dist-gem5 is an extension to gem5 to enable parallel simulation of a
	* distributed system (e.g. simulation of a pool of machines
	* connected by Ethernet links). A dist gem5 run consists of seperate gem5
	* processes running in parallel. Each gem5 process executes
	* the simulation of a component of the simulated distributed system.
	* (An example component can be a dist-core board with an Ethernet NIC.)
	* The DistIface class below provides services to transfer data and
	* control messages among the gem5 processes. The main such services are
	* as follows.
	*
	* 1. Send a data packet coming from a simulated Ethernet link. The packet
	* will be transferred to (all) the target(s) gem5 processes. The send
	* operation is always performed by the simulation thread, i.e. the gem5
	* thread that is processing the event queue associated with the simulated
	* Ethernet link.
	*
	* 2. Spawn a receiver thread to process messages coming in from the
	* from other gem5 processes. Each simulated Ethernet link has its own
	* associated receiver thread. The receiver thread saves the incoming packet
	* and schedule an appropriate receive event in the event queue.
	*
	* 3. Schedule a global barrier event periodically to keep the gem5
	* processes in sync.
	* Periodic barrier event to keep peer gem5 processes in sync. The basic idea
	* is that no gem5 process can go ahead further than the simulated link
	* transmission delay to ensure that a corresponding receive event can always
	* be scheduled for any message coming in from a peer gem5 process.
	*
	*
	*
	* This interface is an abstract class. It can work with various low level
	* send/receive service implementations (e.g. TCP/IP, MPI,...). A TCP
	* stream socket version is implemented in src/dev/net/tcp_iface.[hh,cc].
	*/
	#ifndef __DEV_DIST_IFACE_HH__
	#define __DEV_DIST_IFACE_HH__

	#include <array>
	#include <mutex>
	#include <queue>
	#include <thread>
	#include <utility>

	#include "base/logging.hh"
	#include "dev/net/dist_packet.hh"
	#include "dev/net/etherpkt.hh"
	#include "sim/core.hh"
	#include "sim/drain.hh"
	#include "sim/global_event.hh"
	#include "sim/serialize.hh"

	class EventManager;
	class System;
	class ThreadContext;

	/**
	* The interface class to talk to peer gem5 processes.
	*/
	class DistIface : public Drainable, public Serializable
	{
	public:
	typedef DistHeaderPkt::Header Header;

	protected:
	typedef DistHeaderPkt::MsgType MsgType;
	typedef DistHeaderPkt::ReqType ReqType;

	private:
	class SyncEvent;
	/** @class Sync
	* This class implements global sync operations among gem5 peer processes.
	*
	* @note This class is used as a singleton object (shared by all DistIface
	* objects).
	*/
	class Sync : public Serializable
	{
	protected:
	/**
	* The lock to protect access to the Sync object.
	*/
	std::mutex lock;
	/**
	* Condition variable for the simulation thread to wait on
	* until all receiver threads completes the current global
	* synchronisation.
	*/
	std::condition_variable cv;
	/**
	* Number of receiver threads that not yet completed the current global
	* synchronisation.
	*/
	unsigned waitNum;
	/**
	* Flag is set if exit is permitted upon sync completion
	*/
	bool doExit;
	/**
	* Flag is set if taking a ckpt is permitted upon sync completion
	*/
	bool doCkpt;
	/**
	* Flag is set if sync is to stop upon sync completion
	*/
	bool doStopSync;
	/**
	* The repeat value for the next periodic sync
	*/
	Tick nextRepeat;
	/**
	* Tick for the next periodic sync (if the event is not scheduled yet)
	*/
	Tick nextAt;
	/**
	* Flag is set if the sync is aborted (e.g. due to connection lost)
	*/
	bool isAbort;

	friend class SyncEvent;

	public:
	/**
	* Initialize periodic sync params.
	*
	* @param start Start tick for dist synchronisation
	* @param repeat Frequency of dist synchronisation
	*
	*/
	void init(Tick start, Tick repeat);
	/**
	* Core method to perform a full dist sync.
	*
	* @return true if the sync completes, false if it gets aborted
	*/
	virtual bool run(bool same_tick) = 0;
	/**
	* Callback when the receiver thread gets a sync ack message.
	*
	* @return false if the receiver thread needs to stop (e.g.
	* simulation is to exit)
	*/
	virtual bool progress(Tick send_tick,
	Tick next_repeat,
	ReqType do_ckpt,
	ReqType do_exit,
	ReqType do_stop_sync) = 0;
	/**
	* Abort processing an on-going sync event (in case of an error, e.g.
	* lost connection to a peer gem5)
	*/
	void abort();

	virtual void requestCkpt(ReqType req) = 0;
	virtual void requestExit(ReqType req) = 0;
	virtual void requestStopSync(ReqType req) = 0;

	void drainComplete();

	virtual void serialize(CheckpointOut &cp) const override = 0;
	virtual void unserialize(CheckpointIn &cp) override = 0;
	};

	class SyncNode: public Sync
	{
	private:
	/**
	* Exit requested
	*/
	ReqType needExit;
	/**
	* Ckpt requested
	*/
	ReqType needCkpt;
	/**
	* Sync stop requested
	*/
	ReqType needStopSync;

	public:

	SyncNode();
	~SyncNode() {}
	bool run(bool same_tick) override;
	bool progress(Tick max_req_tick,
	Tick next_repeat,
	ReqType do_ckpt,
	ReqType do_exit,
	ReqType do_stop_sync) override;

	void requestCkpt(ReqType req) override;
	void requestExit(ReqType req) override;
	void requestStopSync(ReqType req) override;

	void serialize(CheckpointOut &cp) const override;
	void unserialize(CheckpointIn &cp) override;
	};

	class SyncSwitch: public Sync
	{
	private:
	/**
	* Counter for recording exit requests
	*/
	unsigned numExitReq;
	/**
	* Counter for recording ckpt requests
	*/
	unsigned numCkptReq;
	/**
	* Counter for recording stop sync requests
	*/
	unsigned numStopSyncReq;
	/**
	* Number of connected simulated nodes
	*/
	unsigned numNodes;

	public:
	SyncSwitch(int num_nodes);
	~SyncSwitch() {}

	bool run(bool same_tick) override;
	bool progress(Tick max_req_tick,
	Tick next_repeat,
	ReqType do_ckpt,
	ReqType do_exit,
	ReqType do_stop_sync) override;

	void requestCkpt(ReqType) override {
	panic("Switch requested checkpoint");
	}
	void requestExit(ReqType) override {
	panic("Switch requested exit");
	}
	void requestStopSync(ReqType) override {
	panic("Switch requested stop sync");
	}

	void serialize(CheckpointOut &cp) const override;
	void unserialize(CheckpointIn &cp) override;
	};

	/**
	* The global event to schedule periodic dist sync. It is used as a
	* singleton object.
	*
	* The periodic synchronisation works as follows.
	* 1. A SyncEvent is scheduled as a global event when startup() is
	* called.
	* 2. The process() method of the SyncEvent initiates a new barrier
	* for each simulated Ethernet link.
	* 3. Simulation thread(s) then waits until all receiver threads
	* complete the ongoing barrier. The global sync event is done.
	*/
	class SyncEvent : public GlobalSyncEvent
	{
	private:
	/**
	* Flag to set when the system is draining
	*/
	bool _draining;
	public:
	/**
	* Only the firstly instantiated DistIface object will
	* call this constructor.
	*/
	SyncEvent() : GlobalSyncEvent(Sim_Exit_Pri, 0), _draining(false) {}

	~SyncEvent() {}
	/**
	* Schedule the first periodic sync event.
	*/
	void start();
	/**
	* This is a global event so process() will only be called by
	* exactly one simulation thread. (See further comments in the .cc
	* file.)
	*/
	void process() override;

	bool draining() const { return _draining; }
	void draining(bool fl) { _draining = fl; }
	};
	/**
	* Class to encapsulate information about data packets received.

	* @note The main purpose of the class to take care of scheduling receive
	* done events for the simulated network link and store incoming packets
	* until they can be received by the simulated network link.
	*/
	class RecvScheduler : public Serializable
	{
	private:
	/**
	* Received packet descriptor. This information is used by the receive
	* thread to schedule receive events and by the simulation thread to
	* process those events.
	*/
	struct Desc : public Serializable
	{
	EthPacketPtr packet;
	Tick sendTick;
	Tick sendDelay;

	Desc() : sendTick(0), sendDelay(0) {}
	Desc(EthPacketPtr p, Tick s, Tick d) :
	packet(p), sendTick(s), sendDelay(d) {}
	Desc(const Desc &d) :
	packet(d.packet), sendTick(d.sendTick), sendDelay(d.sendDelay) {}

	void serialize(CheckpointOut &cp) const override;
	void unserialize(CheckpointIn &cp) override;
	};
	/**
	* The queue to store the receive descriptors.
	*/
	std::queue<Desc> descQueue;
	/**
	* The tick when the most recent receive event was processed.
	*
	* @note This information is necessary to simulate possible receiver
	* link contention when calculating the receive tick for the next
	* incoming data packet (see the calcReceiveTick() method)
	*/
	Tick prevRecvTick;
	/**
	* The receive done event for the simulated Ethernet link.
	*
	* @note This object is constructed by the simulated network link. We
	* schedule this object for each incoming data packet.
	*/
	Event *recvDone;
	/**
	* The link delay in ticks for the simulated Ethernet link.
	*
	* @note This value is used for calculating the receive ticks for
	* incoming data packets.
	*/
	Tick linkDelay;
	/**
	* The event manager associated with the simulated Ethernet link.
	*
	* @note It is used to access the event queue for scheduling receive
	* done events for the link.
	*/
	EventManager *eventManager;
	/**
	* Calculate the tick to schedule the next receive done event.
	*
	* @param send_tick The tick the packet was sent.
	* @param send_delay The simulated delay at the sender side.
	* @param prev_recv_tick Tick when the last receive event was
	* processed.
	*
	* @note This method tries to take into account possible receiver link
	* contention and adjust receive tick for the incoming packets
	* accordingly.
	*/
	Tick calcReceiveTick(Tick send_tick,
	Tick send_delay,
	Tick prev_recv_tick);

	/**
	* Flag to set if receive ticks for pending packets need to be
	* recalculated due to changed link latencies at a resume
	*/
	bool ckptRestore;

	public:
	/**
	* Scheduler for the incoming data packets.
	*
	* @param em The event manager associated with the simulated Ethernet
	* link.
	*/
	RecvScheduler(EventManager *em) :
	prevRecvTick(0), recvDone(nullptr), linkDelay(0),
	eventManager(em), ckptRestore(false) {}

	/**
	* Initialize network link parameters.
	*
	* @note This method is called from the receiver thread (see
	* recvThreadFunc()).
	*/
	void init(Event *recv_done, Tick link_delay);
	/**
	* Fetch the next packet that is to be received by the simulated network
	* link.
	*
	* @note This method is called from the process() method of the receive
	* done event associated with the network link.
	*/
	EthPacketPtr popPacket();
	/**
	* Push a newly arrived packet into the desc queue.
	*/
	void pushPacket(EthPacketPtr new_packet,
	Tick send_tick,
	Tick send_delay);

	void serialize(CheckpointOut &cp) const override;
	void unserialize(CheckpointIn &cp) override;
	/**
	* Adjust receive ticks for pending packets when restoring from a
	* checkpoint
	*
	* @note Link speed and delay parameters may change at resume.
	*/
	void resumeRecvTicks();
	};
	/**
	* Tick to schedule the first dist sync event.
	* This is just as optimization : we do not need any dist sync
	* event until the simulated NIC is brought up by the OS.
	*/
	Tick syncStart;
	/**
	* Frequency of dist sync events in ticks.
	*/
	Tick syncRepeat;
	/**
	* Receiver thread pointer.
	* Each DistIface object must have exactly one receiver thread.
	*/
	std::thread *recvThread;
	/**
	* Meta information about data packets received.
	*/
	RecvScheduler recvScheduler;
	/**
	* Use pseudoOp to start synchronization.
	*/
	bool syncStartOnPseudoOp;

	protected:
	/**
	* The rank of this process among the gem5 peers.
	*/
	unsigned rank;
	/**
	* The number of gem5 processes comprising this dist simulation.
	*/
	unsigned size;
	/**
	* Number of DistIface objects (i.e. dist links in this gem5 process)
	*/
	static unsigned distIfaceNum;
	/**
	* Unique id for the dist link
	*/
	unsigned distIfaceId;

	bool isMaster;

	private:
	/**
	* Number of receiver threads (in this gem5 process)
	*/
	static unsigned recvThreadsNum;
	/**
	* The singleton Sync object to perform dist synchronisation.
	*/
	static Sync *sync;
	/**
	* The singleton SyncEvent object to schedule periodic dist sync.
	*/
	static SyncEvent *syncEvent;
	/**
	* The very first DistIface object created becomes the master. We need
	* a master to co-ordinate the global synchronisation.
	*/
	static DistIface *master;
	/**
	* System pointer used to wakeup sleeping threads when stopping sync.
	*/
	static System *sys;
	/**
	* Is this node a switch?
	*/
	static bool isSwitch;

	private:
	/**
	* Send out a data packet to the remote end.
	* @param header Meta info about the packet (which needs to be transferred
	* to the destination alongside the packet).
	* @param packet Pointer to the packet to send.
	*/
	virtual void sendPacket(const Header &header, const EthPacketPtr &packet) = 0;
	/**
	* Send out a control command to the remote end.
	* @param header Meta info describing the command (e.g. sync request)
	*/
	virtual void sendCmd(const Header &header) = 0;
	/**
	* Receive a header (i.e. meta info describing a data packet or a control command)
	* from the remote end.
	* @param header The meta info structure to store the incoming header.
	*/
	virtual bool recvHeader(Header &header) = 0;
	/**
	* Receive a packet from the remote end.
	* @param header Meta info about the incoming packet (obtanied by a previous
	* call to the recvHedaer() method).
	* @param Pointer to packet received.
	*/
	virtual void recvPacket(const Header &header, EthPacketPtr &packet) = 0;
	/**
	* Init hook for the underlaying transport
	*/
	virtual void initTransport() = 0;
	/**
	* spawn the receiver thread.
	* @param recv_done The receive done event associated with the simulated
	* Ethernet link.
	* @param link_delay The link delay for the simulated Ethernet link.
	*/
	void spawnRecvThread(const Event *recv_done, Tick link_delay);
	/**
	* The function executed by a receiver thread.
	*/
	void recvThreadFunc(Event *recv_done, Tick link_delay);

	public:

	/**
	* ctor
	* @param dist_rank Rank of this gem5 process within the dist run
	* @param sync_start Start tick for dist synchronisation
	* @param sync_repeat Frequency for dist synchronisation
	* @param em The event manager associated with the simulated Ethernet link
	*/
	DistIface(unsigned dist_rank,
	unsigned dist_size,
	Tick sync_start,
	Tick sync_repeat,
	EventManager *em,
	bool use_pseudo_op,
	bool is_switch,
	int num_nodes);

	virtual ~DistIface();
	/**
	* Send out an Ethernet packet.
	* @param pkt The Ethernet packet to send.
	* @param send_delay The delay in ticks for the send completion event.
	*/
	void packetOut(EthPacketPtr pkt, Tick send_delay);
	/**
	* Fetch the packet scheduled to be received next by the simulated
	* network link.
	*
	* @note This method is called within the process() method of the link
	* receive done event. It also schedules the next receive event if the
	* receive queue is not empty.
	*/
	EthPacketPtr packetIn() { return recvScheduler.popPacket(); }

	DrainState drain() override;
	void drainResume() override;
	void init(const Event *e, Tick link_delay);
	void startup();

	void serialize(CheckpointOut &cp) const override;
	void unserialize(CheckpointIn &cp) override;
	/**
	* Initiate the exit from the simulation.
	* @param delay Delay param from the m5 exit command. If Delay is zero
	* then a collaborative exit is requested (i.e. all nodes have to call
	* this method before the distributed simulation can exit). If Delay is
	* not zero then exit is requested asap (and it will happen at the next
	* sync tick).
	* @return False if we are in distributed mode (i.e. exit can happen only
	* at sync), True otherwise.
	*/
	static bool readyToExit(Tick delay);
	/**
	* Initiate taking a checkpoint
	* @param delay Delay param from the m5 checkpoint command. If Delay is
	* zero then a collaborative checkpoint is requested (i.e. all nodes have
	* to call this method before the checkpoint can be taken). If Delay is
	* not zero then a checkpoint is requested asap (and it will happen at the
	* next sync tick).
	* @return False if we are in dist mode (i.e. exit can happen only at
	* sync), True otherwise.
	*/
	static bool readyToCkpt(Tick delay, Tick period);
	/**
	* Getter for the dist rank param.
	*/
	static uint64_t rankParam();
	/**
	* Getter for the dist size param.
	*/
	static uint64_t sizeParam();
	/**
	* Trigger the master to start/stop synchronization.
	*/
	static void toggleSync(ThreadContext *tc);
	};

	#endif