Context.h

Go to the documentation of this file.

/************************************************************
 * The Data Acquisition Backbone Core (DABC)                *
 ************************************************************
 * Copyright (C) 2009 -                                     *
 * GSI Helmholtzzentrum fuer Schwerionenforschung GmbH      *
 * Planckstr. 1, 64291 Darmstadt, Germany                   *
 * Contact:  http://dabc.gsi.de                             *
 ************************************************************
 * This software can be used under the GPL license          *
 * agreements as stated in LICENSE.txt file                 *
 * which is part of the distribution.                       *
 ************************************************************/
 
#ifndef VERBS_Context
#define VERBS_Context
 
#include <infiniband/verbs.h>
 
#ifndef DABC_Object
#include "dabc/Object.h"
#endif
 
#ifndef DABC_MemoryPool
#include "dabc/MemoryPool.h"
#endif
 
 
namespace verbs {
 
   class OpenSM;
   class ContextRef;
 
   extern const char* typeThread;
   extern const char* typeDevice;
 
   // ________________________________________________________
 
 
   class Context : public dabc::Object  {
 
      friend class ContextRef;
 
      protected:
         uint8_t fIbPort;  // infiniband port number
 
         struct ibv_context *fContext;         // device context
         struct ibv_pd *fPD;                   // protection domain
         struct ibv_device_attr fDeviceAttr;  // device attributes
         struct ibv_port_attr fPortAttr;      // port attributes
 
         OpenSM      *fOsm; // osm object created when multicast is required
 
         static bool fThreadSafeVerbs;  // identifies if verbs is thread safe
 
         Context();
 
         bool CloseVerbs();
 
         bool OpenVerbs(bool withmulticast = false, const char* devicename = 0, int ibport = -1);
 
      public:
 
         virtual ~Context();
   };
 
   // _______________________________________________________________
 
   class ContextRef : public dabc::Reference {
 
      DABC_REFERENCE(ContextRef, dabc::Reference, Context)
 
      public:
 
         enum MulticastActions { mcst_Error = 0, mcst_Ok, mcst_Register, mcst_Query, mcst_Init, mcst_Unregister };
 
         bool OpenVerbs(bool withmulticast = false, const char* devicename = 0, int ibport = -1);
 
         int IbPort() const { return GetObject() ? GetObject()->fIbPort : 0; }
         bool IsMulticast() const { return GetObject() ? GetObject()->fOsm!=0 : false; }
         struct ibv_pd *pd() const { return GetObject()->fPD; }
         struct ibv_context *context() const { return GetObject()->fContext; }
         uint16_t lid() const { return GetObject()->fPortAttr.lid; }
         ibv_mtu mtu() const { return GetObject()->fPortAttr.active_mtu; }
         unsigned max_sge() const { return GetObject()->fDeviceAttr.max_sge - 1; }
 
         dabc::Reference RegisterPool(dabc::MemoryPool* pool);
 
         int GetGidIndex(ibv_gid* lookgid);
 
         struct ibv_ah* CreateAH(uint32_t dest_lid);
 
         int ManageMulticast(int action, ibv_gid& mgid, uint16_t& mlid);
         struct ibv_ah* CreateMAH(ibv_gid& mgid, uint32_t mlid);
   };
 
   // ________________________________________________________________________
 
   class PoolRegistry : public dabc::Object {
      protected:
         ContextRef         fContext;
         dabc::MemoryPool  *fPool;
         unsigned           fLastChangeCounter;
         unsigned           f_nummr;     // size of f_mr array
         struct ibv_mr*    *f_mr;
 
         virtual void ObjectCleanup();
 
      public:
         PoolRegistry(ContextRef ctx, dabc::MemoryPool* pool);
         virtual ~PoolRegistry();
 
         dabc::MemoryPool* GetPool() const { return fPool; }
 
         inline void CheckMRStructure()
         {
            if ((fPool!=0) && fPool->CheckChangeCounter(fLastChangeCounter)) {
               EOUT("POOL %p change counter now = %u", fPool, fLastChangeCounter);
               throw dabc::Exception("Memory pool structure should not be changed at all");
            }
         }
 
         inline uint32_t GetLkey(unsigned id) { return id < f_nummr ? f_mr[id]->lkey : 0; }
 
         virtual void ObjectDestroyed(dabc::Object* obj);
 
         void SyncMRStructure();
 
         void CleanMRStructure();
   };
 
   // ______________________________________________________
 
   class PoolRegistryRef : public dabc::Reference {
 
      DABC_REFERENCE(PoolRegistryRef, dabc::Reference, verbs::PoolRegistry)
 
   };
 
}
 
#endif