MatrixProductMMA.h File Reference

Go to the source code of this file.

Namespaces
	Eigen
	Namespace containing all symbols from the Eigen library.
	Eigen::internal

Macros
#define	PEEL_MMA   7
#define	MICRO_MMA_UNROLL(func)    func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)
#define	MICRO_MMA_LOAD_ONE(iter)
#define	MICRO_MMA_WORK_ONE(iter, type, peel)
#define	MICRO_MMA_TYPE_PEEL(func, func2, type, peel)
#define	MICRO_MMA_UNROLL_TYPE_PEEL(func, func2, type)
#define	MICRO_MMA_UNROLL_TYPE_ONE(func, func2, type)
#define	MICRO_MMA_ONE_PEEL
#define	MICRO_MMA_ONE
#define	MICRO_MMA_DST_PTR_ONE(iter)
#define	MICRO_MMA_DST_PTR   MICRO_MMA_UNROLL(MICRO_MMA_DST_PTR_ONE)
#define	MICRO_MMA_SRC_PTR_ONE(iter)
#define	MICRO_MMA_SRC_PTR   MICRO_MMA_UNROLL(MICRO_MMA_SRC_PTR_ONE)
#define	MICRO_MMA_PREFETCH_ONE(iter)
#define	MICRO_MMA_PREFETCH   MICRO_MMA_UNROLL(MICRO_MMA_PREFETCH_ONE)
#define	MICRO_MMA_STORE_ONE(iter)
#define	MICRO_MMA_STORE   MICRO_MMA_UNROLL(MICRO_MMA_STORE_ONE)
#define	MAX_MMA_UNROLL   7
#define	accColsC   (accCols / 2)
#define	advanceRows   ((LhsIsReal) ? 1 : 2)
#define	advanceCols   ((RhsIsReal) ? 1 : 2)
#define	PEEL_COMPLEX_MMA   7
#define	MICRO_COMPLEX_MMA_UNROLL(func)    func(0) func(1) func(2) func(3) func(4)
#define	MICRO_COMPLEX_MMA_LOAD_ONE(iter)
#define	MICRO_COMPLEX_MMA_WORK_ONE(iter, type, peel)
#define	MICRO_COMPLEX_MMA_TYPE_PEEL(func, func2, type, peel)
#define	MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(func, func2, type)
#define	MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(func, func2, type)
#define	MICRO_COMPLEX_MMA_ONE_PEEL
#define	MICRO_COMPLEX_MMA_ONE
#define	MICRO_COMPLEX_MMA_DST_PTR_ONE(iter)
#define	MICRO_COMPLEX_MMA_DST_PTR   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_DST_PTR_ONE)
#define	MICRO_COMPLEX_MMA_SRC_PTR_ONE(iter)
#define	MICRO_COMPLEX_MMA_SRC_PTR   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_SRC_PTR_ONE)
#define	MICRO_COMPLEX_MMA_PREFETCH_ONE(iter)
#define	MICRO_COMPLEX_MMA_PREFETCH   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_PREFETCH_ONE)
#define	MICRO_COMPLEX_MMA_STORE_ONE(iter)
#define	MICRO_COMPLEX_MMA_STORE   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE)
#define	MAX_COMPLEX_MMA_UNROLL   4

Functions
template<typename Scalar , typename Packet >
EIGEN_ALWAYS_INLINE void	Eigen::internal::bsetzeroMMA (__vector_quad *acc)
template<typename DataMapper , typename Index , typename Packet , const Index accCols>
EIGEN_ALWAYS_INLINE void	Eigen::internal::storeAccumulator (Index i, Index j, const DataMapper &data, const Packet &alpha, __vector_quad *acc)
template<typename DataMapper , typename Index , typename Packet , typename Packetc , const Index accColsC, int N>
EIGEN_ALWAYS_INLINE void	Eigen::internal::storeComplexAccumulator (Index i, Index j, const DataMapper &data, const Packet &alphaReal, const Packet &alphaImag, __vector_quad *accReal, __vector_quad *accImag)
template<typename LhsPacket , typename RhsPacket , bool NegativeAccumulate>
EIGEN_ALWAYS_INLINE void	Eigen::internal::pgerMMA (__vector_quad *acc, const RhsPacket &a, const LhsPacket &b)
template<typename LhsPacket , typename RhsPacket , bool NegativeAccumulate>
EIGEN_ALWAYS_INLINE void	Eigen::internal::pgerMMA (__vector_quad *acc, const PacketBlock< Packet2d, 2 > &a, const Packet2d &b)
template<typename LhsPacket , typename RhsPacket , bool NegativeAccumulate>
EIGEN_ALWAYS_INLINE void	Eigen::internal::pgerMMA (__vector_quad *acc, const __vector_pair &a, const Packet2d &b)
template<typename LhsPacket , typename RhsPacket , bool NegativeAccumulate>
EIGEN_ALWAYS_INLINE void	Eigen::internal::pgerMMA (__vector_quad *, const __vector_pair &, const Packet4f &)
template<typename Scalar , typename Packet , typename RhsPacket , bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_ALWAYS_INLINE void	Eigen::internal::pgercMMA (__vector_quad *accReal, __vector_quad *accImag, const Packet &lhsV, const Packet &lhsVi, const RhsPacket &rhsV, const RhsPacket &rhsVi)
template<typename Scalar , typename Packet >
EIGEN_ALWAYS_INLINE void	Eigen::internal::ploadRhsMMA (const Scalar *rhs, Packet &rhsV)
template<>
EIGEN_ALWAYS_INLINE void	Eigen::internal::ploadRhsMMA< double, PacketBlock< Packet2d, 2 > > (const double *rhs, PacketBlock< Packet2d, 2 > &rhsV)
template<>
EIGEN_ALWAYS_INLINE void	Eigen::internal::ploadRhsMMA< double, __vector_pair > (const double *rhs, __vector_pair &rhsV)
template<>
EIGEN_ALWAYS_INLINE void	Eigen::internal::ploadRhsMMA (const float *, __vector_pair &)
template<int unroll_factor, typename Scalar , typename Packet , typename RhsPacket , typename DataMapper , typename Index , const Index accRows, const Index accCols>
EIGEN_STRONG_INLINE void	Eigen::internal::gemm_unrolled_MMA_iteration (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index &row, Index col, const Packet &pAlpha)
template<typename Scalar , typename Index , typename Packet , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols>
void	Eigen::internal::gemmMMA (const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
template<int unroll_factor, typename Scalar , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , typename Index , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
EIGEN_STRONG_INLINE void	Eigen::internal::gemm_complex_unrolled_MMA_iteration (const DataMapper &res, const Scalar *lhs_base, const Scalar *rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index &row, Index col, const Packet &pAlphaReal, const Packet &pAlphaImag)
template<typename LhsScalar , typename RhsScalar , typename Scalarc , typename Scalar , typename Index , typename Packet , typename Packetc , typename RhsPacket , typename DataMapper , const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
void	Eigen::internal::gemm_complexMMA (const DataMapper &res, const LhsScalar *blockAc, const RhsScalar *blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)

Macro Definition Documentation

accColsC

#define accColsC   (accCols / 2)

advanceCols

#define advanceCols   ((RhsIsReal) ? 1 : 2)

advanceRows

#define advanceRows   ((LhsIsReal) ? 1 : 2)

MAX_COMPLEX_MMA_UNROLL

#define MAX_COMPLEX_MMA_UNROLL   4

MAX_MMA_UNROLL

#define MAX_MMA_UNROLL   7

MICRO_COMPLEX_MMA_DST_PTR

#define MICRO_COMPLEX_MMA_DST_PTR   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_DST_PTR_ONE)

MICRO_COMPLEX_MMA_DST_PTR_ONE

#define MICRO_COMPLEX_MMA_DST_PTR_ONE

(

iter

)

Value:

  if (unroll_factor > iter) { \
    bsetzeroMMA<Scalar, Packet>(&accReal##iter); \
    bsetzeroMMA<Scalar, Packet>(&accImag##iter); \
  } else { \
    EIGEN_UNUSED_VARIABLE(accReal##iter); \
    EIGEN_UNUSED_VARIABLE(accImag##iter); \
  }

MICRO_COMPLEX_MMA_LOAD_ONE

#define MICRO_COMPLEX_MMA_LOAD_ONE

(

iter

)

Value:

  if (unroll_factor > iter) { \
    lhsV##iter = ploadLhs<Scalar, Packet>(lhs_ptr_real##iter); \
    lhs_ptr_real##iter += accCols; \
    if(!LhsIsReal) { \
      lhsVi##iter = ploadLhs<Scalar, Packet>(lhs_ptr_imag##iter); \
      lhs_ptr_imag##iter += accCols; \
    } else { \
      EIGEN_UNUSED_VARIABLE(lhsVi##iter); \
    } \
  } else { \
    EIGEN_UNUSED_VARIABLE(lhsV##iter); \
    EIGEN_UNUSED_VARIABLE(lhsVi##iter); \
  }

MICRO_COMPLEX_MMA_ONE

#define MICRO_COMPLEX_MMA_ONE

Value:

  if (sizeof(Scalar) == sizeof(float)) { \
    MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, RhsPacket); \
  } else { \
    MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, __vector_pair); \
  } \
  rhs_ptr_real += accRows; \
  if(!RhsIsReal) rhs_ptr_imag += accRows;

MICRO_COMPLEX_MMA_ONE_PEEL

#define MICRO_COMPLEX_MMA_ONE_PEEL

Value:

  if (sizeof(Scalar) == sizeof(float)) { \
    MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, RhsPacket); \
  } else { \
    MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, __vector_pair); \
  } \
  rhs_ptr_real += (accRows * PEEL_COMPLEX_MMA); \
  if(!RhsIsReal) rhs_ptr_imag += (accRows * PEEL_COMPLEX_MMA);

MICRO_COMPLEX_MMA_PREFETCH

#define MICRO_COMPLEX_MMA_PREFETCH   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_PREFETCH_ONE)

MICRO_COMPLEX_MMA_PREFETCH_ONE

#define MICRO_COMPLEX_MMA_PREFETCH_ONE

(

iter

)

Value:

  if (unroll_factor > iter) { \
    EIGEN_POWER_PREFETCH(lhs_ptr_real##iter); \
    if(!LhsIsReal) { \
      EIGEN_POWER_PREFETCH(lhs_ptr_imag##iter); \
    } \
  }

MICRO_COMPLEX_MMA_SRC_PTR

#define MICRO_COMPLEX_MMA_SRC_PTR   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_SRC_PTR_ONE)

MICRO_COMPLEX_MMA_SRC_PTR_ONE

#define MICRO_COMPLEX_MMA_SRC_PTR_ONE

(

iter

)

Value:

  if (unroll_factor > iter) { \
    lhs_ptr_real##iter = lhs_base + ( ((advanceRows*row)/accCols) + iter*advanceRows )*strideA*accCols + accCols*offsetA; \
    if(!LhsIsReal) { \
      lhs_ptr_imag##iter = lhs_ptr_real##iter + accCols*strideA; \
    } else { \
      EIGEN_UNUSED_VARIABLE(lhs_ptr_imag##iter); \
    } \
  } else { \
    EIGEN_UNUSED_VARIABLE(lhs_ptr_real##iter); \
    EIGEN_UNUSED_VARIABLE(lhs_ptr_imag##iter); \
  }

MICRO_COMPLEX_MMA_STORE

#define MICRO_COMPLEX_MMA_STORE   MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE)

MICRO_COMPLEX_MMA_STORE_ONE

#define MICRO_COMPLEX_MMA_STORE_ONE

(

iter

)

Value:

  if (unroll_factor > iter) { \
    storeComplexAccumulator<DataMapper, Index, Packet, Packetc, accColsC, 0>(row + iter*accCols, col, res, pAlphaReal, pAlphaImag, &accReal##iter, &accImag##iter); \
  }

MICRO_COMPLEX_MMA_TYPE_PEEL

#define MICRO_COMPLEX_MMA_TYPE_PEEL	(		func,
			func2,
			type,
			peel
	)

Value:

  if (PEEL_COMPLEX_MMA > peel) { \
    Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4; \
    Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3, lhsVi4; \
    ploadRhsMMA<Scalar, type>(rhs_ptr_real + (accRows * peel), rhsV##peel); \
    if(!RhsIsReal) { \
      ploadRhsMMA<Scalar, type>(rhs_ptr_imag + (accRows * peel), rhsVi##peel); \
    } else { \
      EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
    } \
    MICRO_COMPLEX_MMA_UNROLL(func2); \
    func(0,type,peel) func(1,type,peel) func(2,type,peel) func(3,type,peel) func(4,type,peel) \
  } else { \
    EIGEN_UNUSED_VARIABLE(rhsV##peel); \
    EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
  }

MICRO_COMPLEX_MMA_UNROLL

#define MICRO_COMPLEX_MMA_UNROLL

(

func

)

func(0) func(1) func(2) func(3) func(4)

MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE

#define MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE	(		func,
			func2,
			type
	)

Value:

  type rhsV0, rhsVi0; \
  MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,0);

MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL

#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL	(		func,
			func2,
			type
	)

Value:

  type rhsV0, rhsV1, rhsV2, rhsV3, rhsV4, rhsV5, rhsV6, rhsV7, rhsV8, rhsV9; \
  type rhsVi0, rhsVi1, rhsVi2, rhsVi3, rhsVi4, rhsVi5, rhsVi6, rhsVi7, rhsVi8, rhsVi9; \
  MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,0); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,1); \
  MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,2); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,3); \
  MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,4); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,5); \
  MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,6); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,7); \
  MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,8); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,9);

MICRO_COMPLEX_MMA_WORK_ONE

#define MICRO_COMPLEX_MMA_WORK_ONE	(		iter,
			type,
			peel
	)

Value:

  if (unroll_factor > iter) { \
    pgercMMA<Scalar, Packet, type, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \
  }

MICRO_MMA_DST_PTR

#define MICRO_MMA_DST_PTR   MICRO_MMA_UNROLL(MICRO_MMA_DST_PTR_ONE)

MICRO_MMA_DST_PTR_ONE

#define MICRO_MMA_DST_PTR_ONE

(

iter

)

Value:

  if (unroll_factor > iter) { \
    bsetzeroMMA<Scalar, Packet>(&accZero##iter); \
  } else { \
    EIGEN_UNUSED_VARIABLE(accZero##iter); \
  }

MICRO_MMA_LOAD_ONE

#define MICRO_MMA_LOAD_ONE

(

iter

)

Value:

  if (unroll_factor > iter) { \
    lhsV##iter = ploadLhs<Scalar, Packet>(lhs_ptr##iter); \
    lhs_ptr##iter += accCols; \
  } else { \
    EIGEN_UNUSED_VARIABLE(lhsV##iter); \
  }

MICRO_MMA_ONE

#define MICRO_MMA_ONE

Value:

  if (sizeof(Scalar) == sizeof(float)) { \
    MICRO_MMA_UNROLL_TYPE_ONE(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, RhsPacket); \
  } else { \
    MICRO_MMA_UNROLL_TYPE_ONE(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, __vector_pair); \
  } \
  rhs_ptr += accRows;

MICRO_MMA_ONE_PEEL

#define MICRO_MMA_ONE_PEEL

Value:

  if (sizeof(Scalar) == sizeof(float)) { \
    MICRO_MMA_UNROLL_TYPE_PEEL(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, RhsPacket); \
  } else { \
    MICRO_MMA_UNROLL_TYPE_PEEL(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, __vector_pair); \
  } \
  rhs_ptr += (accRows * PEEL_MMA);

MICRO_MMA_PREFETCH

#define MICRO_MMA_PREFETCH   MICRO_MMA_UNROLL(MICRO_MMA_PREFETCH_ONE)

MICRO_MMA_PREFETCH_ONE

#define MICRO_MMA_PREFETCH_ONE

(

iter

)

Value:

  if (unroll_factor > iter) { \
    EIGEN_POWER_PREFETCH(lhs_ptr##iter); \
  }

MICRO_MMA_SRC_PTR

#define MICRO_MMA_SRC_PTR   MICRO_MMA_UNROLL(MICRO_MMA_SRC_PTR_ONE)

MICRO_MMA_SRC_PTR_ONE

#define MICRO_MMA_SRC_PTR_ONE

(

iter

)

Value:

  if (unroll_factor > iter) { \
    lhs_ptr##iter = lhs_base + ( (row/accCols) + iter )*strideA*accCols + accCols*offsetA; \
  } else { \
    EIGEN_UNUSED_VARIABLE(lhs_ptr##iter); \
  }

MICRO_MMA_STORE

#define MICRO_MMA_STORE   MICRO_MMA_UNROLL(MICRO_MMA_STORE_ONE)

MICRO_MMA_STORE_ONE

#define MICRO_MMA_STORE_ONE

(

iter

)

Value:

  if (unroll_factor > iter) { \
    storeAccumulator<DataMapper, Index, Packet, accCols>(row + iter*accCols, col, res, pAlpha, &accZero##iter); \
  }

MICRO_MMA_TYPE_PEEL

#define MICRO_MMA_TYPE_PEEL	(		func,
			func2,
			type,
			peel
	)

Value:

  if (PEEL_MMA > peel) { \
    Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \
    ploadRhsMMA<Scalar, type>(rhs_ptr + (accRows * peel), rhsV##peel); \
    MICRO_MMA_UNROLL(func2); \
    func(0,type,peel) func(1,type,peel) func(2,type,peel) func(3,type,peel) \
    func(4,type,peel) func(5,type,peel) func(6,type,peel) func(7,type,peel) \
  } else { \
    EIGEN_UNUSED_VARIABLE(rhsV##peel); \
  }

MICRO_MMA_UNROLL

#define MICRO_MMA_UNROLL

(

func

)

func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)

MICRO_MMA_UNROLL_TYPE_ONE

#define MICRO_MMA_UNROLL_TYPE_ONE	(		func,
			func2,
			type
	)

Value:

  type rhsV0; \
  MICRO_MMA_TYPE_PEEL(func,func2,type,0);

MICRO_MMA_UNROLL_TYPE_PEEL

#define MICRO_MMA_UNROLL_TYPE_PEEL	(		func,
			func2,
			type
	)

Value:

  type rhsV0, rhsV1, rhsV2, rhsV3, rhsV4, rhsV5, rhsV6, rhsV7, rhsV8, rhsV9; \
  MICRO_MMA_TYPE_PEEL(func,func2,type,0); MICRO_MMA_TYPE_PEEL(func,func2,type,1); \
  MICRO_MMA_TYPE_PEEL(func,func2,type,2); MICRO_MMA_TYPE_PEEL(func,func2,type,3); \
  MICRO_MMA_TYPE_PEEL(func,func2,type,4); MICRO_MMA_TYPE_PEEL(func,func2,type,5); \
  MICRO_MMA_TYPE_PEEL(func,func2,type,6); MICRO_MMA_TYPE_PEEL(func,func2,type,7); \
  MICRO_MMA_TYPE_PEEL(func,func2,type,8); MICRO_MMA_TYPE_PEEL(func,func2,type,9);

MICRO_MMA_WORK_ONE

#define MICRO_MMA_WORK_ONE	(		iter,
			type,
			peel
	)

Value:

  if (unroll_factor > iter) { \
    pgerMMA<Packet, type, false>(&accZero##iter, rhsV##peel, lhsV##iter); \
  }

PEEL_COMPLEX_MMA

#define PEEL_COMPLEX_MMA   7

PEEL_MMA

#define PEEL_MMA   7