11 #ifndef EIGEN_COEFFBASED_PRODUCT_H
12 #define EIGEN_COEFFBASED_PRODUCT_H
31 template<
int Traversal,
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename RetScalar>
32 struct product_coeff_impl;
34 template<
int StorageOrder,
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
35 struct product_packet_impl;
37 template<
typename LhsNested,
typename RhsNested,
int NestingFlags>
38 struct traits<CoeffBasedProduct<LhsNested,RhsNested,NestingFlags> >
40 typedef MatrixXpr XprKind;
41 typedef typename remove_all<LhsNested>::type _LhsNested;
42 typedef typename remove_all<RhsNested>::type _RhsNested;
43 typedef typename scalar_product_traits<typename _LhsNested::Scalar, typename _RhsNested::Scalar>::ReturnType Scalar;
44 typedef typename promote_storage_type<typename traits<_LhsNested>::StorageKind,
45 typename traits<_RhsNested>::StorageKind>::ret StorageKind;
46 typedef typename promote_index_type<typename traits<_LhsNested>::Index,
47 typename traits<_RhsNested>::Index>::type Index;
50 LhsCoeffReadCost = _LhsNested::CoeffReadCost,
51 RhsCoeffReadCost = _RhsNested::CoeffReadCost,
52 LhsFlags = _LhsNested::Flags,
53 RhsFlags = _RhsNested::Flags,
55 RowsAtCompileTime = _LhsNested::RowsAtCompileTime,
56 ColsAtCompileTime = _RhsNested::ColsAtCompileTime,
57 InnerSize = EIGEN_SIZE_MIN_PREFER_FIXED(_LhsNested::ColsAtCompileTime, _RhsNested::RowsAtCompileTime),
59 MaxRowsAtCompileTime = _LhsNested::MaxRowsAtCompileTime,
60 MaxColsAtCompileTime = _RhsNested::MaxColsAtCompileTime,
65 SameType = is_same<typename _LhsNested::Scalar,typename _RhsNested::Scalar>::value,
68 && (ColsAtCompileTime ==
Dynamic
69 || ( (ColsAtCompileTime % packet_traits<Scalar>::size) == 0
75 && (RowsAtCompileTime ==
Dynamic
76 || ( (RowsAtCompileTime % packet_traits<Scalar>::size) == 0
81 EvalToRowMajor = (MaxRowsAtCompileTime==1&&MaxColsAtCompileTime!=1) ? 1
82 : (MaxColsAtCompileTime==1&&MaxRowsAtCompileTime!=1) ? 0
83 : (RhsRowMajor && !CanVectorizeLhs),
85 Flags = ((
unsigned int)(LhsFlags | RhsFlags) & HereditaryBits & ~RowMajorBit)
86 | (EvalToRowMajor ? RowMajorBit : 0)
90 | (SameType && (CanVectorizeLhs || CanVectorizeRhs) ?
PacketAccessBit : 0),
94 : InnerSize * (NumTraits<Scalar>::MulCost + LhsCoeffReadCost + RhsCoeffReadCost)
95 + (InnerSize - 1) * NumTraits<Scalar>::AddCost,
102 CanVectorizeInner = SameType
107 && (InnerSize % packet_traits<Scalar>::size == 0)
113 template<
typename LhsNested,
typename RhsNested,
int NestingFlags>
114 class CoeffBasedProduct
115 : internal::no_assignment_operator,
116 public MatrixBase<CoeffBasedProduct<LhsNested, RhsNested, NestingFlags> >
120 typedef MatrixBase<CoeffBasedProduct> Base;
121 EIGEN_DENSE_PUBLIC_INTERFACE(CoeffBasedProduct)
122 typedef typename Base::PlainObject PlainObject;
126 typedef typename internal::traits<CoeffBasedProduct>::_LhsNested _LhsNested;
127 typedef typename internal::traits<CoeffBasedProduct>::_RhsNested _RhsNested;
130 PacketSize = internal::packet_traits<Scalar>::size,
131 InnerSize = internal::traits<CoeffBasedProduct>::InnerSize,
132 Unroll = CoeffReadCost !=
Dynamic && CoeffReadCost <= EIGEN_UNROLLING_LIMIT,
133 CanVectorizeInner = internal::traits<CoeffBasedProduct>::CanVectorizeInner
136 typedef internal::product_coeff_impl<CanVectorizeInner ? InnerVectorizedTraversal : DefaultTraversal,
137 Unroll ? (InnerSize==0 ? 0 : InnerSize-1) :
Dynamic,
138 _LhsNested, _RhsNested, Scalar> ScalarCoeffImpl;
140 typedef CoeffBasedProduct<LhsNested,RhsNested,NestByRefBit> LazyCoeffBasedProductType;
144 inline CoeffBasedProduct(
const CoeffBasedProduct& other)
145 : Base(), m_lhs(other.m_lhs), m_rhs(other.m_rhs)
148 template<
typename Lhs,
typename Rhs>
149 inline CoeffBasedProduct(
const Lhs& lhs,
const Rhs& rhs)
150 : m_lhs(lhs), m_rhs(rhs)
154 EIGEN_STATIC_ASSERT((internal::scalar_product_traits<typename Lhs::RealScalar, typename Rhs::RealScalar>::Defined),
155 YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY)
156 eigen_assert(lhs.cols() == rhs.rows()
157 && "invalid matrix product"
158 && "if you wanted a coeff-wise or a dot product use the respective explicit functions");
161 EIGEN_STRONG_INLINE Index rows()
const {
return m_lhs.rows(); }
162 EIGEN_STRONG_INLINE Index cols()
const {
return m_rhs.cols(); }
164 EIGEN_STRONG_INLINE
const Scalar coeff(Index row, Index col)
const
167 ScalarCoeffImpl::run(row, col, m_lhs, m_rhs, res);
174 EIGEN_STRONG_INLINE
const Scalar coeff(Index index)
const
177 const Index row = RowsAtCompileTime == 1 ? 0 : index;
178 const Index col = RowsAtCompileTime == 1 ? index : 0;
179 ScalarCoeffImpl::run(row, col, m_lhs, m_rhs, res);
183 template<
int LoadMode>
184 EIGEN_STRONG_INLINE
const PacketScalar packet(Index row, Index col)
const
188 Unroll ? (InnerSize==0 ? 0 : InnerSize-1) :
Dynamic,
189 _LhsNested, _RhsNested, PacketScalar, LoadMode>
190 ::run(row, col, m_lhs, m_rhs, res);
195 EIGEN_STRONG_INLINE
operator const PlainObject& ()
const
197 m_result.lazyAssign(*
this);
201 const _LhsNested& lhs()
const {
return m_lhs; }
202 const _RhsNested& rhs()
const {
return m_rhs; }
204 const Diagonal<const LazyCoeffBasedProductType,0> diagonal()
const
205 {
return reinterpret_cast<const LazyCoeffBasedProductType&
>(*this); }
207 template<
int DiagonalIndex>
208 const Diagonal<const LazyCoeffBasedProductType,DiagonalIndex> diagonal()
const
209 {
return reinterpret_cast<const LazyCoeffBasedProductType&
>(*this); }
211 const Diagonal<const LazyCoeffBasedProductType,Dynamic> diagonal(Index index)
const
212 {
return reinterpret_cast<const LazyCoeffBasedProductType&
>(*this).diagonal(index); }
215 typename internal::add_const_on_value_type<LhsNested>::type m_lhs;
216 typename internal::add_const_on_value_type<RhsNested>::type m_rhs;
218 mutable PlainObject m_result;
225 template<
typename Lhs,
typename Rhs,
int N,
typename PlainObject>
228 typedef PlainObject
const& type;
239 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename RetScalar>
240 struct product_coeff_impl<DefaultTraversal, UnrollingIndex, Lhs, Rhs, RetScalar>
242 typedef typename Lhs::Index Index;
243 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, RetScalar &res)
245 product_coeff_impl<DefaultTraversal, UnrollingIndex-1, Lhs, Rhs, RetScalar>::run(row, col, lhs, rhs, res);
246 res += lhs.coeff(row, UnrollingIndex) * rhs.coeff(UnrollingIndex, col);
250 template<
typename Lhs,
typename Rhs,
typename RetScalar>
251 struct product_coeff_impl<DefaultTraversal, 0, Lhs, Rhs, RetScalar>
253 typedef typename Lhs::Index Index;
254 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, RetScalar &res)
256 res = lhs.coeff(row, 0) * rhs.coeff(0, col);
260 template<
typename Lhs,
typename Rhs,
typename RetScalar>
261 struct product_coeff_impl<DefaultTraversal,
Dynamic, Lhs, Rhs, RetScalar>
263 typedef typename Lhs::Index Index;
264 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, RetScalar& res)
266 res = (lhs.row(row).transpose().cwiseProduct( rhs.col(col) )).sum();
274 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet>
275 struct product_coeff_vectorized_unroller
277 typedef typename Lhs::Index Index;
278 enum { PacketSize = packet_traits<typename Lhs::Scalar>::size };
279 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::PacketScalar &pres)
281 product_coeff_vectorized_unroller<UnrollingIndex-PacketSize, Lhs, Rhs, Packet>::run(row, col, lhs, rhs, pres);
282 pres = padd(pres, pmul( lhs.template packet<Aligned>(row, UnrollingIndex) , rhs.template packet<Aligned>(UnrollingIndex, col) ));
286 template<
typename Lhs,
typename Rhs,
typename Packet>
287 struct product_coeff_vectorized_unroller<0, Lhs, Rhs, Packet>
289 typedef typename Lhs::Index Index;
290 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::PacketScalar &pres)
292 pres = pmul(lhs.template packet<Aligned>(row, 0) , rhs.template packet<Aligned>(0, col));
296 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename RetScalar>
297 struct product_coeff_impl<InnerVectorizedTraversal, UnrollingIndex, Lhs, Rhs, RetScalar>
299 typedef typename Lhs::PacketScalar Packet;
300 typedef typename Lhs::Index Index;
301 enum { PacketSize = packet_traits<typename Lhs::Scalar>::size };
302 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, RetScalar &res)
305 product_coeff_vectorized_unroller<UnrollingIndex+1-PacketSize, Lhs, Rhs, Packet>::run(row, col, lhs, rhs, pres);
306 product_coeff_impl<DefaultTraversal,UnrollingIndex,Lhs,Rhs,RetScalar>::run(row, col, lhs, rhs, res);
311 template<
typename Lhs,
typename Rhs,
int LhsRows = Lhs::RowsAtCompileTime,
int RhsCols = Rhs::ColsAtCompileTime>
312 struct product_coeff_vectorized_dyn_selector
314 typedef typename Lhs::Index Index;
315 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
317 res = lhs.row(row).transpose().cwiseProduct(rhs.col(col)).sum();
323 template<
typename Lhs,
typename Rhs,
int RhsCols>
324 struct product_coeff_vectorized_dyn_selector<Lhs,Rhs,1,RhsCols>
326 typedef typename Lhs::Index Index;
327 static EIGEN_STRONG_INLINE
void run(Index , Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
329 res = lhs.transpose().cwiseProduct(rhs.col(col)).sum();
333 template<
typename Lhs,
typename Rhs,
int LhsRows>
334 struct product_coeff_vectorized_dyn_selector<Lhs,Rhs,LhsRows,1>
336 typedef typename Lhs::Index Index;
337 static EIGEN_STRONG_INLINE
void run(Index row, Index ,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
339 res = lhs.row(row).transpose().cwiseProduct(rhs).sum();
343 template<
typename Lhs,
typename Rhs>
344 struct product_coeff_vectorized_dyn_selector<Lhs,Rhs,1,1>
346 typedef typename Lhs::Index Index;
347 static EIGEN_STRONG_INLINE
void run(Index , Index ,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
349 res = lhs.transpose().cwiseProduct(rhs).sum();
353 template<
typename Lhs,
typename Rhs,
typename RetScalar>
354 struct product_coeff_impl<InnerVectorizedTraversal,
Dynamic, Lhs, Rhs, RetScalar>
356 typedef typename Lhs::Index Index;
357 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
359 product_coeff_vectorized_dyn_selector<Lhs,Rhs>::run(row, col, lhs, rhs, res);
367 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
368 struct product_packet_impl<
RowMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode>
370 typedef typename Lhs::Index Index;
371 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet &res)
373 product_packet_impl<RowMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, res);
374 res = pmadd(pset1<Packet>(lhs.coeff(row, UnrollingIndex)), rhs.template packet<LoadMode>(UnrollingIndex, col), res);
378 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
379 struct product_packet_impl<
ColMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode>
381 typedef typename Lhs::Index Index;
382 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet &res)
384 product_packet_impl<ColMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, res);
385 res = pmadd(lhs.template packet<LoadMode>(row, UnrollingIndex), pset1<Packet>(rhs.coeff(UnrollingIndex, col)), res);
389 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
390 struct product_packet_impl<
RowMajor, 0, Lhs, Rhs, Packet, LoadMode>
392 typedef typename Lhs::Index Index;
393 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet &res)
395 res = pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode>(0, col));
399 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
400 struct product_packet_impl<
ColMajor, 0, Lhs, Rhs, Packet, LoadMode>
402 typedef typename Lhs::Index Index;
403 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet &res)
405 res = pmul(lhs.template packet<LoadMode>(row, 0), pset1<Packet>(rhs.coeff(0, col)));
409 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
410 struct product_packet_impl<
RowMajor,
Dynamic, Lhs, Rhs, Packet, LoadMode>
412 typedef typename Lhs::Index Index;
413 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet& res)
415 eigen_assert(lhs.cols()>0 &&
"you are using a non initialized matrix");
416 res = pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode>(0, col));
417 for(Index i = 1; i < lhs.cols(); ++i)
418 res = pmadd(pset1<Packet>(lhs.coeff(row, i)), rhs.template packet<LoadMode>(i, col), res);
422 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
423 struct product_packet_impl<
ColMajor,
Dynamic, Lhs, Rhs, Packet, LoadMode>
425 typedef typename Lhs::Index Index;
426 static EIGEN_STRONG_INLINE
void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet& res)
428 eigen_assert(lhs.cols()>0 &&
"you are using a non initialized matrix");
429 res = pmul(lhs.template packet<LoadMode>(row, 0), pset1<Packet>(rhs.coeff(0, col)));
430 for(Index i = 1; i < lhs.cols(); ++i)
431 res = pmadd(lhs.template packet<LoadMode>(row, i), pset1<Packet>(rhs.coeff(i, col)), res);
439 #endif // EIGEN_COEFFBASED_PRODUCT_H
const unsigned int EvalBeforeNestingBit
Definition: Constants.h:58
const int Dynamic
Definition: Constants.h:21
const unsigned int PacketAccessBit
Definition: Constants.h:81
const unsigned int ActualPacketAccessBit
Definition: Constants.h:92
const unsigned int EvalBeforeAssigningBit
Definition: Constants.h:63
const unsigned int RowMajorBit
Definition: Constants.h:53
const unsigned int AlignedBit
Definition: Constants.h:147
Definition: Constants.h:266
Definition: Constants.h:264