File: vsip/impl/simd/eval-generic.hpp
1|
2|
3|
4|
5|
6|
7|
8|
9| #ifndef VSIP_IMPL_SIMD_EVAL_GENERIC_HPP
10| #define VSIP_IMPL_SIMD_EVAL_GENERIC_HPP
11|
12|
13|
14|
15|
16| #include <vsip/support.hpp>
17| #include <vsip/impl/block-traits.hpp>
18| #include <vsip/impl/expr_serial_evaluator.hpp>
19| #include <vsip/impl/expr_scalar_block.hpp>
20| #include <vsip/impl/expr_binary_block.hpp>
21| #include <vsip/impl/expr_operations.hpp>
22| #include <vsip/impl/fns_elementwise.hpp>
23| #include <vsip/impl/extdata.hpp>
24| #include <vsip/impl/coverage.hpp>
25|
26| #include <vsip/impl/simd/simd.hpp>
27| #include <vsip/impl/simd/vadd.hpp>
28| #include <vsip/impl/simd/vmul.hpp>
29| #include <vsip/impl/simd/rscvmul.hpp>
30| #include <vsip/impl/simd/vgt.hpp>
31| #include <vsip/impl/simd/vlogic.hpp>
32|
33|
34|
35|
36|
37| namespace vsip
38| {
39| namespace impl
40| {
41| namespace simd
42| {
43|
44| template <template <typename, typename> class Operator>
45| struct Map_operator_to_algorithm
46| {
47| typedef Alg_none type;
48| };
49|
50| template <>
51| struct Map_operator_to_algorithm<op::Add> { typedef Alg_vadd type; };
52| template <>
53| struct Map_operator_to_algorithm<op::Mult> { typedef Alg_vmul type; };
54| template <>
55| struct Map_operator_to_algorithm<band_functor> { typedef Alg_vband type; };
56| template <>
57| struct Map_operator_to_algorithm<bor_functor> { typedef Alg_vbor type; };
58| template <>
59| struct Map_operator_to_algorithm<bxor_functor> { typedef Alg_vbxor type; };
60|
61|
62|
63| template <template <typename, typename> class Operator,
64| typename DstBlock,
65| typename LBlock,
66| typename RBlock,
67| typename LType,
68| typename RType>
69| struct Serial_expr_evaluator_base
70| {
71| typedef Binary_expr_block<1, Operator, LBlock, LType, RBlock, RType>
72| SrcBlock;
73|
74| static bool const ct_valid =
75| !Is_expr_block<LBlock>::value &&
76| !Is_expr_block<RBlock>::value &&
77| simd::Is_algorithm_supported<
78| typename DstBlock::value_type,
79| Is_split_block<DstBlock>::value,
80| typename Map_operator_to_algorithm<Operator>::type>::value &&
81| Type_equal<typename DstBlock::value_type, LType>::value &&
82| Type_equal<typename DstBlock::value_type, RType>::value &&
83|
84| Ext_data_cost<DstBlock>::value == 0 &&
85| Ext_data_cost<LBlock>::value == 0 &&
86| Ext_data_cost<RBlock>::value == 0 &&
87|
88| Type_equal<typename Block_layout<DstBlock>::complex_type,
89| typename Block_layout<LBlock>::complex_type>::value &&
90| Type_equal<typename Block_layout<DstBlock>::complex_type,
91| typename Block_layout<RBlock>::complex_type>::value;
92|
93|
94| static bool rt_valid(DstBlock& dst, SrcBlock const& src)
95| {
96| // check if all data is unit stride
97| Ext_data<DstBlock> ext_dst(dst, SYNC_OUT);
98| Ext_data<LBlock> ext_l(src.left(), SYNC_IN);
99| Ext_data<RBlock> ext_r(src.right(), SYNC_IN);
100| return (ext_dst.stride(0) == 1 &&
101| ext_l.stride(0) == 1 &&
102| ext_r.stride(0) == 1);
103| }
104| };
105| }
106|
107|
108|
109| #define VSIP_IMPL_SIMD_V_EXPR(OP, ALG, FCN)
110| template <typename DstBlock,
111| typename LBlock,
112| typename LType>
113| struct Serial_expr_evaluator<
114| 1, DstBlock,
115| const Unary_expr_block<1, OP, LBlock, LType>,
116| Simd_builtin_tag>
117| {
118| typedef Unary_expr_block<1, OP, LBlock, LType>
119| SrcBlock;
120| \
121| typedef typename Adjust_layout_dim<
122| 1, typename Block_layout<DstBlock>::layout_type>::type
123| dst_lp;
124| typedef typename Adjust_layout_dim<
125| 1, typename Block_layout<LBlock>::layout_type>::type
126| lblock_lp;
127| \
128| static char const* name() { return "Expr_SIMD_V-" #FCN; }
129| \
130| static bool const ct_valid =
131| !Is_expr_block<LBlock>::value &&
132| simd::Is_algorithm_supported<
133| typename DstBlock::value_type,
134| Is_split_block<DstBlock>::value,
135| ALG>::value &&
136| Type_equal<typename DstBlock::value_type, LType>::value &&
137| /* check that direct access is supported */
138| Ext_data_cost<DstBlock>::value == 0 &&
139| Ext_data_cost<LBlock>::value == 0 &&
140| /* Must have same complex interleaved/split format */
141| Type_equal<typename Block_layout<DstBlock>::complex_type,
142| typename Block_layout<LBlock>::complex_type>::value;
143| \
144| static bool rt_valid(DstBlock& dst, SrcBlock const& src)
145| {
146| /* check if all data is unit stride */
147| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
148| Ext_data<LBlock, lblock_lp> ext_l(src.op(), SYNC_IN);
149| return (ext_dst.stride(0) == 1 &&
150| ext_l.stride(0) == 1);
151| }
152| \
153| static void exec(DstBlock& dst, SrcBlock const& src)
154| {
155| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
156| Ext_data<LBlock, lblock_lp> ext_l(src.op(), SYNC_IN);
157| FCN(ext_l.data(), ext_dst.data(), dst.size());
158| }
159| };
160|
161| #define VSIP_IMPL_SIMD_VV_EXPR(OP, FCN)
162| template <typename DstBlock,
163| typename LBlock,
164| typename RBlock,
165| typename LType,
166| typename RType>
167| struct Serial_expr_evaluator<
168| 1, DstBlock,
169| const Binary_expr_block<1, OP, LBlock, LType, RBlock, RType>,
170| Simd_builtin_tag>
171| {
172| typedef Binary_expr_block<1, OP, LBlock, LType, RBlock, RType>
173| SrcBlock;
174| \
175| typedef typename Adjust_layout_dim<
176| 1, typename Block_layout<DstBlock>::layout_type>::type
177| dst_lp;
178| typedef typename Adjust_layout_dim<
179| 1, typename Block_layout<LBlock>::layout_type>::type
180| lblock_lp;
181| typedef typename Adjust_layout_dim<
182| 1, typename Block_layout<RBlock>::layout_type>::type
183| rblock_lp;
184| \
185| static char const* name() { return "Expr_SIMD_VV-" #FCN; }
186| \
187| static bool const ct_valid =
188| !Is_expr_block<LBlock>::value &&
189| !Is_expr_block<RBlock>::value &&
190| simd::Is_algorithm_supported<
191| typename DstBlock::value_type,
192| Is_split_block<DstBlock>::value,
193| typename simd::Map_operator_to_algorithm<OP>::type>::value &&
194| Type_equal<typename DstBlock::value_type, LType>::value &&
195| Type_equal<typename DstBlock::value_type, RType>::value &&
196| /* check that direct access is supported */
197| Ext_data_cost<DstBlock>::value == 0 &&
198| Ext_data_cost<LBlock>::value == 0 &&
199| Ext_data_cost<RBlock>::value == 0 &&
200| /* Must have same complex interleaved/split format */
201| Type_equal<typename Block_layout<DstBlock>::complex_type,
202| typename Block_layout<LBlock>::complex_type>::value &&
203| Type_equal<typename Block_layout<DstBlock>::complex_type,
204| typename Block_layout<RBlock>::complex_type>::value;
205| \
206| static bool rt_valid(DstBlock& dst, SrcBlock const& src)
207| {
208| /* check if all data is unit stride */
209| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
210| Ext_data<LBlock, lblock_lp> ext_l(src.left(), SYNC_IN);
211| Ext_data<RBlock, rblock_lp> ext_r(src.right(), SYNC_IN);
212| return (ext_dst.stride(0) == 1 &&
213| ext_l.stride(0) == 1 &&
214| ext_r.stride(0) == 1);
215| }
216| \
217| static void exec(DstBlock& dst, SrcBlock const& src)
218| {
219| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
220| Ext_data<LBlock, lblock_lp> ext_l(src.left(), SYNC_IN);
221| Ext_data<RBlock, rblock_lp> ext_r(src.right(), SYNC_IN);
222| FCN(ext_l.data(), ext_r.data(), ext_dst.data(), dst.size());
223| }
224| };
225|
226|
227| VSIP_IMPL_SIMD_V_EXPR (bnot_functor, simd::Alg_vbnot, simd::vbnot)
228|
229| VSIP_IMPL_SIMD_VV_EXPR(op::Mult, simd::vmul)
230| VSIP_IMPL_SIMD_VV_EXPR(op::Add, simd::vadd)
231| VSIP_IMPL_SIMD_VV_EXPR(band_functor, simd::vband)
232| VSIP_IMPL_SIMD_VV_EXPR(bor_functor, simd::vbor)
233| VSIP_IMPL_SIMD_VV_EXPR(bxor_functor, simd::vbxor)
234|
235| #undef VSIP_IMPL_SIMD_V_EXPR
236| #undef VSIP_IMPL_SIMD_VV_EXPR
237|
238|
239|
240|
241|
242|
243|
244| template <typename DstBlock,
245| typename LBlock,
246| typename RBlock,
247| typename LType,
248| typename RType>
249| struct Serial_expr_evaluator<
250| 1, DstBlock,
251| const Binary_expr_block<1, gt_functor, LBlock, LType, RBlock, RType>,
252| Simd_builtin_tag>
253| {
254| typedef Binary_expr_block<1, gt_functor, LBlock, LType, RBlock, RType>
255| SrcBlock;
256|
257| typedef typename Adjust_layout_dim<
258| 1, typename Block_layout<DstBlock>::layout_type>::type
259| dst_lp;
260| typedef typename Adjust_layout_dim<
261| 1, typename Block_layout<LBlock>::layout_type>::type
262| lblock_lp;
263| typedef typename Adjust_layout_dim<
264| 1, typename Block_layout<RBlock>::layout_type>::type
265| rblock_lp;
266|
267| static char const* name() { return "Expr_SIMD_VV-simd::vgt"; }
268|
269| static bool const ct_valid =
270| !Is_expr_block<LBlock>::value &&
271| !Is_expr_block<RBlock>::value &&
272| Type_equal<typename DstBlock::value_type, bool>::value &&
273| Type_equal<LType, RType>::value &&
274| simd::Is_algorithm_supported<LType, false, simd::Alg_vgt>::value &&
275|
276| Ext_data_cost<DstBlock>::value == 0 &&
277| Ext_data_cost<LBlock>::value == 0 &&
278| Ext_data_cost<RBlock>::value == 0;
279|
280|
281| static bool rt_valid(DstBlock& dst, SrcBlock const& src)
282| {
283| // check if all data is unit stride
284| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
285| Ext_data<LBlock, lblock_lp> ext_l (src.left(), SYNC_IN);
286| Ext_data<RBlock, rblock_lp> ext_r (src.right(), SYNC_IN);
287| return (ext_dst.stride(0) == 1 &&
288| ext_l.stride(0) == 1 &&
289| ext_r.stride(0) == 1);
290| }
291|
292| static void exec(DstBlock& dst, SrcBlock const& src)
293| {
294| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
295| Ext_data<LBlock, lblock_lp> ext_l (src.left(), SYNC_IN);
296| Ext_data<RBlock, rblock_lp> ext_r (src.right(), SYNC_IN);
297| simd::vgt(ext_l.data(), ext_r.data(), ext_dst.data(), dst.size());
298| }
299| };
300|
301|
302|
303|
304|
305|
306|
307| #define VSIP_IMPL_SIMD_LOGIC_V_EXPR(OP, ALG, FCN)
308| template <typename DstBlock,
309| typename BlockT>
310| struct Serial_expr_evaluator<
311| 1, DstBlock,
312| const Unary_expr_block<1, OP, BlockT, bool>,
313| Simd_builtin_tag>
314| {
315| typedef Unary_expr_block<1, OP, BlockT, bool>
316| SrcBlock;
317| \
318| typedef typename Adjust_layout_dim<
319| 1, typename Block_layout<DstBlock>::layout_type>::type
320| dst_lp;
321| typedef typename Adjust_layout_dim<
322| 1, typename Block_layout<BlockT>::layout_type>::type
323| block_lp;
324| \
325| static char const* name() { return "Expr_SIMD_V-" #FCN; }
326| \
327| static bool const ct_valid =
328| !Is_expr_block<BlockT>::value &&
329| Type_equal<typename DstBlock::value_type, bool>::value &&
330| simd::Is_algorithm_supported<bool, false, ALG>::value &&
331| /* check that direct access is supported */
332| Ext_data_cost<DstBlock>::value == 0 &&
333| Ext_data_cost<BlockT>::value == 0;
334| \
335| static bool rt_valid(DstBlock& dst, SrcBlock const& src)
336| {
337| /* check if all data is unit stride */
338| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
339| Ext_data<BlockT, block_lp> ext_l (src.op(), SYNC_IN);
340| return (ext_dst.stride(0) == 1 &&
341| ext_l.stride(0) == 1);
342| }
343| \
344| static void exec(DstBlock& dst, SrcBlock const& src)
345| {
346| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
347| Ext_data<BlockT, block_lp> ext_l (src.op(), SYNC_IN);
348| FCN(ext_l.data(), ext_dst.data(), dst.size());
349| }
350| };
351|
352| #define VSIP_IMPL_SIMD_LOGIC_VV_EXPR(OP, ALG, FCN)
353| template <typename DstBlock,
354| typename LBlock,
355| typename RBlock>
356| struct Serial_expr_evaluator<
357| 1, DstBlock,
358| const Binary_expr_block<1, OP, LBlock, bool, RBlock, bool>,
359| Simd_builtin_tag>
360| {
361| typedef Binary_expr_block<1, OP, LBlock, bool, RBlock, bool>
362| SrcBlock;
363| \
364| typedef typename Adjust_layout_dim<
365| 1, typename Block_layout<DstBlock>::layout_type>::type
366| dst_lp;
367| typedef typename Adjust_layout_dim<
368| 1, typename Block_layout<LBlock>::layout_type>::type
369| lblock_lp;
370| typedef typename Adjust_layout_dim<
371| 1, typename Block_layout<RBlock>::layout_type>::type
372| rblock_lp;
373| \
374| static char const* name() { return "Expr_SIMD_VV-" #FCN; }
375| \
376| static bool const ct_valid =
377| !Is_expr_block<LBlock>::value &&
378| !Is_expr_block<RBlock>::value &&
379| Type_equal<typename DstBlock::value_type, bool>::value &&
380| simd::Is_algorithm_supported<bool, false, ALG>::value &&\
381| /* check that direct access is supported */
382| Ext_data_cost<DstBlock>::value == 0 &&
383| Ext_data_cost<LBlock>::value == 0 &&
384| Ext_data_cost<RBlock>::value == 0;
385| \
386| static bool rt_valid(DstBlock& dst, SrcBlock const& src)
387| {
388| /* check if all data is unit stride */
389| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
390| Ext_data<LBlock, lblock_lp> ext_l (src.left(), SYNC_IN);
391| Ext_data<RBlock, rblock_lp> ext_r (src.right(), SYNC_IN);
392| return (ext_dst.stride(0) == 1 &&
393| ext_l.stride(0) == 1 &&
394| ext_r.stride(0) == 1);
395| }
396| \
397| static void exec(DstBlock& dst, SrcBlock const& src)
398| {
399| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
400| Ext_data<LBlock, lblock_lp> ext_l (src.left(), SYNC_IN);
401| Ext_data<RBlock, rblock_lp> ext_r (src.right(), SYNC_IN);
402| FCN(ext_l.data(), ext_r.data(), ext_dst.data(), dst.size());
403| }
404| };
405|
406| VSIP_IMPL_SIMD_LOGIC_V_EXPR (lnot_functor, simd::Alg_vlnot, simd::vlnot)
407| VSIP_IMPL_SIMD_LOGIC_VV_EXPR(land_functor, simd::Alg_vland, simd::vland)
408| VSIP_IMPL_SIMD_LOGIC_VV_EXPR(lor_functor, simd::Alg_vlor, simd::vlor)
409| VSIP_IMPL_SIMD_LOGIC_VV_EXPR(lxor_functor, simd::Alg_vlxor, simd::vlxor)
410|
411| #undef VSIP_IMPL_SIMD_LOGIC_V_EXPR
412| #undef VSIP_IMPL_SIMD_LOGIC_VV_EXPR
413|
414|
415|
416|
417|
418|
419|
420|
421| template <typename DstBlock,
422| typename T,
423| typename VBlock>
424| struct Serial_expr_evaluator<
425| 1, DstBlock,
426| const Binary_expr_block<1, op::Mult,
427| Scalar_block<1, T>, T,
428| VBlock, std::complex<T> >,
429| Simd_builtin_tag>
430| {
431| typedef Binary_expr_block<1, op::Mult,
432| Scalar_block<1, T>, T,
433| VBlock, complex<T> >
434| SrcBlock;
435|
436| typedef typename Adjust_layout_dim<
437| 1, typename Block_layout<DstBlock>::layout_type>::type
438| dst_lp;
439| typedef typename Adjust_layout_dim<
440| 1, typename Block_layout<VBlock>::layout_type>::type
441| vblock_lp;
442|
443| static char const* name() { return "Expr_SIMD_V-simd::rscvmul"; }
444|
445| static bool const ct_valid =
446| !Is_expr_block<VBlock>::value &&
447| simd::Is_algorithm_supported<
448| T,
449| Is_split_block<DstBlock>::value,
450| typename simd::Map_operator_to_algorithm<op::Mult>::type>::value &&
451|
452| Type_equal<typename DstBlock::value_type, std::complex<T> >::value &&
453|
454| Ext_data_cost<DstBlock>::value == 0 &&
455| Ext_data_cost<VBlock>::value == 0 &&
456|
457| Type_equal<typename Block_layout<DstBlock>::complex_type,
458| typename Block_layout<VBlock>::complex_type>::value;
459|
460| static bool rt_valid(DstBlock& dst, SrcBlock const& src)
461| {
462| // check if all data is unit stride
463| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
464| Ext_data<VBlock, vblock_lp> ext_r(src.right(), SYNC_IN);
465| return (ext_dst.stride(0) == 1 && ext_r.stride(0) == 1);
466| }
467|
468| static void exec(DstBlock& dst, SrcBlock const& src)
469| {
470| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
471| Ext_data<VBlock, vblock_lp> ext_r(src.right(), SYNC_IN);
472| simd::rscvmul(src.left().value(), ext_r.data(), ext_dst.data(),
473| dst.size());
474| }
475| };
476|
477|
478|
479|
480|
481| template <typename DstBlock,
482| typename T,
483| typename VBlock>
484| struct Serial_expr_evaluator<
485| 1, DstBlock,
486| const Binary_expr_block<1, op::Mult,
487| VBlock, std::complex<T>,
488| Scalar_block<1, T>, T>,
489| Simd_builtin_tag>
490| {
491| typedef Binary_expr_block<1, op::Mult,
492| VBlock, std::complex<T>,
493| Scalar_block<1, T>, T>
494| SrcBlock;
495|
496| typedef typename Adjust_layout_dim<
497| 1, typename Block_layout<DstBlock>::layout_type>::type
498| dst_lp;
499| typedef typename Adjust_layout_dim<
500| 1, typename Block_layout<VBlock>::layout_type>::type
501| vblock_lp;
502|
503| static char const* name() { return "Expr_SIMD_V-simd::rscvmul"; }
504|
505| static bool const ct_valid =
506| !Is_expr_block<VBlock>::value &&
507| simd::Is_algorithm_supported<
508| T,
509| Is_split_block<DstBlock>::value,
510| typename simd::Map_operator_to_algorithm<op::Mult>::type>::value &&
511|
512| Type_equal<typename DstBlock::value_type, std::complex<T> >::value &&
513|
514| Ext_data_cost<DstBlock>::value == 0 &&
515| Ext_data_cost<VBlock>::value == 0 &&
516|
517| Type_equal<typename Block_layout<DstBlock>::complex_type,
518| typename Block_layout<VBlock>::complex_type>::value;
519|
520| static bool rt_valid(DstBlock& dst, SrcBlock const& src)
521| {
522| // check if all data is unit stride
523| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
524| Ext_data<VBlock, vblock_lp> ext_l(src.left(), SYNC_IN);
525| return (ext_dst.stride(0) == 1 && ext_l.stride(0) == 1);
526| }
527|
528| static void exec(DstBlock& dst, SrcBlock const& src)
529| {
530| Ext_data<DstBlock, dst_lp> ext_dst(dst, SYNC_OUT);
531| Ext_data<VBlock, vblock_lp> ext_l(src.left(), SYNC_IN);
532| simd::rscvmul(src.right().value(), ext_l.data(), ext_dst.data(),
533| dst.size());
534| }
535| };
536|
537|
538|
539|
540| }
541| }
542|
543| #endif // VSIP_IMPL_SIMD_EVAL_GENERIC_HPP