Vc  1.2.0
SIMD Vector Classes for C++
scatterimplementation.h
1 /* This file is part of the Vc library. {{{
2 Copyright © 2014-2015 Matthias Kretz <kretz@kde.org>
3 All rights reserved.
4 
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions are met:
7  * Redistributions of source code must retain the above copyright
8  notice, this list of conditions and the following disclaimer.
9  * Redistributions in binary form must reproduce the above copyright
10  notice, this list of conditions and the following disclaimer in the
11  documentation and/or other materials provided with the distribution.
12  * Neither the names of contributing organizations nor the
13  names of its contributors may be used to endorse or promote products
14  derived from this software without specific prior written permission.
15 
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
20 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 
27 }}}*/
28 
29 #ifndef VC_COMMON_SCATTERIMPLEMENTATION_H_
30 #define VC_COMMON_SCATTERIMPLEMENTATION_H_
31 
32 #include "gatherimplementation.h"
33 #include "macros.h"
34 
35 namespace Vc_VERSIONED_NAMESPACE
36 {
37 namespace Common
38 {
39 
40 template <typename V, typename MT, typename IT>
41 Vc_ALWAYS_INLINE void executeScatter(SetIndexZeroT,
42  V &v,
43  MT *mem,
44  IT indexes,
45  typename V::MaskArgument mask)
46 {
47  indexes.setZeroInverted(static_cast<typename IT::Mask>(mask));
48  // Huh?
49  const V tmp(mem, indexes);
50  where(mask) | v = tmp;
51 }
52 
53 template <typename V, typename MT, typename IT>
54 Vc_ALWAYS_INLINE void executeScatter(SimpleLoopT,
55  V &v,
56  MT *mem,
57  const IT &indexes,
58  typename V::MaskArgument mask)
59 {
60  if (Vc_IS_UNLIKELY(mask.isEmpty())) {
61  return;
62  }
63  Common::unrolled_loop<std::size_t, 0, V::Size>([&](std::size_t i) {
64  if (mask[i])
65  mem[indexes[i]] = v[i];
66  });
67 }
68 
69 template <typename V, typename MT, typename IT>
70 Vc_ALWAYS_INLINE void executeScatter(BitScanLoopT,
71  V &v,
72  MT *mem,
73  const IT &indexes,
74  typename V::MaskArgument mask)
75 {
76  size_t bits = mask.toInt();
77  while (Vc_IS_LIKELY(bits > 0)) {
78  size_t i, j;
79  asm("bsf %[bits],%[i]\n\t"
80  "bsr %[bits],%[j]\n\t"
81  "btr %[i],%[bits]\n\t"
82  "btr %[j],%[bits]\n\t"
83  : [i] "=r"(i), [j] "=r"(j), [bits] "+r"(bits));
84  mem[indexes[i]] = v[i];
85  mem[indexes[j]] = v[j];
86  }
87 
88  /* Alternative from Vc::SSE (0.7)
89  int bits = mask.toInt();
90  while (bits) {
91  const int i = _bit_scan_forward(bits);
92  bits ^= (1 << i); // btr?
93  mem[indexes[i]] = v[i];
94  }
95  */
96 }
97 
98 template <typename V, typename MT, typename IT>
99 Vc_ALWAYS_INLINE void executeScatter(PopcntSwitchT,
100  V &v,
101  MT *mem,
102  const IT &indexes,
103  typename V::MaskArgument mask,
104  enable_if<V::size() == 16> = nullarg)
105 {
106  unsigned int bits = mask.toInt();
107  unsigned int low, high = 0;
108  switch (Vc::Detail::popcnt16(bits)) {
109  case 16:
110  v.scatter(mem, indexes);
111  break;
112  case 15:
113  low = _bit_scan_forward(bits);
114  bits ^= 1 << low;
115  mem[indexes[low]] = v[low];
116  case 14:
117  high = _bit_scan_reverse(bits);
118  mem[indexes[high]] = v[high];
119  high = (1 << high);
120  case 13:
121  low = _bit_scan_forward(bits);
122  bits ^= high | (1 << low);
123  mem[indexes[low]] = v[low];
124  case 12:
125  high = _bit_scan_reverse(bits);
126  mem[indexes[high]] = v[high];
127  high = (1 << high);
128  case 11:
129  low = _bit_scan_forward(bits);
130  bits ^= high | (1 << low);
131  mem[indexes[low]] = v[low];
132  case 10:
133  high = _bit_scan_reverse(bits);
134  mem[indexes[high]] = v[high];
135  high = (1 << high);
136  case 9:
137  low = _bit_scan_forward(bits);
138  bits ^= high | (1 << low);
139  mem[indexes[low]] = v[low];
140  case 8:
141  high = _bit_scan_reverse(bits);
142  mem[indexes[high]] = v[high];
143  high = (1 << high);
144  case 7:
145  low = _bit_scan_forward(bits);
146  bits ^= high | (1 << low);
147  mem[indexes[low]] = v[low];
148  case 6:
149  high = _bit_scan_reverse(bits);
150  mem[indexes[high]] = v[high];
151  high = (1 << high);
152  case 5:
153  low = _bit_scan_forward(bits);
154  bits ^= high | (1 << low);
155  mem[indexes[low]] = v[low];
156  case 4:
157  high = _bit_scan_reverse(bits);
158  mem[indexes[high]] = v[high];
159  high = (1 << high);
160  case 3:
161  low = _bit_scan_forward(bits);
162  bits ^= high | (1 << low);
163  mem[indexes[low]] = v[low];
164  case 2:
165  high = _bit_scan_reverse(bits);
166  mem[indexes[high]] = v[high];
167  case 1:
168  low = _bit_scan_forward(bits);
169  mem[indexes[low]] = v[low];
170  case 0:
171  break;
172  }
173 }
174 template <typename V, typename MT, typename IT>
175 Vc_ALWAYS_INLINE void executeScatter(PopcntSwitchT,
176  V &v,
177  MT *mem,
178  const IT &indexes,
179  typename V::MaskArgument mask,
180  enable_if<V::size() == 8> = nullarg)
181 {
182  unsigned int bits = mask.toInt();
183  unsigned int low, high = 0;
184  switch (Vc::Detail::popcnt8(bits)) {
185  case 8:
186  v.scatter(mem, indexes);
187  break;
188  case 7:
189  low = _bit_scan_forward(bits);
190  bits ^= 1 << low;
191  mem[indexes[low]] = v[low];
192  case 6:
193  high = _bit_scan_reverse(bits);
194  mem[indexes[high]] = v[high];
195  high = (1 << high);
196  case 5:
197  low = _bit_scan_forward(bits);
198  bits ^= high | (1 << low);
199  mem[indexes[low]] = v[low];
200  case 4:
201  high = _bit_scan_reverse(bits);
202  mem[indexes[high]] = v[high];
203  high = (1 << high);
204  case 3:
205  low = _bit_scan_forward(bits);
206  bits ^= high | (1 << low);
207  mem[indexes[low]] = v[low];
208  case 2:
209  high = _bit_scan_reverse(bits);
210  mem[indexes[high]] = v[high];
211  case 1:
212  low = _bit_scan_forward(bits);
213  mem[indexes[low]] = v[low];
214  case 0:
215  break;
216  }
217 }
218 template <typename V, typename MT, typename IT>
219 Vc_ALWAYS_INLINE void executeScatter(PopcntSwitchT,
220  V &v,
221  MT *mem,
222  const IT &indexes,
223  typename V::MaskArgument mask,
224  enable_if<V::size() == 4> = nullarg)
225 {
226  unsigned int bits = mask.toInt();
227  unsigned int low, high = 0;
228  switch (Vc::Detail::popcnt4(bits)) {
229  case 4:
230  v.scatter(mem, indexes);
231  break;
232  case 3:
233  low = _bit_scan_forward(bits);
234  bits ^= 1 << low;
235  mem[indexes[low]] = v[low];
236  case 2:
237  high = _bit_scan_reverse(bits);
238  mem[indexes[high]] = v[high];
239  case 1:
240  low = _bit_scan_forward(bits);
241  mem[indexes[low]] = v[low];
242  case 0:
243  break;
244  }
245 }
246 template <typename V, typename MT, typename IT>
247 Vc_ALWAYS_INLINE void executeScatter(PopcntSwitchT,
248  V &v,
249  MT *mem,
250  const IT &indexes,
251  typename V::MaskArgument mask,
252  enable_if<V::size() == 2> = nullarg)
253 {
254  unsigned int bits = mask.toInt();
255  unsigned int low;
256  switch (Vc::Detail::popcnt4(bits)) {
257  case 2:
258  v.scatter(mem, indexes);
259  break;
260  case 1:
261  low = _bit_scan_forward(bits);
262  mem[indexes[low]] = v[low];
263  case 0:
264  break;
265  }
266 }
267 
268 } // namespace Common
269 } // namespace Vc
270 
271 #endif // VC_COMMON_SCATTERIMPLEMENTATION_H_
Vc::where
constexpr WhereImpl::WhereMask< M > where(const M &mask)
Conditional assignment.
Definition: where.h:230