simdize.h

/*  This file is part of the Vc library. {{{
Copyright © 2014-2015 Matthias Kretz <kretz@kde.org>

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the names of contributing organizations nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

}}}*/

#ifndef VC_COMMON_SIMDIZE_H_
#define VC_COMMON_SIMDIZE_H_

#include <tuple>
#include <array>

#include "../Allocator"
#include "interleavedmemory.h"

namespace Vc_VERSIONED_NAMESPACE
{
namespace SimdizeDetail  // {{{
{
using std::is_same;
using std::is_base_of;
using std::false_type;
using std::true_type;
using std::iterator_traits;
using std::conditional;
using std::size_t;
template <bool B, typename T, typename F>
using conditional_t = typename conditional<B, T, F>::type;

template <typename... Ts> struct Typelist;

enum class Category {
    None,
    ArithmeticVectorizable,
    InputIterator,
    OutputIterator,
    ForwardIterator,
    BidirectionalIterator,
    RandomAccessIterator,
    ClassTemplate
};

template <typename T, typename ItCat = typename T::iterator_category>
constexpr Category iteratorCategories(int, ItCat * = nullptr)
{
    return is_base_of<std::random_access_iterator_tag, ItCat>::value
               ? Category::RandomAccessIterator
               : is_base_of<std::bidirectional_iterator_tag, ItCat>::value
                     ? Category::BidirectionalIterator
                     : is_base_of<std::forward_iterator_tag, ItCat>::value
                           ? Category::ForwardIterator
                           : is_base_of<std::output_iterator_tag, ItCat>::value
                                 ? Category::OutputIterator
                                 : is_base_of<std::input_iterator_tag, ItCat>::value
                                       ? Category::InputIterator
                                       : Category::None;
}
template <typename T>
constexpr enable_if<std::is_pointer<T>::value, Category> iteratorCategories(float)
{
    return Category::RandomAccessIterator;
}
template <typename T> constexpr Category iteratorCategories(...)
{
    return Category::None;
}

template <typename T> struct is_class_template : public false_type
{
};
template <template <typename...> class C, typename... Ts>
struct is_class_template<C<Ts...>> : public true_type
{
};

template <typename T> constexpr Category typeCategory()
{
    return (is_same<T, bool>::value || is_same<T, short>::value ||
            is_same<T, unsigned short>::value || is_same<T, int>::value ||
            is_same<T, unsigned int>::value || is_same<T, float>::value ||
            is_same<T, double>::value)
               ? Category::ArithmeticVectorizable
               : iteratorCategories<T>(int()) != Category::None
                     ? iteratorCategories<T>(int())
                     : is_class_template<T>::value ? Category::ClassTemplate
                                                   : Category::None;
}

template <typename T, size_t TupleSize = std::tuple_size<T>::value>
constexpr size_t determine_tuple_size()
{
    return TupleSize;
}
template <typename T, size_t TupleSize = T::tuple_size>
constexpr size_t determine_tuple_size(size_t = T::tuple_size)
{
    return TupleSize;
}

// workaround for MSVC limitation: constexpr functions in template arguments
// confuse the compiler
template <typename T> struct determine_tuple_size_
: public std::integral_constant<size_t, determine_tuple_size<T>()>
{};

namespace
{
template <typename T> struct The_simdization_for_the_requested_type_is_not_implemented;
}  // unnamed namespace

template <typename T, size_t N, typename MT, Category = typeCategory<T>()>
struct ReplaceTypes : public The_simdization_for_the_requested_type_is_not_implemented<T>
{
};

template <typename T, size_t N, typename MT> struct ReplaceTypes<T, N, MT, Category::None>
{
    typedef T type;
};

template <typename T, size_t N = 0, typename MT = void>
using simdize = typename SimdizeDetail::ReplaceTypes<T, N, MT>::type;

// Alias for Vector<T, Abi> with size() == N, or SimdArray<T, N> otherwise.
template <class T, size_t N,
          class Best = typename Common::select_best_vector_type<T, N>::type>
using deduce_vector_t =
    typename std::conditional<Best::size() == N, Best, SimdArray<T, N>>::type;

template <typename T, size_t N, typename MT>
struct ReplaceTypes<T, N, MT, Category::ArithmeticVectorizable>
    : public conditional<N == 0, Vector<T>, deduce_vector_t<T, N>> {
};

template <size_t N, typename MT>
struct ReplaceTypes<bool, N, MT, Category::ArithmeticVectorizable>
    : public std::enable_if<true, typename ReplaceTypes<MT, N, MT>::type::mask_type> {
};
template <size_t N>
struct ReplaceTypes<bool, N, void, Category::ArithmeticVectorizable>
    : public ReplaceTypes<bool, N, float, Category::ArithmeticVectorizable>
{
};

template <size_t N, typename MT, typename Replaced, typename... Remaining>
struct SubstituteOneByOne;

template <size_t N, typename MT, typename... Replaced, typename T,
          typename... Remaining>
struct SubstituteOneByOne<N, MT, Typelist<Replaced...>, T, Remaining...>
{
private:
    template <typename U, size_t M = U::Size>
    static std::integral_constant<size_t, M> size_or_0(int);
    template <typename U> static std::integral_constant<size_t, 0> size_or_0(...);

    using V = simdize<T, N, MT>;

    static constexpr auto NewN = N != 0 ? N : decltype(size_or_0<V>(int()))::value;

    typedef conditional_t<(N != NewN && is_same<MT, void>::value),
                          conditional_t<is_same<T, bool>::value, float, T>, MT> NewMT;

public:
    using type = typename SubstituteOneByOne<NewN, NewMT, Typelist<Replaced..., V>,
                                             Remaining...>::type;
};

template <size_t Size, typename... Replaced> struct SubstitutedBase;
template <typename Replaced> struct SubstitutedBase<1, Replaced> {
    template <typename ValueT, template <typename, ValueT...> class C, ValueT... Values>
    using SubstitutedWithValues = C<Replaced, Values...>;
};
template <typename R0, typename R1> struct SubstitutedBase<2, R0, R1>
{
    template <typename ValueT, template <typename, typename, ValueT...> class C,
              ValueT... Values>
    using SubstitutedWithValues = C<R0, R1, Values...>;
};
template <typename R0, typename R1, typename R2> struct SubstitutedBase<3, R0, R1, R2>
{
    template <typename ValueT, template <typename, typename, typename, ValueT...> class C,
              ValueT... Values>
    using SubstitutedWithValues = C<R0, R1, R2, Values...>;
};
#if defined Vc_ICC || defined Vc_MSVC
#define Vc_VALUE_PACK_EXPANSION_IS_BROKEN 1
#endif
template <typename... Replaced> struct SubstitutedBase<4, Replaced...> {
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
    template <typename ValueT,
              template <typename, typename, typename, typename, ValueT...> class C,
              ValueT... Values>
    using SubstitutedWithValues = C<Replaced..., Values...>;
#endif // Vc_VALUE_PACK_EXPANSION_IS_BROKEN
};
template <typename... Replaced> struct SubstitutedBase<5, Replaced...> {
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
    template <typename ValueT, template <typename, typename, typename, typename, typename,
                                         ValueT...> class C,
              ValueT... Values>
    using SubstitutedWithValues = C<Replaced..., Values...>;
#endif // Vc_VALUE_PACK_EXPANSION_IS_BROKEN
};
template <typename... Replaced> struct SubstitutedBase<6, Replaced...> {
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
    template <typename ValueT, template <typename, typename, typename, typename, typename,
                                         typename, ValueT...> class C,
              ValueT... Values>
    using SubstitutedWithValues = C<Replaced..., Values...>;
#endif // Vc_VALUE_PACK_EXPANSION_IS_BROKEN
};
template <typename... Replaced> struct SubstitutedBase<7, Replaced...> {
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
    template <typename ValueT, template <typename, typename, typename, typename, typename,
                                         typename, typename, ValueT...> class C,
              ValueT... Values>
    using SubstitutedWithValues = C<Replaced..., Values...>;
#endif // Vc_VALUE_PACK_EXPANSION_IS_BROKEN
};
template <typename... Replaced> struct SubstitutedBase<8, Replaced...> {
#ifndef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
    template <typename ValueT, template <typename, typename, typename, typename, typename,
                                         typename, typename, typename, ValueT...> class C,
              ValueT... Values>
    using SubstitutedWithValues = C<Replaced..., Values...>;
#endif // Vc_VALUE_PACK_EXPANSION_IS_BROKEN
};

template <size_t N_, typename MT, typename Replaced0, typename... Replaced>
struct SubstituteOneByOne<N_, MT, Typelist<Replaced0, Replaced...>>
{
    struct type
        : public SubstitutedBase<sizeof...(Replaced) + 1, Replaced0, Replaced...> {
        static constexpr auto N = N_;
        template <template <typename...> class C>
        using Substituted = C<Replaced0, Replaced...>;
    };
};

template <typename Scalar, typename Base, size_t N> class Adapter;

template <template <typename...> class C, typename... Ts, size_t N, typename MT>
struct ReplaceTypes<C<Ts...>, N, MT, Category::ClassTemplate>
{
    using SubstitutionResult =
        typename SubstituteOneByOne<N, MT, Typelist<>, Ts...>::type;
    using Vectorized = typename SubstitutionResult::template Substituted<C>;
    using type = conditional_t<is_same<C<Ts...>, Vectorized>::value, C<Ts...>,
                               Adapter<C<Ts...>, Vectorized, SubstitutionResult::N>>;
};

#ifdef Vc_VALUE_PACK_EXPANSION_IS_BROKEN
// ICC barfs on packs of values
#define Vc_DEFINE_NONTYPE_REPLACETYPES_(ValueType_)                                      \
    template <template <typename, ValueType_...> class C, typename T, ValueType_ Value0, \
              ValueType_... Values>                                                      \
    struct is_class_template<C<T, Value0, Values...>> : public true_type {               \
    };                                                                                   \
    template <template <typename, typename, ValueType_...> class C, typename T0,         \
              typename T1, ValueType_ Value0, ValueType_... Values>                      \
    struct is_class_template<C<T0, T1, Value0, Values...>> : public true_type {          \
    };                                                                                   \
    template <template <typename, typename, typename, ValueType_...> class C,            \
              typename T0, typename T1, typename T2, ValueType_ Value0,                  \
              ValueType_... Values>                                                      \
    struct is_class_template<C<T0, T1, T2, Value0, Values...>> : public true_type {      \
    };                                                                                   \
    template <template <typename, typename, typename, typename, ValueType_...> class C,  \
              typename T0, typename T1, typename T2, typename T3, ValueType_ Value0,     \
              ValueType_... Values>                                                      \
    struct is_class_template<C<T0, T1, T2, T3, Value0, Values...>> : public true_type {  \
    };                                                                                   \
    template <template <typename, typename, typename, typename, typename, ValueType_...> \
              class C,                                                                   \
              typename T0, typename T1, typename T2, typename T3, typename T4,           \
              ValueType_ Value0, ValueType_... Values>                                   \
    struct is_class_template<C<T0, T1, T2, T3, T4, Value0, Values...>>                   \
        : public true_type {                                                             \
    };                                                                                   \
    template <template <typename, typename, typename, typename, typename, typename,      \
                        ValueType_...> class C,                                          \
              typename T0, typename T1, typename T2, typename T3, typename T4,           \
              typename T5, ValueType_ Value0, ValueType_... Values>                      \
    struct is_class_template<C<T0, T1, T2, T3, T4, T5, Value0, Values...>>               \
        : public true_type {                                                             \
    };                                                                                   \
    template <template <typename, typename, typename, typename, typename, typename,      \
                        typename, ValueType_...> class C,                                \
              typename T0, typename T1, typename T2, typename T3, typename T4,           \
              typename T5, typename T6, ValueType_ Value0, ValueType_... Values>         \
    struct is_class_template<C<T0, T1, T2, T3, T4, T5, T6, Value0, Values...>>           \
        : public true_type {                                                             \
    };                                                                                   \
    template <template <typename, ValueType_> class C, typename T0, ValueType_ Value0,   \
              size_t N, typename MT>                                                     \
    struct ReplaceTypes<C<T0, Value0>, N, MT, Category::ClassTemplate> {                 \
        typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0>::type tmp;            \
        typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0>      \
            Substituted;                                                                 \
        static constexpr auto NN = tmp::N;                                               \
        typedef conditional_t<is_same<C<T0, Value0>, Substituted>::value, C<T0, Value0>, \
                              Adapter<C<T0, Value0>, Substituted, NN>> type;             \
    };                                                                                   \
    template <template <typename, typename, ValueType_> class C, typename T0,            \
              typename T1, ValueType_ Value0, size_t N, typename MT>                     \
    struct ReplaceTypes<C<T0, T1, Value0>, N, MT, Category::ClassTemplate> {             \
        typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0, T1>::type tmp;        \
        typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0>      \
            Substituted;                                                                 \
        static constexpr auto NN = tmp::N;                                               \
        typedef conditional_t<is_same<C<T0, T1, Value0>, Substituted>::value,            \
                              C<T0, T1, Value0>,                                         \
                              Adapter<C<T0, T1, Value0>, Substituted, NN>> type;         \
    };                                                                                   \
    template <template <typename, typename, typename, ValueType_> class C, typename T0,  \
              typename T1, typename T2, ValueType_ Value0, size_t N, typename MT>        \
    struct ReplaceTypes<C<T0, T1, T2, Value0>, N, MT, Category::ClassTemplate> {         \
        typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0, T1, T2>::type tmp;    \
        typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0>      \
            Substituted;                                                                 \
        static constexpr auto NN = tmp::N;                                               \
        typedef conditional_t<is_same<C<T0, T1, T2, Value0>, Substituted>::value,        \
                              C<T0, T1, T2, Value0>,                                     \
                              Adapter<C<T0, T1, T2, Value0>, Substituted, NN>> type;     \
    }
#else
#define Vc_DEFINE_NONTYPE_REPLACETYPES_(ValueType_)                                      \
    template <template <typename, ValueType_...> class C, typename T, ValueType_ Value0, \
              ValueType_... Values>                                                      \
    struct is_class_template<C<T, Value0, Values...>> : public true_type {               \
    };                                                                                   \
    template <template <typename, typename, ValueType_...> class C, typename T0,         \
              typename T1, ValueType_ Value0, ValueType_... Values>                      \
    struct is_class_template<C<T0, T1, Value0, Values...>> : public true_type {          \
    };                                                                                   \
    template <template <typename, typename, typename, ValueType_...> class C,            \
              typename T0, typename T1, typename T2, ValueType_ Value0,                  \
              ValueType_... Values>                                                      \
    struct is_class_template<C<T0, T1, T2, Value0, Values...>> : public true_type {      \
    };                                                                                   \
    template <template <typename, typename, typename, typename, ValueType_...> class C,  \
              typename T0, typename T1, typename T2, typename T3, ValueType_ Value0,     \
              ValueType_... Values>                                                      \
    struct is_class_template<C<T0, T1, T2, T3, Value0, Values...>> : public true_type {  \
    };                                                                                   \
    template <template <typename, typename, typename, typename, typename, ValueType_...> \
              class C,                                                                   \
              typename T0, typename T1, typename T2, typename T3, typename T4,           \
              ValueType_ Value0, ValueType_... Values>                                   \
    struct is_class_template<C<T0, T1, T2, T3, T4, Value0, Values...>>                   \
        : public true_type {                                                             \
    };                                                                                   \
    template <template <typename, typename, typename, typename, typename, typename,      \
                        ValueType_...> class C,                                          \
              typename T0, typename T1, typename T2, typename T3, typename T4,           \
              typename T5, ValueType_ Value0, ValueType_... Values>                      \
    struct is_class_template<C<T0, T1, T2, T3, T4, T5, Value0, Values...>>               \
        : public true_type {                                                             \
    };                                                                                   \
    template <template <typename, typename, typename, typename, typename, typename,      \
                        typename, ValueType_...> class C,                                \
              typename T0, typename T1, typename T2, typename T3, typename T4,           \
              typename T5, typename T6, ValueType_ Value0, ValueType_... Values>         \
    struct is_class_template<C<T0, T1, T2, T3, T4, T5, T6, Value0, Values...>>           \
        : public true_type {                                                             \
    };                                                                                   \
    template <template <typename, ValueType_...> class C, typename T0,                   \
              ValueType_ Value0, ValueType_... Values, size_t N, typename MT>            \
    struct ReplaceTypes<C<T0, Value0, Values...>, N, MT, Category::ClassTemplate> {      \
        typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0>::type tmp;            \
        typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0,      \
                                                             Values...> Substituted;     \
        static constexpr auto NN = tmp::N;                                               \
        typedef conditional_t<is_same<C<T0, Value0, Values...>, Substituted>::value,     \
                              C<T0, Value0, Values...>,                                  \
                              Adapter<C<T0, Value0, Values...>, Substituted, NN>> type;  \
    };                                                                                   \
    template <template <typename, typename, ValueType_...> class C, typename T0,         \
              typename T1, ValueType_ Value0, ValueType_... Values, size_t N,            \
              typename MT>                                                               \
    struct ReplaceTypes<C<T0, T1, Value0, Values...>, N, MT, Category::ClassTemplate> {  \
        typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0, T1>::type tmp;        \
        typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0,      \
                                                             Values...> Substituted;     \
        static constexpr auto NN = tmp::N;                                               \
        typedef conditional_t<is_same<C<T0, T1, Value0, Values...>, Substituted>::value, \
                              C<T0, T1, Value0, Values...>,                              \
                              Adapter<C<T0, T1, Value0, Values...>, Substituted, NN>>    \
            type;                                                                        \
    };                                                                                   \
    template <template <typename, typename, typename, ValueType_...> class C,            \
              typename T0, typename T1, typename T2, ValueType_ Value0,                  \
              ValueType_... Values, size_t N, typename MT>                               \
    struct ReplaceTypes<C<T0, T1, T2, Value0, Values...>, N, MT,                         \
                        Category::ClassTemplate> {                                       \
        typedef typename SubstituteOneByOne<N, MT, Typelist<>, T0, T1, T2>::type tmp;    \
        typedef typename tmp::template SubstitutedWithValues<ValueType_, C, Value0,      \
                                                             Values...> Substituted;     \
        static constexpr auto NN = tmp::N;                                               \
        typedef conditional_t<                                                           \
            is_same<C<T0, T1, T2, Value0, Values...>, Substituted>::value,               \
            C<T0, T1, T2, Value0, Values...>,                                            \
            Adapter<C<T0, T1, T2, Value0, Values...>, Substituted, NN>> type;            \
    }
#endif  // Vc_VALUE_PACK_EXPANSION_IS_BROKEN
Vc_DEFINE_NONTYPE_REPLACETYPES_(bool);
Vc_DEFINE_NONTYPE_REPLACETYPES_(wchar_t);
Vc_DEFINE_NONTYPE_REPLACETYPES_(char);
Vc_DEFINE_NONTYPE_REPLACETYPES_(  signed char);
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned char);
Vc_DEFINE_NONTYPE_REPLACETYPES_(  signed short);
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned short);
Vc_DEFINE_NONTYPE_REPLACETYPES_(  signed int);
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned int);
Vc_DEFINE_NONTYPE_REPLACETYPES_(  signed long);
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned long);
Vc_DEFINE_NONTYPE_REPLACETYPES_(  signed long long);
Vc_DEFINE_NONTYPE_REPLACETYPES_(unsigned long long);
#undef Vc_DEFINE_NONTYPE_REPLACETYPES_

// preferred_construction {{{
namespace preferred_construction_impl
{
template <typename T> T create();
// 0: paren init
template <class Type, class... Init, class = decltype(Type(create<Init>()...))>
constexpr std::integral_constant<int, 0> test(int);
// 1: 1-brace init
template <class Type, class... Init, class = decltype(Type{create<Init>()...})>
constexpr std::integral_constant<int, 1> test(float);
// 2: 2-brace init
template <class Type, class... Init, class T, class = decltype(Type{{create<Init>()...}})>
constexpr std::integral_constant<int, 2> test(T);
// 3: no init at all
template <class Type, class... Init> constexpr std::integral_constant<int, 3> test(...);
}  // namespace preferred_construction_impl

template <class Type, class... Init>
constexpr inline decltype(preferred_construction_impl::test<Type, Init...>(0))
preferred_construction()
{
    return {};
}

// }}}
// get_dispatcher {{{
template <size_t I, typename T,
          typename R = decltype(std::declval<T &>().template vc_get_<I>())>
R get_dispatcher(T &x, void * = nullptr)
{
    return x.template vc_get_<I>();
}
template <size_t I, typename T,
          typename R = decltype(std::declval<const T &>().template vc_get_<I>())>
R get_dispatcher(const T &x, void * = nullptr)
{
    return x.template vc_get_<I>();
}
template <size_t I, typename T, typename R = decltype(std::get<I>(std::declval<T &>()))>
R get_dispatcher(T &x, int = 0)
{
    return std::get<I>(x);
}
template <size_t I, typename T,
          typename R = decltype(std::get<I>(std::declval<const T &>()))>
R get_dispatcher(const T &x, int = 0)
{
    return std::get<I>(x);
}

// }}}
// homogeneous_sizeof {{{
template <class... Ts> struct homogeneous_sizeof;
template <class T, class = void> struct homogeneous_sizeof_one;

template <class T, size_t... Is>
std::integral_constant<size_t,
                       homogeneous_sizeof<typename std::remove_reference<decltype(
                           get_dispatcher<Is>(std::declval<T>()))>::type...>::value>
homogeneous_sizeof_helper(index_sequence<Is...>);

template <class T>
struct homogeneous_sizeof_one<T,
                              typename std::enable_if<std::is_arithmetic<T>::value>::type>
    : std::integral_constant<size_t, sizeof(T)> {
};

template <class T>
struct homogeneous_sizeof_one<T, typename std::enable_if<std::is_class<T>::value>::type>
    : decltype(homogeneous_sizeof_helper<T>(
          make_index_sequence<determine_tuple_size_<T>::value>())) {
};

template <class T0> struct homogeneous_sizeof<T0> : homogeneous_sizeof_one<T0> {
};

template <class T0, class... Ts>
struct homogeneous_sizeof<T0, Ts...>
    : std::integral_constant<size_t, homogeneous_sizeof<T0>::value ==
                                             homogeneous_sizeof<Ts...>::value
                                         ? homogeneous_sizeof<T0>::value
                                         : 0> {
};
// }}}
// class Adapter {{{
template <typename Scalar, typename Base, size_t N> class Adapter : public Base
{
private:
    template <std::size_t... Indexes, int X>
    Adapter(Vc::index_sequence<Indexes...>, const Scalar,
            std::integral_constant<int, X>)
    {
        static_assert(
            X < 3, "Failed to construct an object of type Base. Neither via "
                   "parenthesis-init, brace-init, nor double-brace init appear to work.");
    }

    template <std::size_t... Indexes>
    Adapter(Vc::index_sequence<Indexes...>, const Scalar &x_,
            std::integral_constant<int, 2>)
        : Base{{get_dispatcher<Indexes>(x_)...}}
    {
    }

    template <std::size_t... Indexes>
    Adapter(Vc::index_sequence<Indexes...>, const Scalar &x_,
            std::integral_constant<int, 1>)
        : Base{get_dispatcher<Indexes>(x_)...}
    {
    }

    template <std::size_t... Indexes>
    Adapter(Vc::index_sequence<Indexes...>, const Scalar &x_,
            std::integral_constant<int, 0>)
        : Base(get_dispatcher<Indexes>(x_)...)
    {
    }

    template <std::size_t... Indexes>
    Adapter(Vc::index_sequence<Indexes...> seq_, const Scalar &x_)
        : Adapter(seq_, x_,
                  preferred_construction<Base, decltype(get_dispatcher<Indexes>(
                                                   std::declval<const Scalar &>()))...>())
    {
    }

public:
    static constexpr size_t size() { return N; }
    static constexpr size_t Size = N;

    using base_type = Base;
    using scalar_type = Scalar;

    Adapter() = default;

#if defined Vc_CLANG && Vc_CLANG < 0x30700
    Vc_INTRINSIC Adapter(const Adapter &x) : Base(x) {}
#else
    Adapter(const Adapter &) = default;
#endif
    Adapter(Adapter &&) = default;
    Adapter &operator=(const Adapter &) = default;
    Adapter &operator=(Adapter &&) = default;

    template <typename U, size_t TupleSize = determine_tuple_size_<Scalar>::value,
              typename Seq = Vc::make_index_sequence<TupleSize>,
              typename = enable_if<std::is_convertible<U, Scalar>::value>>
    Adapter(U &&x_)
        : Adapter(Seq(), static_cast<const Scalar &>(x_))
    {
    }

    template <class F,
              class = decltype(static_cast<Scalar>(std::declval<F>()(
                  size_t())))>  // F returns objects that are convertible to S
    Adapter(F &&fun)
    {
        for (size_t i = 0; i < N; ++i) {
            assign(*this, i, fun(i));
        }
    }

    // }}}
    template <typename A0, typename... Args,
              typename = typename std::enable_if<
                  !Traits::is_index_sequence<A0>::value &&
                  (sizeof...(Args) > 0 || !std::is_convertible<A0, Scalar>::value)>::type>
    Adapter(A0 &&arg0_, Args &&... arguments_)
        : Base(std::forward<A0>(arg0_), std::forward<Args>(arguments_)...)
    {
    }

    template <typename T,
              typename = decltype(Base(std::declval<const std::initializer_list<T> &>()))>
    Adapter(const std::initializer_list<T> &l_)
        : Base(l_)
    {
    }

    void *operator new(size_t size)
    {
        return Vc::Common::aligned_malloc<alignof(Adapter)>(size);
    }
    void *operator new(size_t, void *p_) { return p_; }
    void *operator new[](size_t size)
    {
        return Vc::Common::aligned_malloc<alignof(Adapter)>(size);
    }
    void *operator new[](size_t , void *p_) { return p_; }
    void operator delete(void *ptr_, size_t) { Vc::Common::free(ptr_); }
    void operator delete(void *, void *) {}
    void operator delete[](void *ptr_, size_t) { Vc::Common::free(ptr_); }
    void operator delete[](void *, void *) {}
};  // }}}
// delete compare operators for Adapter {{{
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
inline void operator==(
    const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
    const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
inline void operator!=(
    const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
    const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
inline void operator<=(
    const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
    const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
inline void operator>=(
    const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
    const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
inline void operator<(
    const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
    const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
template <class... TTypes, class... TTypesV, class... UTypes, class... UTypesV, size_t N>
inline void operator>(
    const Adapter<std::tuple<TTypes...>, std::tuple<TTypesV...>, N> &t,
    const Adapter<std::tuple<UTypes...>, std::tuple<UTypesV...>, N> &u) = delete;
// }}}
}  // namespace SimdizeDetail }}}
}  // namespace Vc

namespace std  // {{{
{
template <typename Scalar, typename Base, size_t N>
class tuple_size<Vc::SimdizeDetail::Adapter<Scalar, Base, N>> : public tuple_size<Base>
{
};
template <size_t I, typename Scalar, typename Base, size_t N>
class tuple_element<I, Vc::SimdizeDetail::Adapter<Scalar, Base, N>>
    : public tuple_element<I, Base>
{
};
// std::get does not need additional work because Vc::Adapter derives from
// C<Ts...> and therefore if get<N>(C<Ts...>) works it works for Adapter as well.

template <typename S, typename T, size_t N>
class allocator<Vc::SimdizeDetail::Adapter<S, T, N>>
    : public Vc::Allocator<Vc::SimdizeDetail::Adapter<S, T, N>>
{
public:
    template <typename U> struct rebind
    {
        typedef std::allocator<U> other;
    };
};
}  // namespace std }}}

namespace Vc_VERSIONED_NAMESPACE
{
namespace SimdizeDetail
{
template <typename T> static inline T decay_workaround(const T &x) { return x; }

// assign_impl {{{
template <typename S, typename T, size_t N, size_t... Indexes>
inline void assign_impl(Adapter<S, T, N> &a, size_t i, const S &x,
                        Vc::index_sequence<Indexes...>)
{
    const std::tuple<decltype(decay_workaround(get_dispatcher<Indexes>(x)))...> tmp(
        decay_workaround(get_dispatcher<Indexes>(x))...);
    auto &&unused = {(get_dispatcher<Indexes>(a)[i] = get_dispatcher<Indexes>(tmp), 0)...};
    if (&unused == &unused) {}
}  // }}}
// construct (parens, braces, double-braces) {{{
template <class S, class... Args>
S construct(std::integral_constant<int, 0>, Args &&... args)
{
    return S(std::forward<Args>(args)...);
}
template <class S, class... Args>
S construct(std::integral_constant<int, 1>, Args &&... args)
{
    return S{std::forward<Args>(args)...};
}
template <class S, class... Args>
S construct(std::integral_constant<int, 2>, Args &&... args)
{
    return S{{std::forward<Args>(args)...}};
}
// }}}
// extract_impl {{{
template <typename S, typename T, size_t N, size_t... Indexes>
inline S extract_impl(const Adapter<S, T, N> &a, size_t i, Vc::index_sequence<Indexes...>)
{
    const std::tuple<decltype(decay_workaround(get_dispatcher<Indexes>(a)[i]))...> tmp(
        decay_workaround(get_dispatcher<Indexes>(a)[i])...);
    return construct<S>(
        preferred_construction<S, decltype(decay_workaround(
                                      get_dispatcher<Indexes>(a)[i]))...>(),
        decay_workaround(get_dispatcher<Indexes>(a)[i])...);
    //return S(get_dispatcher<Indexes>(tmp)...);
}
// }}}
// shifted_impl {{{
template <typename S, typename T, std::size_t N, std::size_t... Indexes>
inline Adapter<S, T, N> shifted_impl(const Adapter<S, T, N> &a, int shift,
                                     Vc::index_sequence<Indexes...>)
{
    Adapter<S, T, N> r;
    auto &&unused = {(get_dispatcher<Indexes>(r) = get_dispatcher<Indexes>(a).shifted(shift), 0)...};
    if (&unused == &unused) {}
    return r;
}
// }}}
// shifted(Adapter) {{{
template <typename S, typename T, size_t N>
inline Adapter<S, T, N> shifted(const Adapter<S, T, N> &a, int shift)
{
    return shifted_impl(a, shift, Vc::make_index_sequence<determine_tuple_size<T>()>());
}
// }}}
// swap_impl {{{
template <typename S, typename T, std::size_t N, std::size_t... Indexes>
inline void swap_impl(Adapter<S, T, N> &a, std::size_t i, S &x,
                      Vc::index_sequence<Indexes...>)
{
    const auto &a_const = a;
    const std::tuple<decltype(decay_workaround(get_dispatcher<Indexes>(a_const)[0]))...>
        tmp{decay_workaround(get_dispatcher<Indexes>(a_const)[i])...};
    auto &&unused = {(get_dispatcher<Indexes>(a)[i] = get_dispatcher<Indexes>(x), 0)...};
    auto &&unused2 = {(get_dispatcher<Indexes>(x) = get_dispatcher<Indexes>(tmp), 0)...};
    if (&unused == &unused2) {}
}
template <typename S, typename T, std::size_t N, std::size_t... Indexes>
inline void swap_impl(Adapter<S, T, N> &a, std::size_t i, Adapter<S, T, N> &b,
                      std::size_t j, Vc::index_sequence<Indexes...>)
{
    const auto &a_const = a;
    const auto &b_const = b;
    const std::tuple<decltype(decay_workaround(get_dispatcher<Indexes>(a_const)[0]))...>
        tmp{decay_workaround(get_dispatcher<Indexes>(a_const)[i])...};
    auto &&unused = {(get_dispatcher<Indexes>(a)[i] = get_dispatcher<Indexes>(b_const)[j], 0)...};
    auto &&unused2 = {(get_dispatcher<Indexes>(b)[j] = get_dispatcher<Indexes>(tmp), 0)...};
    if (&unused == &unused2) {}
}
// }}}
// swap(Adapter) {{{
template <typename S, typename T, std::size_t N>
inline void swap(Adapter<S, T, N> &a, std::size_t i, S &x)
{
    swap_impl(a, i, x, Vc::make_index_sequence<determine_tuple_size<T>()>());
}
template <typename S, typename T, std::size_t N>
inline void swap(Adapter<S, T, N> &a, std::size_t i, Adapter<S, T, N> &b, std::size_t j)
{
    swap_impl(a, i, b, j, Vc::make_index_sequence<determine_tuple_size<T>()>());
}
// }}}
template <typename A> class Scalar  // {{{
{
    using reference = typename std::add_lvalue_reference<A>::type;
    using S = typename A::scalar_type;
    using IndexSeq = Vc::make_index_sequence<determine_tuple_size<S>()>;

public:
    Scalar(reference aa, size_t ii) : a(aa), i(ii) {}

    // delete copy and move to keep the type a pure proxy temporary object.
    Scalar(const Scalar &) = delete;
    Scalar(Scalar &&) = delete;
    Scalar &operator=(const Scalar &) = delete;
    Scalar &operator=(Scalar &&) = delete;

    void operator=(const S &x) { assign_impl(a, i, x, IndexSeq()); }
    operator S() const { return extract_impl(a, i, IndexSeq()); }

    template <typename AA>
    friend inline void swap(Scalar<AA> &&a, typename AA::scalar_type &b);
    template <typename AA>
    friend inline void swap(typename AA::scalar_type &b, Scalar<AA> &&a);
    template <typename AA> friend inline void swap(Scalar<AA> &&a, Scalar<AA> &&b);

private:
    reference a;
    size_t i;
};  // }}}
// swap(Scalar) {{{
template <typename A> inline void swap(Scalar<A> &&a, typename A::scalar_type &b)
{
    swap_impl(a.a, a.i, b, typename Scalar<A>::IndexSeq());
}
template <typename A> inline void swap(typename A::scalar_type &b, Scalar<A> &&a)
{
    swap_impl(a.a, a.i, b, typename Scalar<A>::IndexSeq());
}

template <typename A> inline void swap(Scalar<A> &&a, Scalar<A> &&b)
{
    swap_impl(a.a, a.i, b.a, b.i, typename Scalar<A>::IndexSeq());
}
// }}}
// load_interleaved_impl {{{
template <class S, class T, size_t N, size_t... I>
inline void load_interleaved_impl(Vc::index_sequence<I...>, Adapter<S, T, N> &a,
                                  const S *mem)
{
    const InterleavedMemoryWrapper<S, decltype(decay_workaround(get_dispatcher<0>(a)))>
    wrapper(const_cast<S *>(mem));
    Vc::tie(get_dispatcher<I>(a)...) = wrapper[0];
}
// }}}
// store_interleaved_impl {{{
template <class S, class T, size_t N, size_t... I>
inline void store_interleaved_impl(Vc::index_sequence<I...>, const Adapter<S, T, N> &a,
                                   S *mem)
{
    InterleavedMemoryWrapper<S, decltype(decay_workaround(get_dispatcher<0>(a)))> wrapper(
        mem);
    wrapper[0] = Vc::tie(get_dispatcher<I>(a)...);
}
// }}}
template <typename A> class Interface  // {{{
{
    using reference = typename std::add_lvalue_reference<A>::type;
    using IndexSeq =
        Vc::make_index_sequence<determine_tuple_size<typename A::scalar_type>()>;

public:
    Interface(reference aa) : a(aa) {}

    Scalar<A> operator[](size_t i)
    {
        return {a, i};
    }
    typename A::scalar_type operator[](size_t i) const
    {
        return extract_impl(a, i, IndexSeq());
    }

    A shifted(int amount) const
    {
        return shifted_impl(a, amount, IndexSeq());
    }

    void load(const typename A::scalar_type *mem) { load_interleaved(*this, mem); }
    void store(typename A::scalar_type *mem) { store_interleaved(*this, mem); }

private:
    reference a;
};  // }}}
}  // namespace SimdizeDetail
// assign {{{
template <typename S, typename T, size_t N>
inline void assign(SimdizeDetail::Adapter<S, T, N> &a, size_t i, const S &x)
{
    SimdizeDetail::assign_impl(
        a, i, x, Vc::make_index_sequence<SimdizeDetail::determine_tuple_size<T>()>());
}
template <typename V, typename = enable_if<Traits::is_simd_vector<V>::value>>
Vc_INTRINSIC void assign(V &v, size_t i, typename V::EntryType x)
{
    v[i] = x;
}
// }}}
// extract {{{
template <typename S, typename T, size_t N>
inline S extract(const SimdizeDetail::Adapter<S, T, N> &a, size_t i)
{
    return SimdizeDetail::extract_impl(
        a, i, Vc::make_index_sequence<SimdizeDetail::determine_tuple_size<S>()>());
}
template <typename V, typename = enable_if<Traits::is_simd_vector<V>::value>>
Vc_INTRINSIC typename V::EntryType extract(const V &v, size_t i)
{
    return v[i];
}
// }}}
// load_interleaved {{{
template <class S, class T, size_t N>
inline void load_interleaved(SimdizeDetail::Adapter<S, T, N> &a, const S *mem)
{
    if (SimdizeDetail::homogeneous_sizeof<S>::value == 0) {
        Common::unrolled_loop<std::size_t, 0, N>(
            [&](std::size_t i) { assign(a, i, mem[i]); });
    } else {
        constexpr size_t TupleSize = SimdizeDetail::determine_tuple_size_<S>::value;
        SimdizeDetail::load_interleaved_impl(Vc::make_index_sequence<TupleSize>(), a,
                                             mem);
    }
}
template <
    class V, class T,
    class = enable_if<Traits::is_simd_vector<V>::value && std::is_arithmetic<T>::value>>
Vc_INTRINSIC void load_interleaved(V &a, const T *mem)
{
    a.load(mem, Vc::Unaligned);
}
// }}}
// store_interleaved {{{
template <class S, class T, size_t N>
inline void store_interleaved(const SimdizeDetail::Adapter<S, T, N> &a, S *mem)
{
    if (SimdizeDetail::homogeneous_sizeof<S>::value == 0) {
        Common::unrolled_loop<std::size_t, 0, N>(
            [&](std::size_t i) { mem[i] = extract(a, i); });
    } else {
        constexpr size_t TupleSize = SimdizeDetail::determine_tuple_size_<S>::value;
        SimdizeDetail::store_interleaved_impl(Vc::make_index_sequence<TupleSize>(), a,
                                              mem);
    }
}
template <
    class V, class T,
    class = enable_if<Traits::is_simd_vector<V>::value && std::is_arithmetic<T>::value>>
Vc_INTRINSIC void store_interleaved(const V &a, T *mem)
{
    a.store(mem, Vc::Unaligned);
}
// }}}
// decorate(Adapter) {{{
template <typename S, typename T, size_t N>
SimdizeDetail::Interface<SimdizeDetail::Adapter<S, T, N>> decorate(
    SimdizeDetail::Adapter<S, T, N> &a)
{
    return {a};
}
template <typename S, typename T, size_t N>
const SimdizeDetail::Interface<const SimdizeDetail::Adapter<S, T, N>> decorate(
    const SimdizeDetail::Adapter<S, T, N> &a)
{
    return {a};
}
template <class V, class = typename std::enable_if<
                       Traits::is_simd_vector<typename std::decay<V>::type>::value>>
V &&decorate(V &&v)
{
    return std::forward<V>(v);
}
// }}}
namespace SimdizeDetail
{
namespace IteratorDetails  // {{{
{
enum class Mutable { Yes, No };

template <typename It, typename V, size_t I, size_t End>
Vc_INTRINSIC V fromIteratorImpl(enable_if<(I == End), It>)
{
    return {};
}
template <typename It, typename V, size_t I, size_t End>
Vc_INTRINSIC V fromIteratorImpl(enable_if<(I < End), It> it)
{
    V r = fromIteratorImpl<It, V, I + 1, End>(it);
    Traits::decay<decltype(get_dispatcher<I>(r))> tmp;
    for (size_t j = 0; j < V::size(); ++j, ++it) {
        tmp[j] = get_dispatcher<I>(*it);
    }
    get_dispatcher<I>(r) = tmp;
    return r;
}
template <typename It, typename V>
Vc_INTRINSIC V fromIterator(enable_if<!Traits::is_simd_vector<V>::value, const It &> it)
{
    return fromIteratorImpl<It, V, 0, determine_tuple_size<V>()>(it);
}
template <typename It, typename V>
Vc_INTRINSIC V fromIterator(enable_if<Traits::is_simd_vector<V>::value, It> it)
{
    V r;
    for (size_t j = 0; j < V::size(); ++j, ++it) {
        r[j] = *it;
    }
    return r;
}

// Note: §13.5.6 says: “An expression x->m is interpreted as (x.operator->())->m for a
// class object x of type T if T::operator->() exists and if the operator is selected as
// the best match function by the overload resolution mechanism (13.3).”
template <typename T, typename value_vector, Mutable> class Pointer;

template <typename T, typename value_vector> class Pointer<T, value_vector, Mutable::Yes>
{
    static constexpr auto Size = value_vector::size();

public:
    value_vector *operator->() { return &data; }

    Pointer() = delete;
    Pointer(const Pointer &) = delete;
    Pointer &operator=(const Pointer &) = delete;
    Pointer &operator=(Pointer &&) = delete;

    Pointer(Pointer &&) = default;

    ~Pointer()
    {
        // store data back to where it came from
        for (size_t i = 0; i < Size; ++i, ++begin_iterator) {
            *begin_iterator = extract(data, i);
        }
    }

    Pointer(const T &it) : data(fromIterator<T, value_vector>(it)), begin_iterator(it) {}

private:
    value_vector data;
    T begin_iterator;
};
template <typename T, typename value_vector> class Pointer<T, value_vector, Mutable::No>
{
    static constexpr auto Size = value_vector::size();

public:
    const value_vector *operator->() const { return &data; }

    Pointer() = delete;
    Pointer(const Pointer &) = delete;
    Pointer &operator=(const Pointer &) = delete;
    Pointer &operator=(Pointer &&) = delete;

    Pointer(Pointer &&) = default;  // required for returning the Pointer

    Pointer(const T &it) : data(fromIterator<T, value_vector>(it)) {}

private:
    value_vector data;
};

template <typename T, typename value_vector, Mutable M> class Reference;

template <typename T, typename value_vector>
class Reference<T, value_vector, Mutable::Yes> : public value_vector
{
    static constexpr auto Size = value_vector::size();

    using reference = typename std::add_lvalue_reference<T>::type;
    reference scalar_it;

public:
    Reference(reference first_it)
        : value_vector(fromIterator<T, value_vector>(first_it)), scalar_it(first_it)
    {
    }

    Reference(const Reference &) = delete;
    Reference(Reference &&) = default;
    Reference &operator=(const Reference &) = delete;
    Reference &operator=(Reference &&) = delete;

    void operator=(const value_vector &x)
    {
        static_cast<value_vector &>(*this) = x;
        auto it = scalar_it;
        for (size_t i = 0; i < Size; ++i, ++it) {
            *it = extract(x, i);
        }
    }
};
#define Vc_OP(op_)                                                                       \
    template <typename T0, typename V0, typename T1, typename V1>                        \
    decltype(std::declval<const V0 &>() op_ std::declval<const V1 &>()) operator op_(    \
        const Reference<T0, V0, Mutable::Yes> &x,                                        \
        const Reference<T1, V1, Mutable::Yes> &y)                                        \
    {                                                                                    \
        return static_cast<const V0 &>(x) op_ static_cast<const V1 &>(y);                \
    }
Vc_ALL_COMPARES(Vc_OP);
Vc_ALL_ARITHMETICS(Vc_OP);
Vc_ALL_BINARY(Vc_OP);
Vc_ALL_LOGICAL(Vc_OP);
Vc_ALL_SHIFTS(Vc_OP);
#undef Vc_OP

template <typename T, typename value_vector>
class Reference<T, value_vector, Mutable::No> : public value_vector
{
    static constexpr auto Size = value_vector::size();

public:
    Reference(const T &it) : value_vector(fromIterator<T, value_vector>(it)) {}

    Reference(const Reference &) = delete;
    Reference(Reference &&) = default;
    Reference &operator=(const Reference &) = delete;
    Reference &operator=(Reference &&) = delete;

    void operator=(const value_vector &x) = delete;
};

template <typename T, size_t N,
          IteratorDetails::Mutable M =
              (Traits::is_output_iterator<T>::value ? Mutable::Yes : Mutable::No),
          typename V = simdize<typename std::iterator_traits<T>::value_type, N>,
          size_t Size = V::Size,
          typename = typename std::iterator_traits<T>::iterator_category>
class Iterator;

template <typename T, size_t N, IteratorDetails::Mutable M, typename V, size_t Size_>
class Iterator<T, N, M, V, Size_, std::forward_iterator_tag>
    : public std::iterator<typename std::iterator_traits<T>::iterator_category, V,
                           typename std::iterator_traits<T>::difference_type,
                           IteratorDetails::Pointer<T, V, M>,
                           IteratorDetails::Reference<T, V, M>>
{
public:
    using pointer = IteratorDetails::Pointer<T, V, M>;
    using reference = IteratorDetails::Reference<T, V, M>;
    using const_pointer = IteratorDetails::Pointer<T, V, IteratorDetails::Mutable::No>;
    using const_reference =
        IteratorDetails::Reference<T, V, IteratorDetails::Mutable::No>;

    static constexpr std::size_t size() { return Size_; }
    static constexpr std::size_t Size = Size_;

    Iterator() = default;

    Iterator(const T &x) : scalar_it(x) {}
    Iterator(T &&x) : scalar_it(std::move(x)) {}
    Iterator &operator=(const T &x)
    {
        scalar_it = x;
        return *this;
    }
    Iterator &operator=(T &&x)
    {
        scalar_it = std::move(x);
        return *this;
    }

    Iterator(const Iterator &) = default;
    Iterator(Iterator &&) = default;
    Iterator &operator=(const Iterator &) = default;
    Iterator &operator=(Iterator &&) = default;

    Iterator &operator++()
    {
        std::advance(scalar_it, Size);
        return *this;
    }
    Iterator operator++(int)
    {
        Iterator copy(*this);
        operator++();
        return copy;
    }

    bool operator==(const Iterator &rhs) const
    {
#ifndef NDEBUG
        if (scalar_it == rhs.scalar_it) {
            return true;
        } else {
            T it(scalar_it);
            for (size_t i = 1; i < Size; ++i) {
                Vc_ASSERT((++it != rhs.scalar_it));
            }
            return false;
        }
#else
        return scalar_it == rhs.scalar_it;
#endif
    }
    bool operator!=(const Iterator &rhs) const
    {
        return !operator==(rhs);
    }

    pointer operator->() { return scalar_it; }

    reference operator*() { return scalar_it; }

    const_pointer operator->() const { return scalar_it; }

    const_reference operator*() const { return scalar_it; }

    operator const T &() const { return scalar_it; }

protected:
    T scalar_it;
};

template <typename T, size_t N, IteratorDetails::Mutable M, typename V, size_t Size>
class Iterator<T, N, M, V, Size, std::bidirectional_iterator_tag>
    : public Iterator<T, N, M, V, Size, std::forward_iterator_tag>
{
    using Base = Iterator<T, N, M, V, Size, std::forward_iterator_tag>;

protected:
    using Base::scalar_it;

public:
    using pointer = typename Base::pointer;
    using reference = typename Base::reference;
    using const_pointer = typename Base::const_pointer;
    using const_reference = typename Base::const_reference;

    using Iterator<T, N, M, V, Size,
                   std::forward_iterator_tag>::Iterator;  // in short: "using
                                                          // Base::Iterator", but that
                                                          // confuses ICC
    Iterator &operator--()
    {
        std::advance(scalar_it, -Size);
        return *this;
    }
    Iterator operator--(int)
    {
        Iterator copy(*this);
        operator--();
        return copy;
    }
};

template <typename T, size_t N, IteratorDetails::Mutable M, typename V, size_t Size>
class Iterator<T, N, M, V, Size, std::random_access_iterator_tag>
    : public Iterator<T, N, M, V, Size, std::bidirectional_iterator_tag>
{
    using Base = Iterator<T, N, M, V, Size, std::bidirectional_iterator_tag>;

protected:
    using Base::scalar_it;

public:
    using pointer = typename Base::pointer;
    using reference = typename Base::reference;
    using const_pointer = typename Base::const_pointer;
    using const_reference = typename Base::const_reference;
    using difference_type = typename std::iterator_traits<T>::difference_type;

    using Iterator<T, N, M, V, Size, std::bidirectional_iterator_tag>::
        Iterator;  // in short: "using Base::Iterator", but that confuses ICC

    Iterator &operator+=(difference_type n)
    {
        scalar_it += n * difference_type(Size);
        return *this;
    }
    Iterator operator+(difference_type n) const { return Iterator(*this) += n; }

    Iterator &operator-=(difference_type n)
    {
        scalar_it -= n * difference_type(Size);
        return *this;
    }
    Iterator operator-(difference_type n) const { return Iterator(*this) -= n; }

    difference_type operator-(const Iterator &rhs) const
    {
        constexpr difference_type n = Size;
        Vc_ASSERT((scalar_it - rhs.scalar_it) % n ==
                  0);  // if this fails the two iterators are not a multiple of the vector
                       // width apart. The distance would be fractional and that doesn't
                       // make too much sense for iteration. Therefore, it is a
                       // precondition for the distance of the two iterators to be a
                       // multiple of Size.
        return (scalar_it - rhs.scalar_it) / n;
    }

    bool operator<(const Iterator &rhs) const
    {
        return rhs.scalar_it - scalar_it >= difference_type(Size);
    }

    bool operator>(const Iterator &rhs) const
    {
        return scalar_it - rhs.scalar_it >= difference_type(Size);
    }

    bool operator<=(const Iterator &rhs) const
    {
        return rhs.scalar_it - scalar_it >= difference_type(Size) - 1;
    }

    bool operator>=(const Iterator &rhs) const
    {
        return scalar_it - rhs.scalar_it >= difference_type(Size) - 1;
    }

    reference operator[](difference_type i) { return *(*this + i); }
    const_reference operator[](difference_type i) const { return *(*this + i); }
};

template <typename T, size_t N, IteratorDetails::Mutable M, typename V, size_t Size>
Iterator<T, N, M, V, Size, std::random_access_iterator_tag> operator+(
    typename Iterator<T, N, M, V, Size, std::random_access_iterator_tag>::difference_type
        n,
    const Iterator<T, N, M, V, Size, std::random_access_iterator_tag> &i)
{
    return i + n;
}

}  // namespace IteratorDetails }}}

template <typename T, size_t N, typename MT>
struct ReplaceTypes<T, N, MT, Category::ForwardIterator>
{
    using type = IteratorDetails::Iterator<T, N>;
};
template <typename T, size_t N, typename MT>
struct ReplaceTypes<T, N, MT, Category::BidirectionalIterator>
{
    using type = IteratorDetails::Iterator<T, N>;
};
template <typename T, size_t N, typename MT>
struct ReplaceTypes<T, N, MT, Category::RandomAccessIterator>
{
    using type = IteratorDetails::Iterator<T, N>;
};

template <Vc::Operator Op, typename S, typename T, std::size_t N, typename M, typename U,
          std::size_t Offset>
Vc_INTRINSIC Vc::enable_if<(Offset >= determine_tuple_size_<S>::value && M::Size == N), void>
    conditional_assign(Adapter<S, T, N> &, const M &, const U &)
{
}
template <Vc::Operator Op, typename S, typename T, std::size_t N, typename M, typename U,
          std::size_t Offset = 0>
Vc_INTRINSIC Vc::enable_if<(Offset < determine_tuple_size_<S>::value && M::Size == N), void>
    conditional_assign(Adapter<S, T, N> &lhs, const M &mask, const U &rhs)
{
    using V = typename std::decay<decltype(get_dispatcher<Offset>(lhs))>::type;
    using M2 = typename V::mask_type;
    conditional_assign<Op>(get_dispatcher<Offset>(lhs), simd_cast<M2>(mask), get_dispatcher<Offset>(rhs));
    conditional_assign<Op, S, T, N, M, U, Offset + 1>(lhs, mask, rhs);
}
template <Vc::Operator Op, typename S, typename T, std::size_t N, typename M,
          std::size_t Offset>
Vc_INTRINSIC Vc::enable_if<(Offset >= determine_tuple_size_<S>::value && M::Size == N), void>
    conditional_assign(Adapter<S, T, N> &, const M &)
{
}
template <Vc::Operator Op, typename S, typename T, std::size_t N, typename M,
          std::size_t Offset = 0>
Vc_INTRINSIC Vc::enable_if<(Offset < determine_tuple_size_<S>::value && M::Size == N), void>
    conditional_assign(Adapter<S, T, N> &lhs, const M &mask)
{
    using V = typename std::decay<decltype(get_dispatcher<Offset>(lhs))>::type;
    using M2 = typename V::mask_type;
    conditional_assign<Op>(get_dispatcher<Offset>(lhs), simd_cast<M2>(mask));
    conditional_assign<Op, S, T, N, M, Offset + 1>(lhs, mask);
}

}  // namespace SimdizeDetail

// user API {{{
template <typename T, size_t N = 0, typename MT = void>
using simdize = SimdizeDetail::simdize<T, N, MT>;

#define Vc_SIMDIZE_INTERFACE(MEMBERS_)                                                   \
    template <std::size_t N_>                                                            \
    inline auto vc_get_()->decltype(std::get<N_>(std::tie MEMBERS_))                     \
    {                                                                                    \
        return std::get<N_>(std::tie MEMBERS_);                                          \
    }                                                                                    \
    template <std::size_t N_>                                                            \
    inline auto vc_get_() const->decltype(std::get<N_>(std::tie MEMBERS_))               \
    {                                                                                    \
        return std::get<N_>(std::tie MEMBERS_);                                          \
    }                                                                                    \
    enum : std::size_t {                                                                 \
        tuple_size = std::tuple_size<decltype(std::make_tuple MEMBERS_)>::value          \
    }
// }}}
}  // namespace Vc

namespace std  // {{{
{
using Vc::SimdizeDetail::swap;
}  // namespace std }}}

#endif  // VC_COMMON_SIMDIZE_H_

// vim: foldmethod=marker