/Users/mourrain/Devel/mmx/algebramix/include/algebramix/vector_aligned.hpp

Go to the documentation of this file.

/******************************************************************************
* MODULE     : vector_aligned.hpp
* DESCRIPTION: vectors aligned in memory
* COPYRIGHT  : (C) 2008  Joris van der Hoeven and Gregoire Lecerf
*******************************************************************************
* This software falls under the GNU general public license and comes WITHOUT
* ANY WARRANTY WHATSOEVER. See the file $TEXMACS_PATH/LICENSE for more details.
* If you don't have this file, write to the Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
******************************************************************************/

#ifndef __MMX__VECTOR_ALIGNED__HPP
#define __MMX__VECTOR_ALIGNED__HPP
#include <basix/int.hpp>
#include <numerix/simd.hpp>

namespace mmx {

/******************************************************************************
* Variants for memory alignment of vectors
******************************************************************************/

template<typename V, typename W>
struct vector_assume_aligned: public V {
  // These vectors are assumed to be always aligned
  // V is the variant to be used on base types
  // W is the variant to be used on simd types (see vector_sse.hpp)
  typedef vector_assume_aligned<typename V::Naive,
                                typename W::Naive> Naive;
  typedef vector_assume_aligned<typename V::Aligned,
                                typename W::Aligned> Aligned;
  typedef vector_assume_aligned<typename V::No_simd,
                                typename W::No_simd> No_simd;
  typedef vector_assume_aligned<typename V::No_thread,
                                typename W::No_thread> No_thread;
};

template<typename F, typename Z, typename V, typename W>
struct implementation<F,Z,vector_assume_aligned<V,W> >:
    public implementation<F,Z,V> {};

template<typename V, typename W>
struct vector_aligned: public V {
  // These vectors are not necessarily aligned in memory
  typedef vector_aligned       <typename V::Naive,
                                typename W::Naive> Naive;
  typedef vector_assume_aligned<typename V::Aligned,
                                typename W::Aligned> Aligned;
  typedef vector_aligned       <typename V::No_simd,
                                typename W::No_simd> No_simd;
  typedef vector_aligned       <typename V::No_thread,
                                typename W::No_thread> No_thread;
};

template<typename F, typename Z, typename V, typename W>
struct implementation<F,Z,vector_aligned<V,W> >:
    public implementation<F,Z,V> {};

/******************************************************************************
* Helpers to be specialized on aligned hardware data
******************************************************************************/

template<typename V, typename W,
         typename Op, typename T>
struct vec_nullary_aligned_helper {
  typedef implementation<vector_abstractions,V> Vec;
  static inline void op (T* dest, nat n) {
    Vec::template vec_nullary<Op,T> (dest, n); } };

template<typename V, typename W,
         typename Op, typename T, typename C>
struct vec_unary_aligned_helper {
  typedef implementation<vector_abstractions,V> Vec;
  static inline void op (T* dest, const C* s, nat n) {
    Vec::template vec_unary<Op,T,C> (dest, s, n); } };

template<typename V, typename W,
         typename Op, typename T, typename C1, typename C2>
struct vec_binary_aligned_helper {
  typedef implementation<vector_abstractions,V> Vec;
  static inline void op (T* dest, const C1* s1, const C2* s2, nat n) {
    Vec::template vec_binary<Op,T,C1,C2> (dest, s1, s2, n); } };

template<typename V, typename W,
         typename Op, typename T, typename X>
struct vec_unary_scalar_aligned_helper {
  typedef implementation<vector_abstractions,V> Vec;
  static inline void op (T* dest, const X& x, nat n) {
    Vec::template vec_unary_scalar<Op,T,X> (dest, x, n); } };

template<typename V, typename W,
         typename Op, typename T, typename C, typename X>
struct vec_binary_scalar_aligned_helper {
  typedef implementation<vector_abstractions,V> Vec;
  static inline void op (T* dest, const C* s, const X& x, nat n) {
    Vec::template vec_binary_scalar<Op,T,C,X> (dest, s, x, n); } };

template<typename V, typename W,
         typename Op, typename C>
struct vec_unary_test_aligned_helper {
  typedef implementation<vector_abstractions,V> Vec;
  static inline bool op (const C* s, nat n) {
    return Vec::template vec_unary_test (s, n); } };

template<typename V, typename W,
         typename Op, typename C1, typename C2>
struct vec_binary_test_aligned_helper {
  typedef implementation<vector_abstractions,V> Vec;
  static inline bool op (const C1* s1, const C2* s2, nat n) {
    return Vec::template vec_binary_test<Op,C1,C2> (s1, s2, n); } };

template<typename V, typename W,
         typename Op, typename C, typename X>
struct vec_binary_test_scalar_aligned_helper {
  typedef implementation<vector_abstractions,V> Vec;
  static inline bool op (const C* s, const X& x, nat n) {
    return Vec::template vec_binary_test_scalar<Op,C,X> (s, x, n); } };

template<typename V, typename W,
         typename Op, typename C>
struct vec_unary_big_aligned_helper {
  typedef implementation<vector_abstractions,V> Vec;
  static inline C op (const C* s, nat n) {
    return Vec::template vec_unary_big<Op,C> (s, n); }
  static inline C op (const C* s, nat n, const format<C>& fm) {
    return Vec::template vec_unary_big<Op,C> (s, n, fm); } };

template<typename V, typename W,
         typename Op, typename C1, typename C2>
struct vec_binary_big_aligned_helper {
  typedef implementation<vector_abstractions,V> Vec;
  static inline C1 op (const C1* s1, const C2* s2, nat n) {
    return Vec::template vec_binary_big<Op,C1,C2> (s1, s2, n); }
  static inline C1 op (const C1* s1, const C2* s2, nat n,
                       const format<C1>& fm1, const format<C2>& fm2) {
    return Vec::template vec_binary_big<Op,C1,C2> (s1, s2, n, fm1, fm2); } };

/******************************************************************************
* Mask helper
******************************************************************************/

// Alignment 'len' of C is determined from its corresponding simd type.
// So far memory alignment is only suported for multiples of 16 bytes.
template<typename C>
struct mask_helper {
  static const nat m           = Simd_size (C);
  static const nat log_m       = int_bitsize_helper<nat, m>::value - 1;
  static const nat hi_mask_m   = ((nat) -1) << log_m;
  static const nat lo_mask_m   = m - 1;
  static const intptr_t len    = m * sizeof (C);
  static const intptr_t log_len= int_bitsize_helper<nat, len>::value - 1;
  static const intptr_t lo_mask_len= len - 1;
};

/******************************************************************************
* Low level routines on vectors which are assumed to be aligned in memory
******************************************************************************/

template<typename Z, typename V, typename W>
struct implementation<vector_allocate,Z,vector_assume_aligned<V,W> >:
  public implementation<vector_defaults,Z>
{

template<typename C> static inline nat
vec_aligned_size (nat n) {
  VERIFY (mask_helper<C>::m == (((nat) 1) << mask_helper<C>::log_m),
          "simd size must be a power of two");
  return (n + mask_helper<C>::m - 1) & mask_helper<C>::hi_mask_m;
}

template<typename C> static inline nat
vec_floor_aligned_size (nat n) {
  return n & mask_helper<C>::hi_mask_m;
}

template<typename C> static inline bool
vec_is_aligned_size (nat n) {
  return (n & mask_helper<C>::lo_mask_m) == 0; }

template<typename C> static inline bool
vec_is_aligned (const C* s) {
  return (mask_helper<C>::len == 16) ?
    (((intptr_t) s) & mask_helper<C>::lo_mask_len) == 0 : false; }

template<typename C> static inline intptr_t
vec_ceil_shift (const C* s) {
  if (mask_helper<C>::len == 16) {
    intptr_t r= (intptr_t) s & mask_helper<C>::lo_mask_len;
    return (r != 0) ? mask_helper<C>::len - r : r;
  }
  return 0; }

}; // implementation<vector_allocate,Z,vector_assume_aligned<V,W> >

template<typename Z, typename V, typename W>
struct implementation<vector_abstractions,Z,vector_assume_aligned<V,W> >:
  public implementation<vector_allocate,Z>
{
  typedef implementation<vector_allocate,Z> Vec;

template<typename Op, typename T> static inline void
vec_nullary (T* dest, nat n) {
  VERIFY (Vec::vec_is_aligned (dest), "address must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<T> (n), "size must be aligned");
  vec_nullary_aligned_helper<V,W,Op,T>::op (dest, n); }

template<typename Op, typename T, typename C> static inline void
vec_unary (T* dest, const C* s, nat n) {
  VERIFY (Vec::vec_is_aligned (dest), "address must be aligned");
  VERIFY (Vec::vec_is_aligned (s)   , "address must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<T> (n), "size must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C> (n), "size must be aligned");
  vec_unary_aligned_helper<V,W,Op,T,C>::op (dest, s, n); }

template<typename Op, typename T, typename C1, typename C2> static inline void
vec_binary (T* dest, const C1* s1, const C2* s2, nat n) {
  VERIFY (Vec::vec_is_aligned (dest)  , "address must be aligned");
  VERIFY (Vec::vec_is_aligned (s1)    , "address must be aligned");
  VERIFY (Vec::vec_is_aligned (s2)    , "address must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<T> (n), "size must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C1>(n), "size must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C2>(n), "size must be aligned");
  vec_binary_aligned_helper<V,W,Op,T,C1,C2>::op (dest, s1, s2, n); }

template<typename Op, typename T, typename X> static inline void
vec_unary_scalar (T* dest, const X& x, nat n) {
  VERIFY (Vec::vec_is_aligned (dest), "address must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<T> (n), "size must be aligned");
  vec_unary_scalar_aligned_helper<V,W,Op,T,X>::op (dest, x, n); }

template<typename Op, typename T, typename C, typename X> static inline void
vec_binary_scalar (T* dest, const C* s, const X& x, nat n) {
  VERIFY (Vec::vec_is_aligned (dest), "address must be aligned");
  VERIFY (Vec::vec_is_aligned (s)   , "address must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<T> (n), "size must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C> (n), "size must be aligned");
  vec_binary_scalar_aligned_helper<V,W,Op,T,C,X>::op (dest, s, x, n); }

template<typename Op, typename C> static inline bool
vec_unary_test (const C* s, nat n) {
  VERIFY (Vec::vec_is_aligned (s), "address must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C> (n), "size must be aligned");
  return vec_unary_test_aligned_helper<V,W,Op,C>::op (s, n); }

template<typename Op, typename C1, typename C2> static inline bool
vec_binary_test (const C1* s1, const C2* s2, nat n) {
  VERIFY (Vec::vec_is_aligned (s1), "address must be aligned");
  VERIFY (Vec::vec_is_aligned (s2), "address must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C1> (n), "size must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C2> (n), "size must be aligned");
  return vec_binary_test_aligned_helper<V,W,Op,C1,C2>::op (s1, s2, n); }

template<typename Op, typename C, typename X> static inline bool
vec_binary_test_scalar (const C* s, const X& x, nat n) {
  VERIFY (Vec::vec_is_aligned (s), "address must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C> (n), "size must be aligned");
  return vec_binary_test_scalar_aligned_helper<V,W,Op,C,X>::op (s, x, n); }

template<typename Op, typename C> static inline C
vec_unary_big (const C* s, nat n) {
  VERIFY (Vec::vec_is_aligned (s), "address must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C> (n), "size must be aligned");
  return vec_unary_big_aligned_helper<V,W,Op,C>::op (s, n); }

template<typename Op, typename C> static inline C
vec_unary_big (const C* s, nat n, const format<C>& fm) {
  VERIFY (Vec::vec_is_aligned (s), "address must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C> (n), "size must be aligned");
  return vec_unary_big_aligned_helper<V,W,Op,C>::op (s, n, fm); }

template<typename Op, typename C> static inline C
vec_unary_big_dicho (const C* s, nat n) {
  return vec_unary_big<Op, C> (s, n); }

template<typename Op, typename C> static inline C
vec_unary_big_dicho (const C* s, nat n, const format<C>& fm) {
  return vec_unary_big<Op, C> (s, n, fm); }

template<typename Op, typename C1, typename C2> static inline C1
vec_binary_big (const C1* s1, const C2* s2, nat n) {
  VERIFY (Vec::vec_is_aligned (s1), "address must be aligned");
  VERIFY (Vec::vec_is_aligned (s2), "address must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C1> (n), "size must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C2> (n), "size must be aligned");
  return vec_binary_big_aligned_helper<V,W,Op,C1,C2>::op (s1, s2, n); }

template<typename Op, typename C1, typename C2> static inline C1
vec_binary_big (const C1* s1, const C2* s2, nat n,
                const format<C1>& fm1, const format<C2>& fm2) {
  VERIFY (Vec::vec_is_aligned (s1), "address must be aligned");
  VERIFY (Vec::vec_is_aligned (s2), "address must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C1> (n), "size must be aligned");
  VERIFY (Vec::template vec_is_aligned_size<C2> (n), "size must be aligned");
  return
    vec_binary_big_aligned_helper<V,W,Op,C1,C2>::op (s1, s2, n, fm1, fm2); }

}; // implementation<vector_abstractions,Z,vector_assume_aligned<V,W> >

/******************************************************************************
* Abstract low level routines on vectors not necessarily aligned in memory
******************************************************************************/

template<typename Z, typename V, typename W>
struct implementation<vector_allocate,Z,vector_aligned<V,W> >:
    public implementation<vector_allocate,vector_assume_aligned<V,W> >
{}; // implementation<vector_allocate,Z,vector_aligned<V,W> >

template<typename Z, typename V, typename W>
struct implementation<vector_abstractions,Z,vector_aligned<V,W> >:
    public implementation<vector_abstractions,V>
{
  typedef vector_assume_aligned<V,W> Aligned;
  typedef implementation<vector_abstractions,V> NVec;
  typedef implementation<vector_abstractions,Aligned> AVec;

template<typename C> static inline nat
vec_aligned_size (nat n) {
  return AVec::template vec_aligned_size<C> (n);
}

template<typename Op, typename T>
static inline void
vec_nullary (T* dest, nat n) {
  if (mask_helper<T>::len == 16) {
    nat r= min ((nat) AVec::vec_ceil_shift (dest), n);
    NVec::template vec_nullary<Op,T> (dest, r);
    dest += r; n -= r;
    r= AVec::template vec_floor_aligned_size<T> (n);
    AVec::template vec_nullary<Op,T> (dest, r);
    dest += r; n -= r;
  }
  NVec::template vec_nullary<Op,T> (dest, n);
}

template<typename Op, typename T, typename C> static inline void
vec_unary (T* dest, const C* s, nat n) {
  if (mask_helper<T>::len == 16 &&
      mask_helper<C>::len == 16) {
    nat rd= AVec::vec_ceil_shift (dest);
    nat rs= AVec::vec_ceil_shift (s);
    if (rd == rs) {
      nat r= min (rd, n);
      NVec::template vec_unary<Op,T,C> (dest, s, r);
      dest += r; s += r; n -= r;
      rd= AVec::template vec_floor_aligned_size<T> (n);
      rs= AVec::template vec_floor_aligned_size<C> (n);
      r= min (rd, rs);
      AVec::template vec_unary<Op,T,C> (dest, s, r);
      dest += r; s += r; n -= r;
    }
  }
  NVec::template vec_unary<Op,T,C> (dest, s, n);
}

template<typename Op, typename T, typename C1, typename C2> static inline void
vec_binary (T* dest, const C1* s1, const C2* s2, nat n) {
  if (mask_helper<T >::len == 16 &&
      mask_helper<C1>::len == 16 &&
      mask_helper<C2>::len == 16) {
    nat rd = AVec::vec_ceil_shift (dest);
    nat rs1= AVec::vec_ceil_shift (s1);
    nat rs2= AVec::vec_ceil_shift (s2);
    if (rd == rs1 && rd == rs2) {
      nat r= min (rd, n);
      NVec::template vec_binary<Op,T,C1,C2> (dest, s1, s2, r);
      dest += r; s1 += r; s2 += r; n -= r;
      rd = AVec::template vec_floor_aligned_size<T>  (n);
      rs1= AVec::template vec_floor_aligned_size<C1> (n);
      rs2= AVec::template vec_floor_aligned_size<C2> (n);
      r= min (min (rd, rs1), rs2);
      AVec::template vec_binary<Op,T,C1,C2> (dest, s1, s2, r);
      dest += r; s1 += r; s2 += r; n -= r;
    }
  }
  NVec::template vec_binary<Op,T,C1,C2> (dest, s1, s2, n);
}

template<typename Op, typename T, typename X> static inline void
vec_unary_scalar (T* dest, const X& x, nat n) {
  if (mask_helper<T>::len == 16) {
    nat r= min ((nat) AVec::vec_ceil_shift (dest), n);
    NVec::template vec_unary_scalar<Op,T,X> (dest, x, r);
    dest += r; n -= r;
    r= AVec::template vec_floor_aligned_size<T> (n);
    AVec::template vec_unary_scalar<Op,T,X> (dest, x, r);
    dest += r; n -= r;
  }
  NVec::template vec_unary_scalar<Op,T,X> (dest, x, n);
}

template<typename Op, typename T, typename C, typename X> static inline void
vec_binary_scalar (T* dest, const C* s, const X& x, nat n) {
  if (mask_helper<T>::len == 16 &&
      mask_helper<C>::len == 16) {
    nat rd= AVec::vec_ceil_shift (dest);
    nat rs= AVec::vec_ceil_shift (s);
    if (rd == rs) {
      nat r= min (rd, n);
      NVec::template vec_binary_scalar<Op,T,C,X> (dest, s, x, r);
      dest += r; s += r; n -= r;
      rd= AVec::template vec_floor_aligned_size<T> (n);
      rs= AVec::template vec_floor_aligned_size<C> (n);
      r= min (rd, rs);
      AVec::template vec_binary_scalar<Op,T,C,X> (dest, s, x, r);
      dest += r; s += r; n -= r;
    }
  }
  NVec::template vec_binary_scalar<Op,T,C,X> (dest, s, x, n);
}

template<typename Op, typename C> static inline bool
vec_unary_test (const C* s, nat n) {
  if (mask_helper<C>::len == 16) {
    nat r= min ((nat) AVec::vec_ceil_shift (s), n);
    if (! NVec::template vec_unary_test<Op,C> (s, r)) return false;
    s += r; n -= r;
    r= AVec::template vec_floor_aligned_size<C> (n);
    if (! AVec::template vec_unary_test<Op,C> (s, r)) return false;
    s += r; n -= r;
  }
  return NVec::template vec_unary_test<Op,C> (s, n);
}

template<typename Op, typename C1, typename C2> static inline bool
vec_binary_test (const C1* s1, const C2* s2, nat n) {
  if (mask_helper<C1>::len == 16 &&
      mask_helper<C2>::len == 16) {
    nat r1= AVec::vec_ceil_shift (s1);
    nat r2= AVec::vec_ceil_shift (s2);
    if (r1 == r2) {
      nat r= min (r1, r2);
      if (! NVec::template vec_binary_test<Op,C1,C2> (s1, s2, r))
        return false;
      s1 += r; s2 += r; n -= r;
      r1= AVec::template vec_floor_aligned_size<C1> (n);
      r2= AVec::template vec_floor_aligned_size<C2> (n);
      r= min (r1, r2);
      if (! AVec::template vec_binary_test<Op,C1,C2> (s1, s2, r))
        return false;
      s1 += r; s2 += r; n -= r;
    }
  }
  return NVec::template vec_binary_test<Op,C1,C2> (s1, s2, n);
}
  
template<typename Op, typename C, typename X> static inline bool
vec_binary_test_scalar (const C* s, const X& x, nat n) {
  if (mask_helper<C>::len == 16) {
    nat r= min ((nat) AVec::vec_ceil_shift (s), n);
    if (! NVec::template vec_binary_test_scalar<Op,C,X> (s, x, r))
      return false;
    s += r; n -= r;
    r= AVec::template vec_floor_aligned_size<C> (n);
    if (! AVec::template vec_binary_test_scalar<Op,C,X> (s, x, r))
      return false;
    s += r; n -= r;
  }
  return NVec::template vec_binary_test_scalar<Op,C> (s, x, n);
}

template<typename Op, typename C> static inline C
vec_unary_big (const C* s, nat n) {
  if (mask_helper<C>::len == 16) {
    nat r= min ((nat) AVec::vec_ceil_shift (s), n);
    C c= NVec::template vec_unary_big<Op,C> (s, r);
    s += r; n -= r;
    r= AVec::template vec_floor_aligned_size<C> (n);
    Op::set_op (c, AVec::template vec_unary_big<Op,C> (s, r));
    s += r; n -= r;
    Op::set_op (c, NVec::template vec_unary_big<Op,C> (s, n));
    return c;
  }
  return NVec::template vec_unary_big<Op,C> (s, n);
}

template<typename Op, typename C> static inline C
vec_unary_big (const C* s, nat n, const format<C>& fm) {
  if (mask_helper<C>::len == 16) {
    nat r= min ((nat) AVec::vec_ceil_shift (s), n);
    C c= NVec::template vec_unary_big<Op,C> (s, r, fm);
    s += r; n -= r;
    r= AVec::template vec_floor_aligned_size<C> (n);
    Op::set_op (c, AVec::template vec_unary_big<Op,C> (s, r, fm));
    s += r; n -= r;
    Op::set_op (c, NVec::template vec_unary_big<Op,C> (s, n, fm));
    return c;
  }
  return NVec::template vec_unary_big<Op,C> (s, n, fm);
}

template<typename Op, typename C> static inline C
vec_unary_big_dicho (const C* s, nat n) {
  return NVec::template vec_unary_big<Op, C> (s, n);
}

template<typename Op, typename C> static inline C
vec_unary_big_dicho (const C* s, nat n, const format<C>& fm) {
  return NVec::template vec_unary_big<Op, C> (s, n, fm);
}

template<typename Op, typename C1, typename C2> static inline C1
vec_binary_big (const C1* s1, const C2* s2, nat n) {
  if (AVec::vec_is_aligned (s1) &&
      AVec::template vec_is_aligned_size<C1> (n) &&
      AVec::vec_is_aligned (s2) &&
      AVec::template vec_is_aligned_size<C2> (n))
    return AVec::template vec_binary_big<Op,C1,C2> (s1, s2, n);
  return NVec::template vec_binary_big<Op,C1,C2> (s1, s2, n);
}

template<typename Op, typename C1, typename C2> static inline C1
vec_binary_big (const C1* s1, const C2* s2, nat n,
                const format<C1>& fm1, const format<C2>& fm2) {
  if (AVec::vec_is_aligned (s1) &&
      AVec::template vec_is_aligned_size<C1> (n) &&
      AVec::vec_is_aligned (s2) &&
      AVec::template vec_is_aligned_size<C2> (n))
    return AVec::template vec_binary_big<Op,C1,C2> (s1, s2, n, fm1, fm2);
  return NVec::template vec_binary_big<Op,C1,C2> (s1, s2, n, fm1, fm2);
}
  
}; // implementation<vector_abstractions,Z,vector_aligned<V,W> >

} // namespace mmx
#endif // __MMX__VECTOR_ALIGNED__HPP