diff mbox series

[1/2] lib/eal: add amd epyc2 memcpy routine to eal

Message ID 20210823084411.29592-1-aman.kumar@vvdntech.in (mailing list archive)
State Superseded
Delegated to: Thomas Monjalon
Headers show
Series [1/2] lib/eal: add amd epyc2 memcpy routine to eal | expand

Checks

Context Check Description
ci/checkpatch success coding style OK

Commit Message

Aman Kumar Aug. 23, 2021, 8:44 a.m. UTC
This patch provides rte_memcpy* calls optimized for
AMD EPYC Gen2 platforms. This option is disabled by
default and can be enabled by defining 'rte_memcpy_amdepyc2'
in the meson build.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 lib/eal/x86/include/meson.build  |   1 +
 lib/eal/x86/include/rte_memcpy.h | 502 +++++++++++++++++++++++++++++++
 meson_options.txt                |   2 +
 3 files changed, 505 insertions(+)

Comments

Jerin Jacob Aug. 23, 2021, 3:21 p.m. UTC | #1
On Mon, Aug 23, 2021 at 2:14 PM Aman Kumar <aman.kumar@vvdntech.in> wrote:
>
> This patch provides rte_memcpy* calls optimized for
> AMD EPYC Gen2 platforms. This option is disabled by
> default and can be enabled by defining 'rte_memcpy_amdepyc2'

Generic options should support all the architectures.
Another more scalable alternative is to introduce config/x86/x86_amd_epyc2
and new parameter under [properties]


> in the meson build.
>
> Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> ---
>  lib/eal/x86/include/meson.build  |   1 +
>  lib/eal/x86/include/rte_memcpy.h | 502 +++++++++++++++++++++++++++++++
>  meson_options.txt                |   2 +
>  3 files changed, 505 insertions(+)
>
> diff --git a/lib/eal/x86/include/meson.build b/lib/eal/x86/include/meson.build
> index 12c2e00035..a03683779d 100644
> --- a/lib/eal/x86/include/meson.build
> +++ b/lib/eal/x86/include/meson.build
> @@ -27,3 +27,4 @@ arch_indirect_headers = files(
>  )
>  install_headers(arch_headers + arch_indirect_headers, subdir: get_option('include_subdir_arch'))
>  dpdk_chkinc_headers += arch_headers
> +dpdk_conf.set('RTE_MEMCPY_AMDEPYC2', get_option('rte_memcpy_amdepyc2'))
> diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
> index 79f381dd9b..47dda9cb87 100644
> --- a/lib/eal/x86/include/rte_memcpy.h
> +++ b/lib/eal/x86/include/rte_memcpy.h
> @@ -368,6 +368,498 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n)
>         }
>  }
>
> +#if defined RTE_MEMCPY_AMDEPYC2
> +
> +/**
> + * Copy 16 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy16_ts(uint8_t *dst, uint8_t *src)
> +{
> +       __m128i var128;
> +
> +       var128 = _mm_stream_load_si128((__m128i *)src);
> +       _mm_storeu_si128((__m128i *)dst, var128);
> +}
> +
> +/**
> + * Copy 32 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy32_ts(uint8_t *dst, uint8_t *src)
> +{
> +       __m256i ymm0;
> +
> +       ymm0 = _mm256_stream_load_si256((const __m256i *)src);
> +       _mm256_storeu_si256((__m256i *)dst, ymm0);
> +}
> +
> +/**
> + * Copy 64 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy64_ts(uint8_t *dst, uint8_t *src)
> +{
> +       rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
> +       rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
> +}
> +
> +/**
> + * Copy 128 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy128_ts(uint8_t *dst, uint8_t *src)
> +{
> +       rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
> +       rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
> +       rte_copy32_ts(dst + 2 * 32, src + 2 * 32);
> +       rte_copy32_ts(dst + 3 * 32, src + 3 * 32);
> +}
> +
> +/**
> + * Copy len bytes from one location to another,
> + * with temporal stores 16B aligned
> + */
> +static __rte_always_inline void *
> +rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
> +{
> +       void *dest = dst;
> +
> +       while (len >= 128) {
> +               rte_copy128_ts((uint8_t *)dst, (uint8_t *)src);
> +               dst = (uint8_t *)dst + 128;
> +               src = (uint8_t *)src + 128;
> +               len -= 128;
> +       }
> +       while (len >= 64) {
> +               rte_copy64_ts((uint8_t *)dst, (uint8_t *)src);
> +               dst = (uint8_t *)dst + 64;
> +               src = (uint8_t *)src + 64;
> +               len -= 64;
> +       }
> +       while (len >= 32) {
> +               rte_copy32_ts((uint8_t *)dst, (uint8_t *)src);
> +               dst = (uint8_t *)dst + 32;
> +               src = (uint8_t *)src + 32;
> +               len -= 32;
> +       }
> +       if (len >= 16) {
> +               rte_copy16_ts((uint8_t *)dst, (uint8_t *)src);
> +               dst = (uint8_t *)dst + 16;
> +               src = (uint8_t *)src + 16;
> +               len -= 16;
> +       }
> +       if (len >= 8) {
> +               *(uint64_t *)dst = *(const uint64_t *)src;
> +               dst = (uint8_t *)dst + 8;
> +               src = (uint8_t *)src + 8;
> +               len -= 8;
> +       }
> +       if (len >= 4) {
> +               *(uint32_t *)dst = *(const uint32_t *)src;
> +               dst = (uint8_t *)dst + 4;
> +               src = (uint8_t *)src + 4;
> +               len -= 4;
> +       }
> +       if (len != 0) {
> +               dst = (uint8_t *)dst - (4 - len);
> +               src = (uint8_t *)src - (4 - len);
> +               *(uint32_t *)dst = *(const uint32_t *)src;
> +       }
> +
> +       return dest;
> +}
> +
> +static __rte_always_inline void *
> +rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
> +                                           const void *src,
> +                                           size_t size)
> +{
> +       asm volatile goto("movq %0, %%rsi\n\t"
> +       "movq %1, %%rdi\n\t"
> +       "movq %2, %%rdx\n\t"
> +       "cmpq   $(128), %%rdx\n\t"
> +       "jb     202f\n\t"
> +       "201:\n\t"
> +       "vmovntdqa (%%rsi), %%ymm0\n\t"
> +       "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> +       "vmovntdqa 64(%%rsi), %%ymm2\n\t"
> +       "vmovntdqa 96(%%rsi), %%ymm3\n\t"
> +       "vmovdqu  %%ymm0, (%%rdi)\n\t"
> +       "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> +       "vmovdqu  %%ymm2, 64(%%rdi)\n\t"
> +       "vmovdqu  %%ymm3, 96(%%rdi)\n\t"
> +       "addq   $128, %%rsi\n\t"
> +       "addq   $128, %%rdi\n\t"
> +       "subq   $128, %%rdx\n\t"
> +       "jz     %l[done]\n\t"
> +       "cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
> +       "jae    201b\n\t"
> +       "202:\n\t"
> +       "cmpq   $64, %%rdx\n\t"
> +       "jb     203f\n\t"
> +       "vmovntdqa (%%rsi), %%ymm0\n\t"
> +       "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> +       "vmovdqu  %%ymm0, (%%rdi)\n\t"
> +       "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> +       "addq   $64, %%rsi\n\t"
> +       "addq   $64, %%rdi\n\t"
> +       "subq   $64, %%rdx\n\t"
> +       "jz     %l[done]\n\t"
> +       "203:\n\t"
> +       "cmpq   $32, %%rdx\n\t"
> +       "jb     204f\n\t"
> +       "vmovntdqa (%%rsi), %%ymm0\n\t"
> +       "vmovdqu  %%ymm0, (%%rdi)\n\t"
> +       "addq   $32, %%rsi\n\t"
> +       "addq   $32, %%rdi\n\t"
> +       "subq   $32, %%rdx\n\t"
> +       "jz     %l[done]\n\t"
> +       "204:\n\t"
> +       "cmpb   $16, %%dl\n\t"
> +       "jb     205f\n\t"
> +       "vmovntdqa (%%rsi), %%xmm0\n\t"
> +       "vmovdqu  %%xmm0, (%%rdi)\n\t"
> +       "addq   $16, %%rsi\n\t"
> +       "addq   $16, %%rdi\n\t"
> +       "subq   $16, %%rdx\n\t"
> +       "jz     %l[done]\n\t"
> +       "205:\n\t"
> +       "cmpb   $2, %%dl\n\t"
> +       "jb     208f\n\t"
> +       "cmpb   $4, %%dl\n\t"
> +       "jbe    207f\n\t"
> +       "cmpb   $8, %%dl\n\t"
> +       "jbe    206f\n\t"
> +       "movq   -8(%%rsi,%%rdx), %%rcx\n\t"
> +       "movq   (%%rsi), %%rsi\n\t"
> +       "movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
> +       "movq   %%rsi, (%%rdi)\n\t"
> +       "jmp    %l[done]\n\t"
> +       "206:\n\t"
> +       "movl   -4(%%rsi,%%rdx), %%ecx\n\t"
> +       "movl   (%%rsi), %%esi\n\t"
> +       "movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
> +       "movl   %%esi, (%%rdi)\n\t"
> +       "jmp    %l[done]\n\t"
> +       "207:\n\t"
> +       "movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
> +       "movzwl (%%rsi), %%esi\n\t"
> +       "movw   %%cx, -2(%%rdi,%%rdx)\n\t"
> +       "movw   %%si, (%%rdi)\n\t"
> +       "jmp    %l[done]\n\t"
> +       "208:\n\t"
> +       "movzbl (%%rsi), %%ecx\n\t"
> +       "movb   %%cl, (%%rdi)"
> +       :
> +       : "r"(src), "r"(dst), "r"(size)
> +       : "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3", "memory"
> +       : done
> +       );
> +done:
> +       return dst;
> +}
> +
> +static __rte_always_inline void *
> +rte_memcpy_generic(void *dst, const void *src, size_t len)
> +{
> +       asm goto("movq  %0, %%rsi\n\t"
> +       "movq   %1, %%rdi\n\t"
> +       "movq   %2, %%rdx\n\t"
> +       "movq    %%rdi, %%rax\n\t"
> +       "cmp     $32, %%rdx\n\t"
> +       "jb      101f\n\t"
> +       "cmp     $(32 * 2), %%rdx\n\t"
> +       "ja      108f\n\t"
> +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
> +       "vmovdqu   %%ymm0, (%%rdi)\n\t"
> +       "vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]\n\t"
> +       "101:\n\t"
> +       /* Less than 1 VEC.  */
> +       "cmpb    $32, %%dl\n\t"
> +       "jae     103f\n\t"
> +       "cmpb    $16, %%dl\n\t"
> +       "jae     104f\n\t"
> +       "cmpb    $8, %%dl\n\t"
> +       "jae     105f\n\t"
> +       "cmpb    $4, %%dl\n\t"
> +       "jae     106f\n\t"
> +       "cmpb    $1, %%dl\n\t"
> +       "ja      107f\n\t"
> +       "jb      102f\n\t"
> +       "movzbl  (%%rsi), %%ecx\n\t"
> +       "movb    %%cl, (%%rdi)\n\t"
> +       "102:\n\t"
> +       "jmp %l[done]\n\t"
> +       "103:\n\t"
> +       /* From 32 to 63.  No branch when size == 32.  */
> +       "vmovdqu (%%rsi), %%ymm0\n\t"
> +       "vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
> +       "vmovdqu %%ymm0, (%%rdi)\n\t"
> +       "vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]\n\t"
> +       /* From 16 to 31.  No branch when size == 16.  */
> +       "104:\n\t"
> +       "vmovdqu (%%rsi), %%xmm0\n\t"
> +       "vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
> +       "vmovdqu %%xmm0, (%%rdi)\n\t"
> +       "vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
> +       "jmp %l[done]\n\t"
> +       "105:\n\t"
> +       /* From 8 to 15.  No branch when size == 8.  */
> +       "movq    -8(%%rsi,%%rdx), %%rcx\n\t"
> +       "movq    (%%rsi), %%rsi\n\t"
> +       "movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
> +       "movq    %%rsi, (%%rdi)\n\t"
> +       "jmp %l[done]\n\t"
> +       "106:\n\t"
> +       /* From 4 to 7.  No branch when size == 4.  */
> +       "movl    -4(%%rsi,%%rdx), %%ecx\n\t"
> +       "movl    (%%rsi), %%esi\n\t"
> +       "movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
> +       "movl    %%esi, (%%rdi)\n\t"
> +       "jmp %l[done]\n\t"
> +       "107:\n\t"
> +       /* From 2 to 3.  No branch when size == 2.  */
> +       "movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
> +       "movzwl  (%%rsi), %%esi\n\t"
> +       "movw    %%cx, -2(%%rdi,%%rdx)\n\t"
> +       "movw    %%si, (%%rdi)\n\t"
> +       "jmp %l[done]\n\t"
> +       "108:\n\t"
> +       /* More than 2 * VEC and there may be overlap between destination */
> +       /* and source.  */
> +       "cmpq    $(32 * 8), %%rdx\n\t"
> +       "ja      111f\n\t"
> +       "cmpq    $(32 * 4), %%rdx\n\t"
> +       "jb      109f\n\t"
> +       /* Copy from 4 * VEC to 8 * VEC, inclusively. */
> +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> +       "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> +       "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
> +       "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
> +       "vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
> +       "vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
> +       "vmovdqu   %%ymm0, (%%rdi)\n\t"
> +       "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> +       "vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
> +       "vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
> +       "vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
> +       "vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
> +       "vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
> +       "vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]\n\t"
> +       "109:\n\t"
> +       /* Copy from 2 * VEC to 4 * VEC. */
> +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
> +       "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
> +       "vmovdqu   %%ymm0, (%%rdi)\n\t"
> +       "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> +       "vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
> +       "vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
> +       "vzeroupper\n\t"
> +       "110:\n\t"
> +       "jmp %l[done]\n\t"
> +       "111:\n\t"
> +       "cmpq    %%rsi, %%rdi\n\t"
> +       "ja      113f\n\t"
> +       /* Source == destination is less common.  */
> +       "je      110b\n\t"
> +       /* Load the first VEC and last 4 * VEC to
> +        * support overlapping addresses.
> +        */
> +       "vmovdqu   (%%rsi), %%ymm4\n\t"
> +       "vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
> +       "vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
> +       "vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
> +       "vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
> +       /* Save start and stop of the destination buffer.  */
> +       "movq    %%rdi, %%r11\n\t"
> +       "leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
> +       /* Align destination for aligned stores in the loop.  Compute */
> +       /* how much destination is misaligned.  */
> +       "movq    %%rdi, %%r8\n\t"
> +       "andq    $(32 - 1), %%r8\n\t"
> +       /* Get the negative of offset for alignment.  */
> +       "subq    $32, %%r8\n\t"
> +       /* Adjust source.  */
> +       "subq    %%r8, %%rsi\n\t"
> +       /* Adjust destination which should be aligned now.  */
> +       "subq    %%r8, %%rdi\n\t"
> +       /* Adjust length.  */
> +       "addq    %%r8, %%rdx\n\t"
> +       /* Check non-temporal store threshold.  */
> +       "cmpq    $(1024*1024), %%rdx\n\t"
> +       "ja      115f\n\t"
> +       "112:\n\t"
> +       /* Copy 4 * VEC a time forward.  */
> +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> +       "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> +       "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> +       "addq    $(32 * 4), %%rsi\n\t"
> +       "subq    $(32 * 4), %%rdx\n\t"
> +       "vmovdqa   %%ymm0, (%%rdi)\n\t"
> +       "vmovdqa   %%ymm1, 32(%%rdi)\n\t"
> +       "vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
> +       "vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
> +       "addq    $(32 * 4), %%rdi\n\t"
> +       "cmpq    $(32 * 4), %%rdx\n\t"
> +       "ja      112b\n\t"
> +       /* Store the last 4 * VEC.  */
> +       "vmovdqu   %%ymm5, (%%rcx)\n\t"
> +       "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> +       "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> +       "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> +       /* Store the first VEC.  */
> +       "vmovdqu   %%ymm4, (%%r11)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]\n\t"
> +       "113:\n\t"
> +       /* Load the first 4*VEC and last VEC to support overlapping addresses.*/
> +       "vmovdqu   (%%rsi), %%ymm4\n\t"
> +       "vmovdqu   32(%%rsi), %%ymm5\n\t"
> +       "vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
> +       "vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
> +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
> +       /* Save stop of the destination buffer.  */
> +       "leaq    -32(%%rdi, %%rdx), %%r11\n\t"
> +       /* Align destination end for aligned stores in the loop.  Compute */
> +       /* how much destination end is misaligned.  */
> +       "leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
> +       "movq    %%r11, %%r9\n\t"
> +       "movq    %%r11, %%r8\n\t"
> +       "andq    $(32 - 1), %%r8\n\t"
> +       /* Adjust source.  */
> +       "subq    %%r8, %%rcx\n\t"
> +       /* Adjust the end of destination which should be aligned now.  */
> +       "subq    %%r8, %%r9\n\t"
> +       /* Adjust length.  */
> +       "subq    %%r8, %%rdx\n\t"
> +        /* Check non-temporal store threshold.  */
> +       "cmpq    $(1024*1024), %%rdx\n\t"
> +       "ja      117f\n\t"
> +       "114:\n\t"
> +       /* Copy 4 * VEC a time backward.  */
> +       "vmovdqu   (%%rcx), %%ymm0\n\t"
> +       "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> +       "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> +       "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> +       "subq    $(32 * 4), %%rcx\n\t"
> +       "subq    $(32 * 4), %%rdx\n\t"
> +       "vmovdqa   %%ymm0, (%%r9)\n\t"
> +       "vmovdqa   %%ymm1, -32(%%r9)\n\t"
> +       "vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
> +       "vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
> +       "subq    $(32 * 4), %%r9\n\t"
> +       "cmpq    $(32 * 4), %%rdx\n\t"
> +       "ja      114b\n\t"
> +       /* Store the first 4 * VEC. */
> +       "vmovdqu   %%ymm4, (%%rdi)\n\t"
> +       "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> +       "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> +       "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> +       /* Store the last VEC. */
> +       "vmovdqu   %%ymm8, (%%r11)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]\n\t"
> +
> +       "115:\n\t"
> +       /* Don't use non-temporal store if there is overlap between */
> +       /* destination and source since destination may be in cache */
> +       /* when source is loaded. */
> +       "leaq    (%%rdi, %%rdx), %%r10\n\t"
> +       "cmpq    %%r10, %%rsi\n\t"
> +       "jb      112b\n\t"
> +       "116:\n\t"
> +       /* Copy 4 * VEC a time forward with non-temporal stores.  */
> +       "prefetcht0 (32*4*2)(%%rsi)\n\t"
> +       "prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
> +       "prefetcht0 (32*4*3)(%%rsi)\n\t"
> +       "prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
> +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> +       "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> +       "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> +       "addq    $(32*4), %%rsi\n\t"
> +       "subq    $(32*4), %%rdx\n\t"
> +       "vmovntdq  %%ymm0, (%%rdi)\n\t"
> +       "vmovntdq  %%ymm1, 32(%%rdi)\n\t"
> +       "vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
> +       "vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
> +       "addq    $(32*4), %%rdi\n\t"
> +       "cmpq    $(32*4), %%rdx\n\t"
> +       "ja      116b\n\t"
> +       "sfence\n\t"
> +       /* Store the last 4 * VEC.  */
> +       "vmovdqu   %%ymm5, (%%rcx)\n\t"
> +       "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> +       "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> +       "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> +       /* Store the first VEC.  */
> +       "vmovdqu   %%ymm4, (%%r11)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]\n\t"
> +       "117:\n\t"
> +       /* Don't use non-temporal store if there is overlap between */
> +       /* destination and source since destination may be in cache */
> +       /* when source is loaded.  */
> +       "leaq    (%%rcx, %%rdx), %%r10\n\t"
> +       "cmpq    %%r10, %%r9\n\t"
> +       "jb      114b\n\t"
> +       "118:\n\t"
> +       /* Copy 4 * VEC a time backward with non-temporal stores. */
> +       "prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
> +       "prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
> +       "prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
> +       "prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
> +       "vmovdqu   (%%rcx), %%ymm0\n\t"
> +       "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> +       "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> +       "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> +       "subq    $(32*4), %%rcx\n\t"
> +       "subq    $(32*4), %%rdx\n\t"
> +       "vmovntdq  %%ymm0, (%%r9)\n\t"
> +       "vmovntdq  %%ymm1, -32(%%r9)\n\t"
> +       "vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
> +       "vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
> +       "subq    $(32 * 4), %%r9\n\t"
> +       "cmpq    $(32 * 4), %%rdx\n\t"
> +       "ja      118b\n\t"
> +       "sfence\n\t"
> +       /* Store the first 4 * VEC.  */
> +       "vmovdqu   %%ymm4, (%%rdi)\n\t"
> +       "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> +       "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> +       "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> +       /* Store the last VEC.  */
> +       "vmovdqu   %%ymm8, (%%r11)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]"
> +       :
> +       : "r"(src), "r"(dst), "r"(len)
> +       : "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "ymm0",
> +       "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "memory"
> +       : done
> +       );
> +done:
> +       return dst;
> +}
> +
> +#else
>  static __rte_always_inline void *
>  rte_memcpy_generic(void *dst, const void *src, size_t n)
>  {
> @@ -479,6 +971,8 @@ rte_memcpy_generic(void *dst, const void *src, size_t n)
>         goto COPY_BLOCK_128_BACK31;
>  }
>
> +#endif /* RTE_MEMCPY_AMDEPYC2 */
> +
>  #else /* __AVX512F__ */
>
>  #define ALIGNMENT_MASK 0x0F
> @@ -874,6 +1368,14 @@ rte_memcpy(void *dst, const void *src, size_t n)
>                 return rte_memcpy_generic(dst, src, n);
>  }
>
> +#if defined __AVX2__ && defined(RTE_MEMCPY_AMDEPYC2)
> +static __rte_always_inline void *
> +rte_memcpy_aligned_tstore16(void *dst, void *src, int len)
> +{
> +       return rte_memcpy_aligned_ntload_tstore16_amdepyc2(dst, src, len);
> +}
> +#endif
> +
>  #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 100000)
>  #pragma GCC diagnostic pop
>  #endif
> diff --git a/meson_options.txt b/meson_options.txt
> index 0e92734c49..e232c9c340 100644
> --- a/meson_options.txt
> +++ b/meson_options.txt
> @@ -42,6 +42,8 @@ option('platform', type: 'string', value: 'native', description:
>         'Platform to build, either "native", "generic" or a SoC. Please refer to the Linux build guide for more information.')
>  option('enable_trace_fp', type: 'boolean', value: false, description:
>         'enable fast path trace points.')
> +option('rte_memcpy_amdepyc2', type: 'boolean', value: false, description:
> +       'to enable amd epyc memcpy routines')
>  option('tests', type: 'boolean', value: true, description:
>         'build unit tests')
>  option('use_hpet', type: 'boolean', value: false, description:
> --
> 2.25.1
>
Aman Kumar Aug. 30, 2021, 9:39 a.m. UTC | #2
Hi Jerin,
Thanks for your comments and suggestions.
I will try to update this in V2.

On Mon, Aug 23, 2021 at 8:51 PM Jerin Jacob <jerinjacobk@gmail.com> wrote:

> On Mon, Aug 23, 2021 at 2:14 PM Aman Kumar <aman.kumar@vvdntech.in> wrote:
> >
> > This patch provides rte_memcpy* calls optimized for
> > AMD EPYC Gen2 platforms. This option is disabled by
> > default and can be enabled by defining 'rte_memcpy_amdepyc2'
>
> Generic options should support all the architectures.
> Another more scalable alternative is to introduce config/x86/x86_amd_epyc2
> and new parameter under [properties]
>
>
> > in the meson build.
> >
> > Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> > ---
> >  lib/eal/x86/include/meson.build  |   1 +
> >  lib/eal/x86/include/rte_memcpy.h | 502 +++++++++++++++++++++++++++++++
> >  meson_options.txt                |   2 +
> >  3 files changed, 505 insertions(+)
> >
> > diff --git a/lib/eal/x86/include/meson.build
> b/lib/eal/x86/include/meson.build
> > index 12c2e00035..a03683779d 100644
> > --- a/lib/eal/x86/include/meson.build
> > +++ b/lib/eal/x86/include/meson.build
> > @@ -27,3 +27,4 @@ arch_indirect_headers = files(
> >  )
> >  install_headers(arch_headers + arch_indirect_headers, subdir:
> get_option('include_subdir_arch'))
> >  dpdk_chkinc_headers += arch_headers
> > +dpdk_conf.set('RTE_MEMCPY_AMDEPYC2', get_option('rte_memcpy_amdepyc2'))
> > diff --git a/lib/eal/x86/include/rte_memcpy.h
> b/lib/eal/x86/include/rte_memcpy.h
> > index 79f381dd9b..47dda9cb87 100644
> > --- a/lib/eal/x86/include/rte_memcpy.h
> > +++ b/lib/eal/x86/include/rte_memcpy.h
> > @@ -368,6 +368,498 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src,
> size_t n)
> >         }
> >  }
> >
> > +#if defined RTE_MEMCPY_AMDEPYC2
> > +
> > +/**
> > + * Copy 16 bytes from one location to another,
> > + * with temporal stores
> > + */
> > +static __rte_always_inline void
> > +rte_copy16_ts(uint8_t *dst, uint8_t *src)
> > +{
> > +       __m128i var128;
> > +
> > +       var128 = _mm_stream_load_si128((__m128i *)src);
> > +       _mm_storeu_si128((__m128i *)dst, var128);
> > +}
> > +
> > +/**
> > + * Copy 32 bytes from one location to another,
> > + * with temporal stores
> > + */
> > +static __rte_always_inline void
> > +rte_copy32_ts(uint8_t *dst, uint8_t *src)
> > +{
> > +       __m256i ymm0;
> > +
> > +       ymm0 = _mm256_stream_load_si256((const __m256i *)src);
> > +       _mm256_storeu_si256((__m256i *)dst, ymm0);
> > +}
> > +
> > +/**
> > + * Copy 64 bytes from one location to another,
> > + * with temporal stores
> > + */
> > +static __rte_always_inline void
> > +rte_copy64_ts(uint8_t *dst, uint8_t *src)
> > +{
> > +       rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
> > +       rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
> > +}
> > +
> > +/**
> > + * Copy 128 bytes from one location to another,
> > + * with temporal stores
> > + */
> > +static __rte_always_inline void
> > +rte_copy128_ts(uint8_t *dst, uint8_t *src)
> > +{
> > +       rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
> > +       rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
> > +       rte_copy32_ts(dst + 2 * 32, src + 2 * 32);
> > +       rte_copy32_ts(dst + 3 * 32, src + 3 * 32);
> > +}
> > +
> > +/**
> > + * Copy len bytes from one location to another,
> > + * with temporal stores 16B aligned
> > + */
> > +static __rte_always_inline void *
> > +rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
> > +{
> > +       void *dest = dst;
> > +
> > +       while (len >= 128) {
> > +               rte_copy128_ts((uint8_t *)dst, (uint8_t *)src);
> > +               dst = (uint8_t *)dst + 128;
> > +               src = (uint8_t *)src + 128;
> > +               len -= 128;
> > +       }
> > +       while (len >= 64) {
> > +               rte_copy64_ts((uint8_t *)dst, (uint8_t *)src);
> > +               dst = (uint8_t *)dst + 64;
> > +               src = (uint8_t *)src + 64;
> > +               len -= 64;
> > +       }
> > +       while (len >= 32) {
> > +               rte_copy32_ts((uint8_t *)dst, (uint8_t *)src);
> > +               dst = (uint8_t *)dst + 32;
> > +               src = (uint8_t *)src + 32;
> > +               len -= 32;
> > +       }
> > +       if (len >= 16) {
> > +               rte_copy16_ts((uint8_t *)dst, (uint8_t *)src);
> > +               dst = (uint8_t *)dst + 16;
> > +               src = (uint8_t *)src + 16;
> > +               len -= 16;
> > +       }
> > +       if (len >= 8) {
> > +               *(uint64_t *)dst = *(const uint64_t *)src;
> > +               dst = (uint8_t *)dst + 8;
> > +               src = (uint8_t *)src + 8;
> > +               len -= 8;
> > +       }
> > +       if (len >= 4) {
> > +               *(uint32_t *)dst = *(const uint32_t *)src;
> > +               dst = (uint8_t *)dst + 4;
> > +               src = (uint8_t *)src + 4;
> > +               len -= 4;
> > +       }
> > +       if (len != 0) {
> > +               dst = (uint8_t *)dst - (4 - len);
> > +               src = (uint8_t *)src - (4 - len);
> > +               *(uint32_t *)dst = *(const uint32_t *)src;
> > +       }
> > +
> > +       return dest;
> > +}
> > +
> > +static __rte_always_inline void *
> > +rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
> > +                                           const void *src,
> > +                                           size_t size)
> > +{
> > +       asm volatile goto("movq %0, %%rsi\n\t"
> > +       "movq %1, %%rdi\n\t"
> > +       "movq %2, %%rdx\n\t"
> > +       "cmpq   $(128), %%rdx\n\t"
> > +       "jb     202f\n\t"
> > +       "201:\n\t"
> > +       "vmovntdqa (%%rsi), %%ymm0\n\t"
> > +       "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> > +       "vmovntdqa 64(%%rsi), %%ymm2\n\t"
> > +       "vmovntdqa 96(%%rsi), %%ymm3\n\t"
> > +       "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> > +       "vmovdqu  %%ymm2, 64(%%rdi)\n\t"
> > +       "vmovdqu  %%ymm3, 96(%%rdi)\n\t"
> > +       "addq   $128, %%rsi\n\t"
> > +       "addq   $128, %%rdi\n\t"
> > +       "subq   $128, %%rdx\n\t"
> > +       "jz     %l[done]\n\t"
> > +       "cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
> > +       "jae    201b\n\t"
> > +       "202:\n\t"
> > +       "cmpq   $64, %%rdx\n\t"
> > +       "jb     203f\n\t"
> > +       "vmovntdqa (%%rsi), %%ymm0\n\t"
> > +       "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> > +       "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> > +       "addq   $64, %%rsi\n\t"
> > +       "addq   $64, %%rdi\n\t"
> > +       "subq   $64, %%rdx\n\t"
> > +       "jz     %l[done]\n\t"
> > +       "203:\n\t"
> > +       "cmpq   $32, %%rdx\n\t"
> > +       "jb     204f\n\t"
> > +       "vmovntdqa (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > +       "addq   $32, %%rsi\n\t"
> > +       "addq   $32, %%rdi\n\t"
> > +       "subq   $32, %%rdx\n\t"
> > +       "jz     %l[done]\n\t"
> > +       "204:\n\t"
> > +       "cmpb   $16, %%dl\n\t"
> > +       "jb     205f\n\t"
> > +       "vmovntdqa (%%rsi), %%xmm0\n\t"
> > +       "vmovdqu  %%xmm0, (%%rdi)\n\t"
> > +       "addq   $16, %%rsi\n\t"
> > +       "addq   $16, %%rdi\n\t"
> > +       "subq   $16, %%rdx\n\t"
> > +       "jz     %l[done]\n\t"
> > +       "205:\n\t"
> > +       "cmpb   $2, %%dl\n\t"
> > +       "jb     208f\n\t"
> > +       "cmpb   $4, %%dl\n\t"
> > +       "jbe    207f\n\t"
> > +       "cmpb   $8, %%dl\n\t"
> > +       "jbe    206f\n\t"
> > +       "movq   -8(%%rsi,%%rdx), %%rcx\n\t"
> > +       "movq   (%%rsi), %%rsi\n\t"
> > +       "movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
> > +       "movq   %%rsi, (%%rdi)\n\t"
> > +       "jmp    %l[done]\n\t"
> > +       "206:\n\t"
> > +       "movl   -4(%%rsi,%%rdx), %%ecx\n\t"
> > +       "movl   (%%rsi), %%esi\n\t"
> > +       "movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
> > +       "movl   %%esi, (%%rdi)\n\t"
> > +       "jmp    %l[done]\n\t"
> > +       "207:\n\t"
> > +       "movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
> > +       "movzwl (%%rsi), %%esi\n\t"
> > +       "movw   %%cx, -2(%%rdi,%%rdx)\n\t"
> > +       "movw   %%si, (%%rdi)\n\t"
> > +       "jmp    %l[done]\n\t"
> > +       "208:\n\t"
> > +       "movzbl (%%rsi), %%ecx\n\t"
> > +       "movb   %%cl, (%%rdi)"
> > +       :
> > +       : "r"(src), "r"(dst), "r"(size)
> > +       : "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3",
> "memory"
> > +       : done
> > +       );
> > +done:
> > +       return dst;
> > +}
> > +
> > +static __rte_always_inline void *
> > +rte_memcpy_generic(void *dst, const void *src, size_t len)
> > +{
> > +       asm goto("movq  %0, %%rsi\n\t"
> > +       "movq   %1, %%rdi\n\t"
> > +       "movq   %2, %%rdx\n\t"
> > +       "movq    %%rdi, %%rax\n\t"
> > +       "cmp     $32, %%rdx\n\t"
> > +       "jb      101f\n\t"
> > +       "cmp     $(32 * 2), %%rdx\n\t"
> > +       "ja      108f\n\t"
> > +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
> > +       "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "101:\n\t"
> > +       /* Less than 1 VEC.  */
> > +       "cmpb    $32, %%dl\n\t"
> > +       "jae     103f\n\t"
> > +       "cmpb    $16, %%dl\n\t"
> > +       "jae     104f\n\t"
> > +       "cmpb    $8, %%dl\n\t"
> > +       "jae     105f\n\t"
> > +       "cmpb    $4, %%dl\n\t"
> > +       "jae     106f\n\t"
> > +       "cmpb    $1, %%dl\n\t"
> > +       "ja      107f\n\t"
> > +       "jb      102f\n\t"
> > +       "movzbl  (%%rsi), %%ecx\n\t"
> > +       "movb    %%cl, (%%rdi)\n\t"
> > +       "102:\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "103:\n\t"
> > +       /* From 32 to 63.  No branch when size == 32.  */
> > +       "vmovdqu (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
> > +       "vmovdqu %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]\n\t"
> > +       /* From 16 to 31.  No branch when size == 16.  */
> > +       "104:\n\t"
> > +       "vmovdqu (%%rsi), %%xmm0\n\t"
> > +       "vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
> > +       "vmovdqu %%xmm0, (%%rdi)\n\t"
> > +       "vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "105:\n\t"
> > +       /* From 8 to 15.  No branch when size == 8.  */
> > +       "movq    -8(%%rsi,%%rdx), %%rcx\n\t"
> > +       "movq    (%%rsi), %%rsi\n\t"
> > +       "movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
> > +       "movq    %%rsi, (%%rdi)\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "106:\n\t"
> > +       /* From 4 to 7.  No branch when size == 4.  */
> > +       "movl    -4(%%rsi,%%rdx), %%ecx\n\t"
> > +       "movl    (%%rsi), %%esi\n\t"
> > +       "movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
> > +       "movl    %%esi, (%%rdi)\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "107:\n\t"
> > +       /* From 2 to 3.  No branch when size == 2.  */
> > +       "movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
> > +       "movzwl  (%%rsi), %%esi\n\t"
> > +       "movw    %%cx, -2(%%rdi,%%rdx)\n\t"
> > +       "movw    %%si, (%%rdi)\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "108:\n\t"
> > +       /* More than 2 * VEC and there may be overlap between
> destination */
> > +       /* and source.  */
> > +       "cmpq    $(32 * 8), %%rdx\n\t"
> > +       "ja      111f\n\t"
> > +       "cmpq    $(32 * 4), %%rdx\n\t"
> > +       "jb      109f\n\t"
> > +       /* Copy from 4 * VEC to 8 * VEC, inclusively. */
> > +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +       "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > +       "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
> > +       "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
> > +       "vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
> > +       "vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
> > +       "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
> > +       "vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
> > +       "vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
> > +       "vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "109:\n\t"
> > +       /* Copy from 2 * VEC to 4 * VEC. */
> > +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
> > +       "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
> > +       "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
> > +       "vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
> > +       "vzeroupper\n\t"
> > +       "110:\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "111:\n\t"
> > +       "cmpq    %%rsi, %%rdi\n\t"
> > +       "ja      113f\n\t"
> > +       /* Source == destination is less common.  */
> > +       "je      110b\n\t"
> > +       /* Load the first VEC and last 4 * VEC to
> > +        * support overlapping addresses.
> > +        */
> > +       "vmovdqu   (%%rsi), %%ymm4\n\t"
> > +       "vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
> > +       "vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
> > +       "vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
> > +       "vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
> > +       /* Save start and stop of the destination buffer.  */
> > +       "movq    %%rdi, %%r11\n\t"
> > +       "leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
> > +       /* Align destination for aligned stores in the loop.  Compute */
> > +       /* how much destination is misaligned.  */
> > +       "movq    %%rdi, %%r8\n\t"
> > +       "andq    $(32 - 1), %%r8\n\t"
> > +       /* Get the negative of offset for alignment.  */
> > +       "subq    $32, %%r8\n\t"
> > +       /* Adjust source.  */
> > +       "subq    %%r8, %%rsi\n\t"
> > +       /* Adjust destination which should be aligned now.  */
> > +       "subq    %%r8, %%rdi\n\t"
> > +       /* Adjust length.  */
> > +       "addq    %%r8, %%rdx\n\t"
> > +       /* Check non-temporal store threshold.  */
> > +       "cmpq    $(1024*1024), %%rdx\n\t"
> > +       "ja      115f\n\t"
> > +       "112:\n\t"
> > +       /* Copy 4 * VEC a time forward.  */
> > +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +       "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > +       "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > +       "addq    $(32 * 4), %%rsi\n\t"
> > +       "subq    $(32 * 4), %%rdx\n\t"
> > +       "vmovdqa   %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqa   %%ymm1, 32(%%rdi)\n\t"
> > +       "vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
> > +       "vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
> > +       "addq    $(32 * 4), %%rdi\n\t"
> > +       "cmpq    $(32 * 4), %%rdx\n\t"
> > +       "ja      112b\n\t"
> > +       /* Store the last 4 * VEC.  */
> > +       "vmovdqu   %%ymm5, (%%rcx)\n\t"
> > +       "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> > +       "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> > +       "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> > +       /* Store the first VEC.  */
> > +       "vmovdqu   %%ymm4, (%%r11)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "113:\n\t"
> > +       /* Load the first 4*VEC and last VEC to support overlapping
> addresses.*/
> > +       "vmovdqu   (%%rsi), %%ymm4\n\t"
> > +       "vmovdqu   32(%%rsi), %%ymm5\n\t"
> > +       "vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
> > +       "vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
> > +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
> > +       /* Save stop of the destination buffer.  */
> > +       "leaq    -32(%%rdi, %%rdx), %%r11\n\t"
> > +       /* Align destination end for aligned stores in the loop.
> Compute */
> > +       /* how much destination end is misaligned.  */
> > +       "leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
> > +       "movq    %%r11, %%r9\n\t"
> > +       "movq    %%r11, %%r8\n\t"
> > +       "andq    $(32 - 1), %%r8\n\t"
> > +       /* Adjust source.  */
> > +       "subq    %%r8, %%rcx\n\t"
> > +       /* Adjust the end of destination which should be aligned now.  */
> > +       "subq    %%r8, %%r9\n\t"
> > +       /* Adjust length.  */
> > +       "subq    %%r8, %%rdx\n\t"
> > +        /* Check non-temporal store threshold.  */
> > +       "cmpq    $(1024*1024), %%rdx\n\t"
> > +       "ja      117f\n\t"
> > +       "114:\n\t"
> > +       /* Copy 4 * VEC a time backward.  */
> > +       "vmovdqu   (%%rcx), %%ymm0\n\t"
> > +       "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> > +       "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> > +       "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> > +       "subq    $(32 * 4), %%rcx\n\t"
> > +       "subq    $(32 * 4), %%rdx\n\t"
> > +       "vmovdqa   %%ymm0, (%%r9)\n\t"
> > +       "vmovdqa   %%ymm1, -32(%%r9)\n\t"
> > +       "vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
> > +       "vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
> > +       "subq    $(32 * 4), %%r9\n\t"
> > +       "cmpq    $(32 * 4), %%rdx\n\t"
> > +       "ja      114b\n\t"
> > +       /* Store the first 4 * VEC. */
> > +       "vmovdqu   %%ymm4, (%%rdi)\n\t"
> > +       "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> > +       /* Store the last VEC. */
> > +       "vmovdqu   %%ymm8, (%%r11)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]\n\t"
> > +
> > +       "115:\n\t"
> > +       /* Don't use non-temporal store if there is overlap between */
> > +       /* destination and source since destination may be in cache */
> > +       /* when source is loaded. */
> > +       "leaq    (%%rdi, %%rdx), %%r10\n\t"
> > +       "cmpq    %%r10, %%rsi\n\t"
> > +       "jb      112b\n\t"
> > +       "116:\n\t"
> > +       /* Copy 4 * VEC a time forward with non-temporal stores.  */
> > +       "prefetcht0 (32*4*2)(%%rsi)\n\t"
> > +       "prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
> > +       "prefetcht0 (32*4*3)(%%rsi)\n\t"
> > +       "prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
> > +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +       "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > +       "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > +       "addq    $(32*4), %%rsi\n\t"
> > +       "subq    $(32*4), %%rdx\n\t"
> > +       "vmovntdq  %%ymm0, (%%rdi)\n\t"
> > +       "vmovntdq  %%ymm1, 32(%%rdi)\n\t"
> > +       "vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
> > +       "vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
> > +       "addq    $(32*4), %%rdi\n\t"
> > +       "cmpq    $(32*4), %%rdx\n\t"
> > +       "ja      116b\n\t"
> > +       "sfence\n\t"
> > +       /* Store the last 4 * VEC.  */
> > +       "vmovdqu   %%ymm5, (%%rcx)\n\t"
> > +       "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> > +       "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> > +       "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> > +       /* Store the first VEC.  */
> > +       "vmovdqu   %%ymm4, (%%r11)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "117:\n\t"
> > +       /* Don't use non-temporal store if there is overlap between */
> > +       /* destination and source since destination may be in cache */
> > +       /* when source is loaded.  */
> > +       "leaq    (%%rcx, %%rdx), %%r10\n\t"
> > +       "cmpq    %%r10, %%r9\n\t"
> > +       "jb      114b\n\t"
> > +       "118:\n\t"
> > +       /* Copy 4 * VEC a time backward with non-temporal stores. */
> > +       "prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
> > +       "prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
> > +       "prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
> > +       "prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
> > +       "vmovdqu   (%%rcx), %%ymm0\n\t"
> > +       "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> > +       "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> > +       "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> > +       "subq    $(32*4), %%rcx\n\t"
> > +       "subq    $(32*4), %%rdx\n\t"
> > +       "vmovntdq  %%ymm0, (%%r9)\n\t"
> > +       "vmovntdq  %%ymm1, -32(%%r9)\n\t"
> > +       "vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
> > +       "vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
> > +       "subq    $(32 * 4), %%r9\n\t"
> > +       "cmpq    $(32 * 4), %%rdx\n\t"
> > +       "ja      118b\n\t"
> > +       "sfence\n\t"
> > +       /* Store the first 4 * VEC.  */
> > +       "vmovdqu   %%ymm4, (%%rdi)\n\t"
> > +       "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> > +       /* Store the last VEC.  */
> > +       "vmovdqu   %%ymm8, (%%r11)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]"
> > +       :
> > +       : "r"(src), "r"(dst), "r"(len)
> > +       : "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11",
> "r12", "ymm0",
> > +       "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8",
> "memory"
> > +       : done
> > +       );
> > +done:
> > +       return dst;
> > +}
> > +
> > +#else
> >  static __rte_always_inline void *
> >  rte_memcpy_generic(void *dst, const void *src, size_t n)
> >  {
> > @@ -479,6 +971,8 @@ rte_memcpy_generic(void *dst, const void *src,
> size_t n)
> >         goto COPY_BLOCK_128_BACK31;
> >  }
> >
> > +#endif /* RTE_MEMCPY_AMDEPYC2 */
> > +
> >  #else /* __AVX512F__ */
> >
> >  #define ALIGNMENT_MASK 0x0F
> > @@ -874,6 +1368,14 @@ rte_memcpy(void *dst, const void *src, size_t n)
> >                 return rte_memcpy_generic(dst, src, n);
> >  }
> >
> > +#if defined __AVX2__ && defined(RTE_MEMCPY_AMDEPYC2)
> > +static __rte_always_inline void *
> > +rte_memcpy_aligned_tstore16(void *dst, void *src, int len)
> > +{
> > +       return rte_memcpy_aligned_ntload_tstore16_amdepyc2(dst, src,
> len);
> > +}
> > +#endif
> > +
> >  #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 100000)
> >  #pragma GCC diagnostic pop
> >  #endif
> > diff --git a/meson_options.txt b/meson_options.txt
> > index 0e92734c49..e232c9c340 100644
> > --- a/meson_options.txt
> > +++ b/meson_options.txt
> > @@ -42,6 +42,8 @@ option('platform', type: 'string', value: 'native',
> description:
> >         'Platform to build, either "native", "generic" or a SoC. Please
> refer to the Linux build guide for more information.')
> >  option('enable_trace_fp', type: 'boolean', value: false, description:
> >         'enable fast path trace points.')
> > +option('rte_memcpy_amdepyc2', type: 'boolean', value: false,
> description:
> > +       'to enable amd epyc memcpy routines')
> >  option('tests', type: 'boolean', value: true, description:
> >         'build unit tests')
> >  option('use_hpet', type: 'boolean', value: false, description:
> > --
> > 2.25.1
> >
>
diff mbox series

Patch

diff --git a/lib/eal/x86/include/meson.build b/lib/eal/x86/include/meson.build
index 12c2e00035..a03683779d 100644
--- a/lib/eal/x86/include/meson.build
+++ b/lib/eal/x86/include/meson.build
@@ -27,3 +27,4 @@  arch_indirect_headers = files(
 )
 install_headers(arch_headers + arch_indirect_headers, subdir: get_option('include_subdir_arch'))
 dpdk_chkinc_headers += arch_headers
+dpdk_conf.set('RTE_MEMCPY_AMDEPYC2', get_option('rte_memcpy_amdepyc2'))
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 79f381dd9b..47dda9cb87 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -368,6 +368,498 @@  rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n)
 	}
 }
 
+#if defined RTE_MEMCPY_AMDEPYC2
+
+/**
+ * Copy 16 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy16_ts(uint8_t *dst, uint8_t *src)
+{
+	__m128i var128;
+
+	var128 = _mm_stream_load_si128((__m128i *)src);
+	_mm_storeu_si128((__m128i *)dst, var128);
+}
+
+/**
+ * Copy 32 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy32_ts(uint8_t *dst, uint8_t *src)
+{
+	__m256i ymm0;
+
+	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
+	_mm256_storeu_si256((__m256i *)dst, ymm0);
+}
+
+/**
+ * Copy 64 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy64_ts(uint8_t *dst, uint8_t *src)
+{
+	rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
+	rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
+}
+
+/**
+ * Copy 128 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy128_ts(uint8_t *dst, uint8_t *src)
+{
+	rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
+	rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
+	rte_copy32_ts(dst + 2 * 32, src + 2 * 32);
+	rte_copy32_ts(dst + 3 * 32, src + 3 * 32);
+}
+
+/**
+ * Copy len bytes from one location to another,
+ * with temporal stores 16B aligned
+ */
+static __rte_always_inline void *
+rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
+{
+	void *dest = dst;
+
+	while (len >= 128) {
+		rte_copy128_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 128;
+		src = (uint8_t *)src + 128;
+		len -= 128;
+	}
+	while (len >= 64) {
+		rte_copy64_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 64;
+		src = (uint8_t *)src + 64;
+		len -= 64;
+	}
+	while (len >= 32) {
+		rte_copy32_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 32;
+		src = (uint8_t *)src + 32;
+		len -= 32;
+	}
+	if (len >= 16) {
+		rte_copy16_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 16;
+		src = (uint8_t *)src + 16;
+		len -= 16;
+	}
+	if (len >= 8) {
+		*(uint64_t *)dst = *(const uint64_t *)src;
+		dst = (uint8_t *)dst + 8;
+		src = (uint8_t *)src + 8;
+		len -= 8;
+	}
+	if (len >= 4) {
+		*(uint32_t *)dst = *(const uint32_t *)src;
+		dst = (uint8_t *)dst + 4;
+		src = (uint8_t *)src + 4;
+		len -= 4;
+	}
+	if (len != 0) {
+		dst = (uint8_t *)dst - (4 - len);
+		src = (uint8_t *)src - (4 - len);
+		*(uint32_t *)dst = *(const uint32_t *)src;
+	}
+
+	return dest;
+}
+
+static __rte_always_inline void *
+rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
+					    const void *src,
+					    size_t size)
+{
+	asm volatile goto("movq %0, %%rsi\n\t"
+	"movq %1, %%rdi\n\t"
+	"movq %2, %%rdx\n\t"
+	"cmpq   $(128), %%rdx\n\t"
+	"jb     202f\n\t"
+	"201:\n\t"
+	"vmovntdqa (%%rsi), %%ymm0\n\t"
+	"vmovntdqa 32(%%rsi), %%ymm1\n\t"
+	"vmovntdqa 64(%%rsi), %%ymm2\n\t"
+	"vmovntdqa 96(%%rsi), %%ymm3\n\t"
+	"vmovdqu  %%ymm0, (%%rdi)\n\t"
+	"vmovdqu  %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqu  %%ymm2, 64(%%rdi)\n\t"
+	"vmovdqu  %%ymm3, 96(%%rdi)\n\t"
+	"addq   $128, %%rsi\n\t"
+	"addq   $128, %%rdi\n\t"
+	"subq   $128, %%rdx\n\t"
+	"jz     %l[done]\n\t"
+	"cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
+	"jae    201b\n\t"
+	"202:\n\t"
+	"cmpq   $64, %%rdx\n\t"
+	"jb     203f\n\t"
+	"vmovntdqa (%%rsi), %%ymm0\n\t"
+	"vmovntdqa 32(%%rsi), %%ymm1\n\t"
+	"vmovdqu  %%ymm0, (%%rdi)\n\t"
+	"vmovdqu  %%ymm1, 32(%%rdi)\n\t"
+	"addq   $64, %%rsi\n\t"
+	"addq   $64, %%rdi\n\t"
+	"subq   $64, %%rdx\n\t"
+	"jz     %l[done]\n\t"
+	"203:\n\t"
+	"cmpq   $32, %%rdx\n\t"
+	"jb     204f\n\t"
+	"vmovntdqa (%%rsi), %%ymm0\n\t"
+	"vmovdqu  %%ymm0, (%%rdi)\n\t"
+	"addq   $32, %%rsi\n\t"
+	"addq   $32, %%rdi\n\t"
+	"subq   $32, %%rdx\n\t"
+	"jz     %l[done]\n\t"
+	"204:\n\t"
+	"cmpb   $16, %%dl\n\t"
+	"jb     205f\n\t"
+	"vmovntdqa (%%rsi), %%xmm0\n\t"
+	"vmovdqu  %%xmm0, (%%rdi)\n\t"
+	"addq   $16, %%rsi\n\t"
+	"addq   $16, %%rdi\n\t"
+	"subq   $16, %%rdx\n\t"
+	"jz     %l[done]\n\t"
+	"205:\n\t"
+	"cmpb   $2, %%dl\n\t"
+	"jb     208f\n\t"
+	"cmpb   $4, %%dl\n\t"
+	"jbe    207f\n\t"
+	"cmpb   $8, %%dl\n\t"
+	"jbe    206f\n\t"
+	"movq   -8(%%rsi,%%rdx), %%rcx\n\t"
+	"movq   (%%rsi), %%rsi\n\t"
+	"movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
+	"movq   %%rsi, (%%rdi)\n\t"
+	"jmp    %l[done]\n\t"
+	"206:\n\t"
+	"movl   -4(%%rsi,%%rdx), %%ecx\n\t"
+	"movl   (%%rsi), %%esi\n\t"
+	"movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
+	"movl   %%esi, (%%rdi)\n\t"
+	"jmp    %l[done]\n\t"
+	"207:\n\t"
+	"movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
+	"movzwl (%%rsi), %%esi\n\t"
+	"movw   %%cx, -2(%%rdi,%%rdx)\n\t"
+	"movw   %%si, (%%rdi)\n\t"
+	"jmp    %l[done]\n\t"
+	"208:\n\t"
+	"movzbl (%%rsi), %%ecx\n\t"
+	"movb   %%cl, (%%rdi)"
+	:
+	: "r"(src), "r"(dst), "r"(size)
+	: "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3", "memory"
+	: done
+	);
+done:
+	return dst;
+}
+
+static __rte_always_inline void *
+rte_memcpy_generic(void *dst, const void *src, size_t len)
+{
+	asm goto("movq	%0, %%rsi\n\t"
+	"movq	%1, %%rdi\n\t"
+	"movq	%2, %%rdx\n\t"
+	"movq    %%rdi, %%rax\n\t"
+	"cmp     $32, %%rdx\n\t"
+	"jb      101f\n\t"
+	"cmp     $(32 * 2), %%rdx\n\t"
+	"ja      108f\n\t"
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
+	"vmovdqu   %%ymm0, (%%rdi)\n\t"
+	"vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"101:\n\t"
+	/* Less than 1 VEC.  */
+	"cmpb    $32, %%dl\n\t"
+	"jae     103f\n\t"
+	"cmpb    $16, %%dl\n\t"
+	"jae     104f\n\t"
+	"cmpb    $8, %%dl\n\t"
+	"jae     105f\n\t"
+	"cmpb    $4, %%dl\n\t"
+	"jae     106f\n\t"
+	"cmpb    $1, %%dl\n\t"
+	"ja      107f\n\t"
+	"jb      102f\n\t"
+	"movzbl  (%%rsi), %%ecx\n\t"
+	"movb    %%cl, (%%rdi)\n\t"
+	"102:\n\t"
+	"jmp %l[done]\n\t"
+	"103:\n\t"
+	/* From 32 to 63.  No branch when size == 32.  */
+	"vmovdqu (%%rsi), %%ymm0\n\t"
+	"vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
+	"vmovdqu %%ymm0, (%%rdi)\n\t"
+	"vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	/* From 16 to 31.  No branch when size == 16.  */
+	"104:\n\t"
+	"vmovdqu (%%rsi), %%xmm0\n\t"
+	"vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
+	"vmovdqu %%xmm0, (%%rdi)\n\t"
+	"vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
+	"jmp %l[done]\n\t"
+	"105:\n\t"
+	/* From 8 to 15.  No branch when size == 8.  */
+	"movq    -8(%%rsi,%%rdx), %%rcx\n\t"
+	"movq    (%%rsi), %%rsi\n\t"
+	"movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
+	"movq    %%rsi, (%%rdi)\n\t"
+	"jmp %l[done]\n\t"
+	"106:\n\t"
+	/* From 4 to 7.  No branch when size == 4.  */
+	"movl    -4(%%rsi,%%rdx), %%ecx\n\t"
+	"movl    (%%rsi), %%esi\n\t"
+	"movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
+	"movl    %%esi, (%%rdi)\n\t"
+	"jmp %l[done]\n\t"
+	"107:\n\t"
+	/* From 2 to 3.  No branch when size == 2.  */
+	"movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
+	"movzwl  (%%rsi), %%esi\n\t"
+	"movw    %%cx, -2(%%rdi,%%rdx)\n\t"
+	"movw    %%si, (%%rdi)\n\t"
+	"jmp %l[done]\n\t"
+	"108:\n\t"
+	/* More than 2 * VEC and there may be overlap between destination */
+	/* and source.  */
+	"cmpq    $(32 * 8), %%rdx\n\t"
+	"ja      111f\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"jb      109f\n\t"
+	/* Copy from 4 * VEC to 8 * VEC, inclusively. */
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
+	"vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
+	"vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
+	"vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
+	"vmovdqu   %%ymm0, (%%rdi)\n\t"
+	"vmovdqu   %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
+	"vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
+	"vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"109:\n\t"
+	/* Copy from 2 * VEC to 4 * VEC. */
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
+	"vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
+	"vmovdqu   %%ymm0, (%%rdi)\n\t"
+	"vmovdqu   %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"110:\n\t"
+	"jmp %l[done]\n\t"
+	"111:\n\t"
+	"cmpq    %%rsi, %%rdi\n\t"
+	"ja      113f\n\t"
+	/* Source == destination is less common.  */
+	"je      110b\n\t"
+	/* Load the first VEC and last 4 * VEC to
+	 * support overlapping addresses.
+	 */
+	"vmovdqu   (%%rsi), %%ymm4\n\t"
+	"vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
+	"vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
+	"vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
+	"vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
+	/* Save start and stop of the destination buffer.  */
+	"movq    %%rdi, %%r11\n\t"
+	"leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
+	/* Align destination for aligned stores in the loop.  Compute */
+	/* how much destination is misaligned.  */
+	"movq    %%rdi, %%r8\n\t"
+	"andq    $(32 - 1), %%r8\n\t"
+	/* Get the negative of offset for alignment.  */
+	"subq    $32, %%r8\n\t"
+	/* Adjust source.  */
+	"subq    %%r8, %%rsi\n\t"
+	/* Adjust destination which should be aligned now.  */
+	"subq    %%r8, %%rdi\n\t"
+	/* Adjust length.  */
+	"addq    %%r8, %%rdx\n\t"
+	/* Check non-temporal store threshold.  */
+	"cmpq	 $(1024*1024), %%rdx\n\t"
+	"ja      115f\n\t"
+	"112:\n\t"
+	/* Copy 4 * VEC a time forward.  */
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
+	"addq    $(32 * 4), %%rsi\n\t"
+	"subq    $(32 * 4), %%rdx\n\t"
+	"vmovdqa   %%ymm0, (%%rdi)\n\t"
+	"vmovdqa   %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
+	"vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
+	"addq    $(32 * 4), %%rdi\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"ja      112b\n\t"
+	/* Store the last 4 * VEC.  */
+	"vmovdqu   %%ymm5, (%%rcx)\n\t"
+	"vmovdqu   %%ymm6, -32(%%rcx)\n\t"
+	"vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
+	"vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
+	/* Store the first VEC.  */
+	"vmovdqu   %%ymm4, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"113:\n\t"
+	/* Load the first 4*VEC and last VEC to support overlapping addresses.*/
+	"vmovdqu   (%%rsi), %%ymm4\n\t"
+	"vmovdqu   32(%%rsi), %%ymm5\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
+	/* Save stop of the destination buffer.  */
+	"leaq    -32(%%rdi, %%rdx), %%r11\n\t"
+	/* Align destination end for aligned stores in the loop.  Compute */
+	/* how much destination end is misaligned.  */
+	"leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
+	"movq    %%r11, %%r9\n\t"
+	"movq    %%r11, %%r8\n\t"
+	"andq    $(32 - 1), %%r8\n\t"
+	/* Adjust source.  */
+	"subq    %%r8, %%rcx\n\t"
+	/* Adjust the end of destination which should be aligned now.  */
+	"subq    %%r8, %%r9\n\t"
+	/* Adjust length.  */
+	"subq    %%r8, %%rdx\n\t"
+	 /* Check non-temporal store threshold.  */
+	"cmpq	 $(1024*1024), %%rdx\n\t"
+	"ja      117f\n\t"
+	"114:\n\t"
+	/* Copy 4 * VEC a time backward.  */
+	"vmovdqu   (%%rcx), %%ymm0\n\t"
+	"vmovdqu   -32(%%rcx), %%ymm1\n\t"
+	"vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
+	"vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
+	"subq    $(32 * 4), %%rcx\n\t"
+	"subq    $(32 * 4), %%rdx\n\t"
+	"vmovdqa   %%ymm0, (%%r9)\n\t"
+	"vmovdqa   %%ymm1, -32(%%r9)\n\t"
+	"vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
+	"vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
+	"subq    $(32 * 4), %%r9\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"ja      114b\n\t"
+	/* Store the first 4 * VEC. */
+	"vmovdqu   %%ymm4, (%%rdi)\n\t"
+	"vmovdqu   %%ymm5, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
+	"vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
+	/* Store the last VEC. */
+	"vmovdqu   %%ymm8, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+
+	"115:\n\t"
+	/* Don't use non-temporal store if there is overlap between */
+	/* destination and source since destination may be in cache */
+	/* when source is loaded. */
+	"leaq    (%%rdi, %%rdx), %%r10\n\t"
+	"cmpq    %%r10, %%rsi\n\t"
+	"jb      112b\n\t"
+	"116:\n\t"
+	/* Copy 4 * VEC a time forward with non-temporal stores.  */
+	"prefetcht0 (32*4*2)(%%rsi)\n\t"
+	"prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
+	"prefetcht0 (32*4*3)(%%rsi)\n\t"
+	"prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
+	"addq    $(32*4), %%rsi\n\t"
+	"subq    $(32*4), %%rdx\n\t"
+	"vmovntdq  %%ymm0, (%%rdi)\n\t"
+	"vmovntdq  %%ymm1, 32(%%rdi)\n\t"
+	"vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
+	"vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
+	"addq    $(32*4), %%rdi\n\t"
+	"cmpq    $(32*4), %%rdx\n\t"
+	"ja      116b\n\t"
+	"sfence\n\t"
+	/* Store the last 4 * VEC.  */
+	"vmovdqu   %%ymm5, (%%rcx)\n\t"
+	"vmovdqu   %%ymm6, -32(%%rcx)\n\t"
+	"vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
+	"vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
+	/* Store the first VEC.  */
+	"vmovdqu   %%ymm4, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"117:\n\t"
+	/* Don't use non-temporal store if there is overlap between */
+	/* destination and source since destination may be in cache */
+	/* when source is loaded.  */
+	"leaq    (%%rcx, %%rdx), %%r10\n\t"
+	"cmpq    %%r10, %%r9\n\t"
+	"jb      114b\n\t"
+	"118:\n\t"
+	/* Copy 4 * VEC a time backward with non-temporal stores. */
+	"prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
+	"prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
+	"prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
+	"prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
+	"vmovdqu   (%%rcx), %%ymm0\n\t"
+	"vmovdqu   -32(%%rcx), %%ymm1\n\t"
+	"vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
+	"vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
+	"subq    $(32*4), %%rcx\n\t"
+	"subq    $(32*4), %%rdx\n\t"
+	"vmovntdq  %%ymm0, (%%r9)\n\t"
+	"vmovntdq  %%ymm1, -32(%%r9)\n\t"
+	"vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
+	"vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
+	"subq    $(32 * 4), %%r9\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"ja      118b\n\t"
+	"sfence\n\t"
+	/* Store the first 4 * VEC.  */
+	"vmovdqu   %%ymm4, (%%rdi)\n\t"
+	"vmovdqu   %%ymm5, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
+	"vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
+	/* Store the last VEC.  */
+	"vmovdqu   %%ymm8, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]"
+	:
+	: "r"(src), "r"(dst), "r"(len)
+	: "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "ymm0",
+	"ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "memory"
+	: done
+	);
+done:
+	return dst;
+}
+
+#else
 static __rte_always_inline void *
 rte_memcpy_generic(void *dst, const void *src, size_t n)
 {
@@ -479,6 +971,8 @@  rte_memcpy_generic(void *dst, const void *src, size_t n)
 	goto COPY_BLOCK_128_BACK31;
 }
 
+#endif /* RTE_MEMCPY_AMDEPYC2 */
+
 #else /* __AVX512F__ */
 
 #define ALIGNMENT_MASK 0x0F
@@ -874,6 +1368,14 @@  rte_memcpy(void *dst, const void *src, size_t n)
 		return rte_memcpy_generic(dst, src, n);
 }
 
+#if defined __AVX2__ && defined(RTE_MEMCPY_AMDEPYC2)
+static __rte_always_inline void *
+rte_memcpy_aligned_tstore16(void *dst, void *src, int len)
+{
+	return rte_memcpy_aligned_ntload_tstore16_amdepyc2(dst, src, len);
+}
+#endif
+
 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 100000)
 #pragma GCC diagnostic pop
 #endif
diff --git a/meson_options.txt b/meson_options.txt
index 0e92734c49..e232c9c340 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -42,6 +42,8 @@  option('platform', type: 'string', value: 'native', description:
        'Platform to build, either "native", "generic" or a SoC. Please refer to the Linux build guide for more information.')
 option('enable_trace_fp', type: 'boolean', value: false, description:
        'enable fast path trace points.')
+option('rte_memcpy_amdepyc2', type: 'boolean', value: false, description:
+       'to enable amd epyc memcpy routines')
 option('tests', type: 'boolean', value: true, description:
        'build unit tests')
 option('use_hpet', type: 'boolean', value: false, description: