| /* |
| * ==================================================== |
| * Copyright (C) 2007 by Ellips BV. All rights reserved. |
| * |
| * Permission to use, copy, modify, and distribute this |
| * software is freely granted, provided that this notice |
| * is preserved. |
| * ==================================================== |
| */ |
| |
| #include "x86_64mach.h" |
| |
| .global SYM (memcpy) |
| SOTYPE_FUNCTION(memcpy) |
| |
| SYM (memcpy): |
| movq rdi, rax /* Store destination in return value */ |
| cmpq $16, rdx |
| jb byte_copy |
| |
| movq rdi, r8 /* Align destination on quad word boundary */ |
| andq $7, r8 |
| jz quadword_aligned |
| movq $8, rcx |
| subq r8, rcx |
| subq rcx, rdx |
| rep movsb |
| |
| quadword_aligned: |
| cmpq $256, rdx |
| jb quadword_copy |
| |
| pushq rax |
| pushq r12 |
| pushq r13 |
| pushq r14 |
| |
| movq rdx, rcx /* Copy 128 bytes at a time with minimum cache polution */ |
| shrq $7, rcx |
| |
| .p2align 4 |
| loop: |
| prefetchnta 768 (rsi) |
| prefetchnta 832 (rsi) |
| |
| movq (rsi), rax |
| movq 8 (rsi), r8 |
| movq 16 (rsi), r9 |
| movq 24 (rsi), r10 |
| movq 32 (rsi), r11 |
| movq 40 (rsi), r12 |
| movq 48 (rsi), r13 |
| movq 56 (rsi), r14 |
| |
| movntiq rax, (rdi) |
| movntiq r8 , 8 (rdi) |
| movntiq r9 , 16 (rdi) |
| movntiq r10, 24 (rdi) |
| movntiq r11, 32 (rdi) |
| movntiq r12, 40 (rdi) |
| movntiq r13, 48 (rdi) |
| movntiq r14, 56 (rdi) |
| |
| movq 64 (rsi), rax |
| movq 72 (rsi), r8 |
| movq 80 (rsi), r9 |
| movq 88 (rsi), r10 |
| movq 96 (rsi), r11 |
| movq 104 (rsi), r12 |
| movq 112 (rsi), r13 |
| movq 120 (rsi), r14 |
| |
| movntiq rax, 64 (rdi) |
| movntiq r8 , 72 (rdi) |
| movntiq r9 , 80 (rdi) |
| movntiq r10, 88 (rdi) |
| movntiq r11, 96 (rdi) |
| movntiq r12, 104 (rdi) |
| movntiq r13, 112 (rdi) |
| movntiq r14, 120 (rdi) |
| |
| leaq 128 (rsi), rsi |
| leaq 128 (rdi), rdi |
| |
| dec rcx |
| jnz loop |
| |
| sfence |
| movq rdx, rcx |
| andq $127, rcx |
| rep movsb |
| popq r14 |
| popq r13 |
| popq r12 |
| popq rax |
| ret |
| |
| |
| byte_copy: |
| movq rdx, rcx |
| rep movsb |
| ret |
| |
| |
| quadword_copy: |
| movq rdx, rcx |
| shrq $3, rcx |
| .p2align 4 |
| rep movsq |
| movq rdx, rcx |
| andq $7, rcx |
| rep movsb /* Copy the remaining bytes */ |
| ret |
