我们最近购买了一些新服务器，但 memcpy 性能不佳。与我们的笔记本电脑相比，服务器上的 memcpy 性能要慢 3 倍。

服务器规范

$ cat /etc/redhat-release
Scientific Linux release 6.5 (Carbon) 
$ uname -a                      
Linux r113 2.6.32-431.1.2.el6.x86_64 #1 SMP Thu Dec 12 13:59:19 CST 2013 x86_64 x86_64 x86_64 GNU/Linux

$ gcc --version
gcc (GCC) 4.6.1

#include <chrono>
#include <cstring>
#include <iostream>
#include <cstdint>

class Timer
{
 public:
  Timer()
      : mStart(),
        mStop()
  {
    update();
  }

  void update()
  {
    mStart = std::chrono::high_resolution_clock::now();
    mStop  = mStart;
  }

  double elapsedMs()
  {
    mStop = std::chrono::high_resolution_clock::now();
    std::chrono::milliseconds elapsed_ms =
        std::chrono::duration_cast<std::chrono::milliseconds>(mStop - mStart);
    return elapsed_ms.count();
  }

 private:
  std::chrono::high_resolution_clock::time_point mStart;
  std::chrono::high_resolution_clock::time_point mStop;
};

std::string formatBytes(std::uint64_t bytes)
{
  static const int num_suffix = 5;
  static const char* suffix[num_suffix] = { "B", "KB", "MB", "GB", "TB" };
  double dbl_s_byte = bytes;
  int i = 0;
  for (; (int)(bytes / 1024.) > 0 && i < num_suffix;
       ++i, bytes /= 1024.)
  {
    dbl_s_byte = bytes / 1024.0;
  }

  const int buf_len = 64;
  char buf[buf_len];

  // use snprintf so there is no buffer overrun
  int res = snprintf(buf, buf_len,"%0.2f%s", dbl_s_byte, suffix[i]);

  // snprintf returns number of characters that would have been written if n had
  //       been sufficiently large, not counting the terminating null character.
  //       if an encoding error occurs, a negative number is returned.
  if (res >= 0)
  {
    return std::string(buf);
  }
  return std::string();
}

void doMemmove(void* pDest, const void* pSource, std::size_t sizeBytes)
{
  memmove(pDest, pSource, sizeBytes);
}

int main(int argc, char* argv[])
{
  std::uint64_t SIZE_BYTES = 1073741824; // 1GB

  if (argc > 1)
  {
    SIZE_BYTES = std::stoull(argv[1]);
    std::cout << "Using buffer size from command line: " << formatBytes(SIZE_BYTES)
              << std::endl;
  }
  else
  {
    std::cout << "To specify a custom buffer size: big_memcpy_test [SIZE_BYTES] \n"
              << "Using built in buffer size: " << formatBytes(SIZE_BYTES)
              << std::endl;
  }


  // big array to use for testing
  char* p_big_array = NULL;

  /////////////
  // malloc 
  {
    Timer timer;

    p_big_array = (char*)malloc(SIZE_BYTES * sizeof(char));
    if (p_big_array == NULL)
    {
      std::cerr << "ERROR: malloc of " << SIZE_BYTES << " returned NULL!"
                << std::endl;
      return 1;
    }

    std::cout << "malloc for " << formatBytes(SIZE_BYTES) << " took "
              << timer.elapsedMs() << "ms"
              << std::endl;
  }

  /////////////
  // memset
  {
    Timer timer;

    // set all data in p_big_array to 0
    memset(p_big_array, 0xF, SIZE_BYTES * sizeof(char));

    double elapsed_ms = timer.elapsedMs();
    std::cout << "memset for " << formatBytes(SIZE_BYTES) << " took "
              << elapsed_ms << "ms "
              << "(" << formatBytes(SIZE_BYTES / (elapsed_ms / 1.0e3)) << " bytes/sec)"
              << std::endl;
  }

  /////////////
  // memcpy 
  {
    char* p_dest_array = (char*)malloc(SIZE_BYTES);
    if (p_dest_array == NULL)
    {
      std::cerr << "ERROR: malloc of " << SIZE_BYTES << " for memcpy test"
                << " returned NULL!"
                << std::endl;
      return 1;
    }
    memset(p_dest_array, 0xF, SIZE_BYTES * sizeof(char));

    // time only the memcpy FROM p_big_array TO p_dest_array
    Timer timer;

    memcpy(p_dest_array, p_big_array, SIZE_BYTES * sizeof(char));

    double elapsed_ms = timer.elapsedMs();
    std::cout << "memcpy for " << formatBytes(SIZE_BYTES) << " took "
              << elapsed_ms << "ms "
              << "(" << formatBytes(SIZE_BYTES / (elapsed_ms / 1.0e3)) << " bytes/sec)"
              << std::endl;

    // cleanup p_dest_array
    free(p_dest_array);
    p_dest_array = NULL;
  }

  /////////////
  // memmove
  {
    char* p_dest_array = (char*)malloc(SIZE_BYTES);
    if (p_dest_array == NULL)
    {
      std::cerr << "ERROR: malloc of " << SIZE_BYTES << " for memmove test"
                << " returned NULL!"
                << std::endl;
      return 1;
    }
    memset(p_dest_array, 0xF, SIZE_BYTES * sizeof(char));

    // time only the memmove FROM p_big_array TO p_dest_array
    Timer timer;

    // memmove(p_dest_array, p_big_array, SIZE_BYTES * sizeof(char));
    doMemmove(p_dest_array, p_big_array, SIZE_BYTES * sizeof(char));

    double elapsed_ms = timer.elapsedMs();
    std::cout << "memmove for " << formatBytes(SIZE_BYTES) << " took "
              << elapsed_ms << "ms "
              << "(" << formatBytes(SIZE_BYTES / (elapsed_ms / 1.0e3)) << " bytes/sec)"
              << std::endl;

    // cleanup p_dest_array
    free(p_dest_array);
    p_dest_array = NULL;
  }


  // cleanup
  free(p_big_array);
  p_big_array = NULL;

  return 0;
}

project(big_memcpy_test)
cmake_minimum_required(VERSION 2.4.0)

include_directories(${CMAKE_CURRENT_SOURCE_DIR})

# create verbose makefiles that show each command line as it is issued
set( CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "Verbose" FORCE )
# release mode
set( CMAKE_BUILD_TYPE Release )
# grab in CXXFLAGS environment variable and append C++11 and -Wall options
set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x -Wall -march=native -mtune=native" )
message( INFO "CMAKE_CXX_FLAGS = ${CMAKE_CXX_FLAGS}" )

# sources to build
set(big_memcpy_test_SRCS
  main.cpp
)

# create an executable file named "big_memcpy_test" from
# the source files in the variable "big_memcpy_test_SRCS".
add_executable(big_memcpy_test ${big_memcpy_test_SRCS})

Buffer Size: 1GB | malloc (ms) | memset (ms) | memcpy (ms) | NUMA nodes (numactl --hardware)
---------------------------------------------------------------------------------------------
Laptop 1         | 0           | 127         | 113         | 1
Laptop 2         | 0           | 180         | 120         | 1
Server 1         | 0           | 306         | 301         | 2
Server 2         | 0           | 352         | 325         | 2

$ numactl --hardware                                                            
available: 2 nodes (0-1)                                                                     
node 0 cpus: 0 1 2 3 4 5 6 7 16 17 18 19 20 21 22 23                                         
node 0 size: 65501 MB                                                                        
node 0 free: 62608 MB                                                                        
node 1 cpus: 8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31                                   
node 1 size: 65536 MB                                                                        
node 1 free: 63837 MB                                                                        
node distances:                                                                              
node   0   1                                                                                 
  0:  10  21                                                                                 
  1:  21  10 

$ numactl --hardware
available: 1 nodes (0)
node 0 cpus: 0 1 2 3 4 5 6 7
node 0 size: 16018 MB
node 0 free: 6622 MB
node distances:
node   0 
  0:  10

$ numactl --cpunodebind=0 --membind=0 ./big_memcpy_test

g++ -std=c++0x -Wall -march=native -mtune=native -O3 -DNDEBUG -o big_memcpy_test main.cpp 

g++ -std=c++0x -Wall -march=native -mtune=native -O2 -DNDEBUG -o big_memcpy_test main.cpp

Buffer Size: 1GB | malloc (ms) | memset (ms) | memcpy (ms) | memmove() | NUMA nodes (numactl --hardware)
---------------------------------------------------------------------------------------------------------
Laptop 1         | 0           | 127         | 113         | 161       | 1
Laptop 2         | 0           | 180         | 120         | 160       | 1
Server 1         | 0           | 306         | 301         | 159       | 2
Server 2         | 0           | 352         | 325         | 159       | 2

void naiveMemcpy(void* pDest, const void* pSource, std::size_t sizeBytes)
{
  char* p_dest = (char*)pDest;
  const char* p_source = (const char*)pSource;
  for (std::size_t i = 0; i < sizeBytes; ++i)
  {
    *p_dest++ = *p_source++;
  }
}

Buffer Size: 1GB | memcpy (ms) | memmove(ms) | naiveMemcpy()
------------------------------------------------------------
Laptop 1         | 113         | 161         | 160
Server 1         | 301         | 159         | 159
Server 2         | 325         | 159         | 159

#include <cstring>
#include <cstdlib>

int main(int argc, char* argv[])
{
  size_t SIZE_BYTES = 1073741824; // 1GB

  char* p_big_array  = (char*)malloc(SIZE_BYTES * sizeof(char));
  char* p_dest_array = (char*)malloc(SIZE_BYTES * sizeof(char));

  memset(p_big_array,  0xA, SIZE_BYTES * sizeof(char));
  memset(p_dest_array, 0xF, SIZE_BYTES * sizeof(char));

  memcpy(p_dest_array, p_big_array, SIZE_BYTES * sizeof(char));

  free(p_dest_array);
  free(p_big_array);

  return 0;
}

        .file   "main_memcpy.cpp"
        .section        .text.startup,"ax",@progbits
        .p2align 4,,15
        .globl  main
        .type   main, @function
main:
.LFB25:
        .cfi_startproc
        pushq   %rbp
        .cfi_def_cfa_offset 16
        .cfi_offset 6, -16
        movl    $1073741824, %edi
        pushq   %rbx
        .cfi_def_cfa_offset 24
        .cfi_offset 3, -24
        subq    $8, %rsp
        .cfi_def_cfa_offset 32
        call    malloc
        movl    $1073741824, %edi
        movq    %rax, %rbx
        call    malloc
        movl    $1073741824, %edx
        movq    %rax, %rbp
        movl    $10, %esi
        movq    %rbx, %rdi
        call    memset
        movl    $1073741824, %edx
        movl    $15, %esi
        movq    %rbp, %rdi
        call    memset
        movl    $1073741824, %edx
        movq    %rbx, %rsi
        movq    %rbp, %rdi
        call    memcpy
        movq    %rbp, %rdi
        call    free
        movq    %rbx, %rdi
        call    free
        addq    $8, %rsp
        .cfi_def_cfa_offset 24
        xorl    %eax, %eax
        popq    %rbx
        .cfi_def_cfa_offset 16
        popq    %rbp
        .cfi_def_cfa_offset 8
        ret
        .cfi_endproc
.LFE25:
        .size   main, .-main
        .ident  "GCC: (GNU) 4.6.1"
        .section        .note.GNU-stack,"",@progbits

Buffer Size: 1GB  | memcpy (ms) | memmove(ms) | naiveMemcpy()
------------------------------------------------------------
Laptop            | 136         | 132         | 161
Laptop SetCache   | 182         | 137         | 161
Server 1          | 305         | 302         | 164
Server 1 SetCache | 162         | 303         | 164
Server 2          | 300         | 299         | 166
Server 2 SetCache | 166         | 301         | 166

最佳答案

[我会发表评论，但没有足够的声誉这样做。]

我有一个类似的系统并看到类似的结果，但可以添加一些数据点:

关于c++ - Linux 上的 memcpy 性能不佳，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/22793669/