我在golang中设计并实现了文件的文件轮换。 根据设计，我根据 filesize >= FileSizeThreshold(50000bytes) 或 file duration >= FileDurationThreshold(1 minute)(以先到者为准)旋转文件。

下面是golang中的实现。

package main

import (
    "os"
    "path/filepath"
    "time"
    "log"
    "strings"
    "flag"
    "os/exec"
)
type FileStruct struct{
    Filename string
    CreatedAt time.Time
}
type FileRotate struct {
    Dir string
    File chan FileStruct
}

const(
    MAX_FILE_SIZE  = 50000
    MAX_FILE_DURATION = time.Minute * 1
    filename_time_format = "20060102150405000"
    MAX_TRY = 5
)
var blockingChan chan int
func main(){
    path := flag.String("dir", "", "absolute path of dir ")
    flag.Parse()
    if strings.Contains(*path, "./") {
        log.Fatalln("ERROR: please give absolute path")
    }
    if info, err := os.Stat(*path); err == nil{
        if ! info.IsDir(){
            log.Fatalln(*path," is not a directory")
        }
        log.Println("directory found..")
    } else {
        if os.IsNotExist(err){
            log.Println("directory not found..")
            log.Println("creating the directory..",*path)
            if err := exec.Command("mkdir","-p",*path).Run(); err != nil{
                log.Fatalln("failed to create the directory ERROR:",err)
            }
            log.Println("directory created successfully")
        }
    }
    filerotate := &FileRotate{*path,make(chan FileStruct,1)}
    go filerotate.FileOperationsRoutine()

    log.Println("generating file name struct..")
    filerotate.File <- GetFileStruct()
    <- blockingChan
}

func (rotate *FileRotate) FileOperationsRoutine(){
    try := 0
    var f *os.File
    for{
        if file, ok := <- rotate.File; ok{
            if f == nil {
                log.Println("WARN: file ptr is nil")
            }
            filePath := filepath.Join(rotate.Dir, file.Filename)
            fileInfo, err := os.Stat(filePath)
            if err != nil && os.IsNotExist(err) {
                log.Println("file:", filePath, " does not exist...creating file")
                _, err = os.Create(filePath)
                if err != nil {
                    log.Println("failed to create the file ERROR:",err)
                    try++
                    if try == MAX_TRY {
                        log.Println("tried creating the file ",MAX_TRY," times. No luck")
                        time.Sleep(time.Second * 3)
                        continue
                    }
                    rotate.File <- file
                    continue
                }
                log.Println("file:", filePath, " created successfully")
                fileInfo,err = os.Stat(filePath)
            }
            sizeCheck := fileInfo.Size() >= MAX_FILE_SIZE
            durationCheck := time.Now().After(file.CreatedAt.Add(MAX_FILE_DURATION))
            if sizeCheck || durationCheck {
                log.Println("filesize of ",filePath," is ",fileInfo.Size(),"..filesizeCheck=",sizeCheck)
                log.Println("fileDurationCheck=",durationCheck)
                log.Println("rotating the file..")
                f.Close()
                f = nil
                go ZipAndSendRoutine(filePath)
                rotate.File <- GetFileStruct()
            }else{
                if f == nil {
                    f, err = os.OpenFile(filePath, os.O_RDWR | os.O_APPEND, 0644)
                    if err != nil {
                        log.Println("failed to open the file ERROR:", err)
                        try++
                        if try == MAX_TRY {
                            log.Println("tried opening the file ", MAX_TRY, " times. No luck")
                            time.Sleep(time.Second * 3)
                            continue
                        }
                        rotate.File <- file
                        continue
                    }
                    log.Println("file opened in append mode")
                }
                rotate.File <- file
            }
        }
    }
}

func GetFileStruct() FileStruct{
    current_time := time.Now()
    log.Println("returning the filestruct..")
    return FileStruct{"example_" + current_time.Format(filename_time_format),current_time}
}

func ZipAndSendRoutine(file string){
    log.Println("zipping and sending the file:",file,"to remote server")
}

执行日志:

root@workstation:/media/sf_golang# ./bin/file_rotation -dir "/tmp/file_rotaion"
2017/01/16 15:05:03 directory found..
2017/01/16 15:05:03 starting file operations routine...
2017/01/16 15:05:03 generating file name struct..
2017/01/16 15:05:03 returning the filestruct..
2017/01/16 15:05:03 WARN: file ptr is nil
2017/01/16 15:05:03 file: /tmp/file_rotaion/example_20170116150503000  does not exist...creating file
2017/01/16 15:05:03 file: /tmp/file_rotaion/example_20170116150503000  created successfully
2017/01/16 15:05:03 file opened in append mode
2017/01/16 15:06:03 filesize of  /tmp/file_rotaion/example_20170116150503000  is  0 ..filesizeCheck= false ...fileDurationCheck= true
2017/01/16 15:06:03 rotating the file..
2017/01/16 15:06:03 returning the filestruct..
2017/01/16 15:06:03 WARN: file ptr is nil
2017/01/16 15:06:03 file: /tmp/file_rotaion/example_20170116150603000  does not exist...creating file
2017/01/16 15:06:03 file: /tmp/file_rotaion/example_20170116150603000  created successfully
2017/01/16 15:06:03 file opened in append mode
2017/01/16 15:06:03 zipping and sending the file: /tmp/file_rotaion/example_20170116150503000 to remote server
2017/01/16 15:07:03 filesize of  /tmp/file_rotaion/example_20170116150603000  is  0 ..filesizeCheck= false ...fileDurationCheck= true
2017/01/16 15:07:03 rotating the file..
2017/01/16 15:07:03 returning the filestruct..
2017/01/16 15:07:03 WARN: file ptr is nil
2017/01/16 15:07:03 file: /tmp/file_rotaion/example_20170116150703000  does not exist...creating file
2017/01/16 15:07:03 file: /tmp/file_rotaion/example_20170116150703000  created successfully
2017/01/16 15:07:03 file opened in append mode
2017/01/16 15:07:03 zipping and sending the file: /tmp/file_rotaion/example_20170116150603000 to remote server

从日志中可以看出，该实用程序按预期工作。 但是在这个实用程序的执行过程中，CPU 使用率几乎是 100%

停止实用程序后.. 我已经确定了造成这种情况的原因: FileOperations goroutine 无限期运行，在此例程中，我将文件指针发送到 rotate.File channel

我卡在了这一点上，不确定如何进一步优化它。 谁能告诉我应该如何优化此实用程序的 CPU 利用率？

最佳答案

您的代码的主要问题是在您传递 FileStruct 时一直在 for 循环中到 channel old或 new .因此， channel 接收数据没有等待时间，并且在 if 循环内，您正在对文件进行统计以获取其数据，这大部分是您必须已经完成的

这是你程序上的 strace

% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 93.58   11.835475        3793      3120       227 futex
  6.38    0.807279           4    192048         1 stat
  0.03    0.003284           9       366           sched_yield
  0.01    0.000759           7       114           rt_sigaction
  0.00    0.000271          90         3           openat
  0.00    0.000197          10        19           mmap
  0.00    0.000143          20         7           write
  0.00    0.000071          24         3           clone
  0.00    0.000064           8         8           rt_sigprocmask
  0.00    0.000034          17         2           select
  0.00    0.000021          11         2           read
  0.00    0.000016          16         1           sched_getaffinity
  0.00    0.000014          14         1           munmap
  0.00    0.000014          14         1           execve
  0.00    0.000013          13         1           arch_prctl
  0.00    0.000011          11         1           close
  0.00    0.000000           0         2           sigaltstack
  0.00    0.000000           0         1           gettid
------ ----------- ----------- --------- --------- ----------------
100.00   12.647666                195700       228 total

这里在大约 40 秒内有 195k 个系统调用

你可以做的是在 for 之后添加一个等待时间

for { 
      <- time.After(time.Second)
       if file, ok := <- rotate.File; ok{

您可以添加 fileinfo在FileStruct在每次循环中，您可以先在结构中检查它，然后只执行 stat

这是添加<- time.After(time.Second) 后的strace

% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 65.65    0.001512          35        43         1 futex
 23.71    0.000546           5       114           rt_sigaction
  3.04    0.000070           9         8           mmap
  2.43    0.000056          19         3           clone
  2.26    0.000052           7         8           rt_sigprocmask
  0.56    0.000013           7         2           stat
  0.48    0.000011          11         1           munmap
  0.48    0.000011           6         2           sigaltstack
  0.43    0.000010          10         1           execve
  0.39    0.000009           9         1           sched_getaffinity
  0.35    0.000008           8         1           arch_prctl
  0.22    0.000005           5         1           gettid
  0.00    0.000000           0         2           read
  0.00    0.000000           0         3           write
  0.00    0.000000           0         1           close
  0.00    0.000000           0         1           openat
------ ----------- ----------- --------- --------- ----------------
100.00    0.002303                   192         1 total

结论

对于没有 time.After() 的相同持续时间代码进行了 195K 次系统调用，其中带有 time.After(time.Second) 的系统调用只进行了 192 次系统调用。您可以通过添加已获取的文件信息作为 FileStruct 的一部分来进一步改进它

关于go - 我应该如何优化 golang 中的文件轮换？，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/41673576/