Add in process log rotation (#5778)

cmd/telegraf/telegraf.go

@@ -115,7 +115,7 @@ func runAgent(ctx context.Context,
 ) error {
  // Setup default logging. This may need to change after reading the config
  // file, but we can configure it to use our logger implementation now.
- logger.SetupLogging(false, false, "")
+ logger.SetupLogging(logger.LogConfig{})
  log.Printf("I! Starting Telegraf %s", version)
 
  // If no other options are specified, load the config file and run.
@@ -156,11 +156,16 @@ func runAgent(ctx context.Context,
  }
 
  // Setup logging as configured.
- logger.SetupLogging(
- ag.Config.Agent.Debug || *fDebug,
- ag.Config.Agent.Quiet || *fQuiet,
- ag.Config.Agent.Logfile,
- )
+ logConfig := logger.LogConfig{
+ Debug: ag.Config.Agent.Debug || *fDebug,
+ Quiet: ag.Config.Agent.Quiet || *fQuiet,
+ Logfile: ag.Config.Agent.Logfile,
+ RotationInterval: ag.Config.Agent.LogfileRotationInterval,
+ RotationMaxSize: ag.Config.Agent.LogfileRotationMaxSize,
+ RotationMaxArchives: ag.Config.Agent.LogfileRotationMaxArchives,
+ }
+
+ logger.SetupLogging(logConfig)
 
  if *fTest {
  return ag.Test(ctx)

docs/CONFIGURATION.md

@@ -141,6 +141,14 @@ The agent table configures Telegraf and the defaults used across all plugins.
  Run telegraf in quiet mode (error log messages only).
 - **logfile**:
  Specify the log file name. The empty string means to log to stderr.
+- **logfile_rotation_interval**:
+ Log file rotation time [interval][], e.g. "1d" means logs will rotated every day. Default is 0 => no rotation based on time.
+- **logfile_rotation_max_size**:
+ The log file max [size][]. Log files will be rotated when they exceed this size. Default is 0 => no rotation based on file size.
+- **logfile_rotation_max_archives**:
+ Maximum number of archives (rotated) files to keep. Older log files are deleted first.
+ This setting is only applicable if `logfile_rotation_interval` and/or `logfile_rotation_max_size` settings have been specified (otherwise there is no rotation)
+ Default is 0 => all rotated files are deleted. Use -1 to keep all archives.
 
 - **hostname**:
  Override default hostname, if empty use os.Hostname()

internal/config/config.go

@@ -145,6 +145,15 @@ type AgentConfig struct {
  // Logfile specifies the file to send logs to
  Logfile string
 
+ // The log file rotation interval
+ LogfileRotationInterval internal.Duration
+
+ // The log file max size. Logs will rotated when they exceed this size.
+ LogfileRotationMaxSize internal.Size
+
+ // The max number of log archives to keep
+ LogfileRotationMaxArchives int
+
  // Quiet is the option for running in quiet mode
  Quiet bool
  Hostname string
@@ -273,6 +282,17 @@ var agentConfig = `
  quiet = false
  ## Specify the log file name. The empty string means to log to stderr.
  logfile = ""
+ ## Rotation settings, only applicable when log file name is specified.
+ ## Log file rotation time interval, e.g. "1d" means logs will rotated every day. Default is 0 => no rotation based on time.
+ # logfile_rotation_interval = "1d"
+ ## The log file max size. Log files will be rotated when they exceed this size. Default is 0 => no rotation based on file size.
+ # logfile_rotation_max_size = "10 MB"
+ ## Maximum number of archives (rotated) files to keep. Older log files are deleted first.
+ ## This setting is only applicable if logfile_rotation_interval and/or logfile_rotation_max_size settings have been specified (otherwise there is no rotation)
+ ## Default is 0 => all rotated files are deleted. 
+ ## Use -1 to keep all archives.
+ ## Analogous to logrotate "rotate" setting http://man7.org/linux/man-pages/man8/logrotate.8.html
+ # logfile_rotation_max_archives = 0
 
  ## Override default hostname, if empty use os.Hostname()
  hostname = ""

internal/rotate/file_writer.go

@@ -0,0 +1,183 @@
+package rotate
+
+// Rotating things
+import (
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+)
+
+// FilePerm defines the permissions that Writer will use for all
+// the files it creates.
+const (
+ FilePerm = os.FileMode(0644)
+ DateFormat = "2006-01-02"
+)
+
+// FileWriter implements the io.Writer interface and writes to the
+// filename specified.
+// Will rotate at the specified interval and/or when the current file size exceeds maxSizeInBytes
+// At rotation time, current file is renamed and a new file is created.
+// If the number of archives exceeds maxArchives, older files are deleted.
+type FileWriter struct {
+ filename string
+ filenameRotationTemplate string
+ current *os.File
+ interval time.Duration
+ maxSizeInBytes int64
+ maxArchives int
+ expireTime time.Time
+ bytesWritten int64
+ sync.Mutex
+}
+
+// NewFileWriter creates a new file writer.
+func NewFileWriter(filename string, interval time.Duration, maxSizeInBytes int64, maxArchives int) (io.WriteCloser, error) {
+ if interval == 0 && maxSizeInBytes <= 0 {
+ // No rotation needed so a basic io.Writer will do the trick
+ return openFile(filename)
+ }
+
+ w := &FileWriter{
+ filename: filename,
+ interval: interval,
+ maxSizeInBytes: maxSizeInBytes,
+ maxArchives: maxArchives,
+ filenameRotationTemplate: getFilenameRotationTemplate(filename),
+ }
+
+ if err := w.openCurrent(); err != nil {
+ return nil, err
+ }
+
+ return w, nil
+}
+
+func openFile(filename string) (*os.File, error) {
+ return os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, FilePerm)
+}
+
+func getFilenameRotationTemplate(filename string) string {
+ // Extract the file extension
+ fileExt := filepath.Ext(filename)
+ // Remove the file extension from the filename (if any)
+ stem := strings.TrimSuffix(filename, fileExt)
+ return stem + ".%s-%s" + fileExt
+}
+
+// Write writes p to the current file, then checks to see if
+// rotation is necessary.
+func (w *FileWriter) Write(p []byte) (n int, err error) {
+ w.Lock()
+ defer w.Unlock()
+ if n, err = w.current.Write(p); err != nil {
+ return 0, err
+ }
+ w.bytesWritten += int64(n)
+
+ if err = w.rotateIfNeeded(); err != nil {
+ return 0, err
+ }
+
+ return n, nil
+}
+
+// Close closes the current file. Writer is unusable after this
+// is called.
+func (w *FileWriter) Close() (err error) {
+ w.Lock()
+ defer w.Unlock()
+
+ // Rotate before closing
+ if err = w.rotate(); err != nil {
+ return err
+ }
+
+ if err = w.current.Close(); err != nil {
+ return err
+ }
+ w.current = nil
+ return nil
+}
+
+func (w *FileWriter) openCurrent() (err error) {
+ // In case ModTime() fails, we use time.Now()
+ w.expireTime = time.Now().Add(w.interval)
+ w.bytesWritten = 0
+ w.current, err = openFile(w.filename)
+
+ if err != nil {
+ return err
+ }
+
+ // Goal here is to rotate old pre-existing files.
+ // For that we use fileInfo.ModTime, instead of time.Now().
+ // Example: telegraf is restarted every 23 hours and
+ // the rotation interval is set to 24 hours.
+ // With time.now() as a reference we'd never rotate the file.
+ if fileInfo, err := w.current.Stat(); err == nil {
+ w.expireTime = fileInfo.ModTime().Add(w.interval)
+ }
+ return nil
+}
+
+func (w *FileWriter) rotateIfNeeded() error {
+ if (w.interval > 0 && time.Now().After(w.expireTime)) ||
+ (w.maxSizeInBytes > 0 && w.bytesWritten >= w.maxSizeInBytes) {
+ if err := w.rotate(); err != nil {
+ //Ignore rotation errors and keep the log open
+ fmt.Printf("unable to rotate the file '%s', %s", w.filename, err.Error())
+ }
+ return w.openCurrent()
+ }
+ return nil
+}
+
+func (w *FileWriter) rotate() (err error) {
+ if err = w.current.Close(); err != nil {
+ return err
+ }
+
+ // Use year-month-date for readability, unix time to make the file name unique with second precision
+ now := time.Now()
+ rotatedFilename := fmt.Sprintf(w.filenameRotationTemplate, now.Format(DateFormat), strconv.FormatInt(now.Unix(), 10))
+ if err = os.Rename(w.filename, rotatedFilename); err != nil {
+ return err
+ }
+
+ if err = w.purgeArchivesIfNeeded(); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (w *FileWriter) purgeArchivesIfNeeded() (err error) {
+ if w.maxArchives == -1 {
+ //Skip archiving
+ return nil
+ }
+
+ var matches []string
+ if matches, err = filepath.Glob(fmt.Sprintf(w.filenameRotationTemplate, "*", "*")); err != nil {
+ return err
+ }
+
+ //if there are more archives than the configured maximum, then purge older files
+ if len(matches) > w.maxArchives {
+ //sort files alphanumerically to delete older files first
+ sort.Strings(matches)
+ for _, filename := range matches[:len(matches)-w.maxArchives] {
+ if err = os.Remove(filename); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}

internal/rotate/file_writer_test.go

@@ -0,0 +1,115 @@
+package rotate
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestFileWriter_NoRotation(t *testing.T) {
+ tempDir, err := ioutil.TempDir("", "RotationNo")
+ require.NoError(t, err)
+ writer, err := NewFileWriter(filepath.Join(tempDir, "test"), 0, 0, 0)
+ require.NoError(t, err)
+ defer func() { writer.Close(); os.RemoveAll(tempDir) }()
+
+ _, err = writer.Write([]byte("Hello World"))
+ require.NoError(t, err)
+ _, err = writer.Write([]byte("Hello World 2"))
+ require.NoError(t, err)
+ files, _ := ioutil.ReadDir(tempDir)
+ assert.Equal(t, 1, len(files))
+}
+
+func TestFileWriter_TimeRotation(t *testing.T) {
+ tempDir, err := ioutil.TempDir("", "RotationTime")
+ require.NoError(t, err)
+ interval, _ := time.ParseDuration("1s")
+ writer, err := NewFileWriter(filepath.Join(tempDir, "test"), interval, 0, -1)
+ require.NoError(t, err)
+ defer func() { writer.Close(); os.RemoveAll(tempDir) }()
+
+ _, err = writer.Write([]byte("Hello World"))
+ require.NoError(t, err)
+ time.Sleep(1 * time.Second)
+ _, err = writer.Write([]byte("Hello World 2"))
+ require.NoError(t, err)
+ files, _ := ioutil.ReadDir(tempDir)
+ assert.Equal(t, 2, len(files))
+}
+
+func TestFileWriter_SizeRotation(t *testing.T) {
+ tempDir, err := ioutil.TempDir("", "RotationSize")
+ require.NoError(t, err)
+ maxSize := int64(9)
+ writer, err := NewFileWriter(filepath.Join(tempDir, "test.log"), 0, maxSize, -1)
+ require.NoError(t, err)
+ defer func() { writer.Close(); os.RemoveAll(tempDir) }()
+
+ _, err = writer.Write([]byte("Hello World"))
+ require.NoError(t, err)
+ _, err = writer.Write([]byte("World 2"))
+ require.NoError(t, err)
+ files, _ := ioutil.ReadDir(tempDir)
+ assert.Equal(t, 2, len(files))
+}
+
+func TestFileWriter_DeleteArchives(t *testing.T) {
+ tempDir, err := ioutil.TempDir("", "RotationDeleteArchives")
+ require.NoError(t, err)
+ maxSize := int64(5)
+ writer, err := NewFileWriter(filepath.Join(tempDir, "test.log"), 0, maxSize, 2)
+ require.NoError(t, err)
+ defer func() { writer.Close(); os.RemoveAll(tempDir) }()
+
+ _, err = writer.Write([]byte("First file"))
+ require.NoError(t, err)
+ // File names include the date with second precision
+ // So, to force rotation with different file names
+ // we need to wait
+ time.Sleep(1 * time.Second)
+ _, err = writer.Write([]byte("Second file"))
+ require.NoError(t, err)
+ time.Sleep(1 * time.Second)
+ _, err = writer.Write([]byte("Third file"))
+ require.NoError(t, err)
+
+ files, _ := ioutil.ReadDir(tempDir)
+ assert.Equal(t, 3, len(files))
+
+ for _, tempFile := range files {
+ var bytes []byte
+ var err error
+ path := filepath.Join(tempDir, tempFile.Name())
+ if bytes, err = ioutil.ReadFile(path); err != nil {
+ t.Error(err.Error())
+ return
+ }
+ contents := string(bytes)
+
+ if contents != "" && contents != "Second file" && contents != "Third file" {
+ t.Error("Should have deleted the eldest log file")
+ return
+ }
+ }
+}
+
+func TestFileWriter_CloseRotates(t *testing.T) {
+ tempDir, err := ioutil.TempDir("", "RotationClose")
+ require.NoError(t, err)
+ defer os.RemoveAll(tempDir)
+ maxSize := int64(9)
+ writer, err := NewFileWriter(filepath.Join(tempDir, "test.log"), 0, maxSize, -1)
+ require.NoError(t, err)
+
+ writer.Close()
+
+ files, _ := ioutil.ReadDir(tempDir)
+ assert.Equal(t, 1, len(files))
+ assert.Regexp(t, "^test\\.[^\\.]+\\.log$", files[0].Name())
+}