Puppet Class: simp_logstash::clean

Defined in:
manifests/clean.pp

Overview

Create a cron job to clean your ElasticSearch indices on a regular basis using elasticsearch-curator.

Parameters:

  • ensure (String) (defaults to: 'present') —

    Whether to add, or delete, the index job. Allowed Values: present, absent

  • host (Hostname) (defaults to: '127.0.0.1') —

    The host upon which to operate. Ideally, you will position this job locally but it will work over thet network just as well provided your access controls allow it.

  • keep_days (Integer) (defaults to: '356') —

    The number of days to keep within ElasticSearch. Mutually exclusive with keep_hours and keep_space.

  • keep_hours (Integer) (defaults to: '') —

    The number of hours to keep within ElasticSearch. Mutually exclusive with keep_days and keep_space.

  • keep_space (Integer) (defaults to: '') —

    The number of Gigabytes to keep within ElasticSearch. This applies to each index individually, not the entire storage space used by the prefix. Mutually exclusive with keep_days and keep_hours.

  • prefix (String) (defaults to: 'logstash-') —

    The prefix to use to identify relevant logs. This is a match so ‘foo’ will match ‘foo’, ‘foosball’, and ‘foot’.

  • port (Port) (defaults to: '9199') —

    The port to which to connect. Since this is SIMP tailored, we use our local unencrypted default.

  • separator (Char) (defaults to: '.') —

    The index separator.

  • es_timeout (Integer) (defaults to: '30') —

    The timeout, in seconds, to wait for a response from Elasticsearch.

  • log_file (Absolute Path) (defaults to: '/var/log/logstash/curator_clean.log') —

    The log file to which to print curator output.

  • cron_hour (Integer or '*') (defaults to: '1') —

    The hour at which to run the index cleanup.

  • cron_minute (Integer or '*') (defaults to: '15') —

    The minute at which to run the index cleanup.

  • cron_month (Integer or '*') (defaults to: '*') —

    The month within which to run the index cleanup.

  • cron_monthday (Integer or '*') (defaults to: '*') —

    The day of the month upon which to run the index cleanup.

  • cron_weekday (Integer or '*') (defaults to: '*') —

    The day of the week upon which to run the index cleanup.

Author:

  • Trevor Vaughan <tvaughan@onyxpoint.com>



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
 
# File 'manifests/clean.pp', line 53

class simp_logstash::clean (
  $ensure = 'present',
  $host = '127.0.0.1',
  $keep_days = '356',
  $keep_hours = '',
  $keep_space = '',
  $prefix = 'logstash-',
  $port = '9199',
  $separator = '.',
  $es_timeout = '30',
  $log_file = '/var/log/logstash/curator_clean.log',
  $cron_hour = '1',
  $cron_minute = '15',
  $cron_month = '*',
  $cron_monthday = '*',
  $cron_weekday = '*'
) {
  validate_array_member($ensure,['present','absent'])

  if defined('$::simp_logstash::auto_clean') and getvar('::simp_logstash::auto_clean') {
    $_simp_ls_auto_clean = true
  }
  else {
    $_simp_ls_auto_clean = false
  }

  if ($ensure == 'present') and $_simp_ls_auto_clean {
    validate_string($prefix)
    validate_port($port)
    validate_string($separator)
    validate_integer($es_timeout)
    validate_absolute_path($log_file)
    if ($cron_hour != '*') { validate_integer($cron_hour) }
    if ($cron_minute != '*') { validate_integer($cron_minute) }
    if ($cron_month != '*') { validate_integer($cron_month) }
    if ($cron_monthday != '*') { validate_integer($cron_monthday) }
    if ($cron_weekday != '*') { validate_integer($cron_weekday) }

    if size(reject([$keep_days, $keep_hours, $keep_space],'^\s*$')) > 1 {
      fail('You may only specify one of $keep_days, $keep_hours, or $keep_space')
    }

    if !empty($keep_hours) {
      validate_integer($keep_hours)
      $_limit = "-T hours --older-than ${keep_hours}"
    }
    elsif ! empty($keep_days) {
      validate_integer($keep_days)
      $_limit = "-T days --older-than ${keep_days}"
    }
    elsif ! empty($keep_space) {
      validate_integer($keep_space)
      $_limit = "--disk-space ${keep_space}"
    }
    else {
      fail('You must specify one of $keep_days, $keep_hours, or $keep_space')
    }

    cron { 'logstash_index_cleanup' :
      ensure   => $ensure,
      command  => "/usr/bin/curator --host ${host} --port ${port} -t ${es_timeout} delete -p '${prefix}' -s '${separator}' ${_limit} >> ${log_file} 2>&1",
      hour     => $cron_hour,
      minute   => $cron_minute,
      month    => $cron_month,
      monthday => $cron_monthday,
      weekday  => $cron_weekday,
      require  => Class['simp_logstash::curator']
    }
  }
  else {
    cron { 'logstash_index_cleanup' : ensure => 'absent' }
  }
}