diff --git a/.gitignore b/.gitignore index 6362a7b..aa73350 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ vendor/ .kitchen/ .kitchen.local.yml .bundle/ +bin/ \ No newline at end of file diff --git a/.kitchen.yml b/.kitchen.yml index d850083..0085af5 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -13,6 +13,7 @@ provisioner: platforms: - name: ubuntu-12.04 + - name: centos-6.5 suites: - name: server @@ -20,46 +21,22 @@ suites: - recipe[apt::default] - recipe[java::default] - recipe[logstash::server] - - recipe[logstash::agent] - - recipe[kibana::default] attributes: java: - jdk_version: 7 - oracle: + install_flavor: "oracle" + jdk_version: "7" + oracle: accept_oracle_download_terms: true - install_flavor: oracle logstash: - supervisor_gid: 'adm' - agent: - server_ipaddress: 127.0.0.1 - xms: 128m - xmx: 128m - enable_embedded_es: false - inputs: - - file: - type: syslog - path: - - /var/log/syslog - - /var/log/messages - start_position: beginning - filters: - - condition: 'if [type] == "syslog"' - block: - grok: - match: - - 'message' - - '%{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:host} (?:%{PROG:program}(?:\[%{POSINT:pid}\])?: )?%{GREEDYDATA:message}' - date: - match: - - 'timestamp' - - 'MMM d HH:mm:ss' - - 'MMM dd HH:mm:ss' - - 'ISO8601' - server: - enable_embedded_es: true - web: - enabled: true - kibana: - webserver_listen: '0.0.0.0' - webserver: 'nginx' - install_type: 'file' + instance: + server: + xms: '256M' + xmx: '256M' + config_templates: + input_syslog: 'config/input_syslog.conf.erb' + output_stdout: 'config/output_stdout.conf.erb' + output_elasticsearch: 'config/output_elasticsearch.conf.erb' + pattern_templates: + default: 'patterns/custom_patterns.erb' + elasticsearch_ip: '127.0.0.1' + enable_embedded_es: true \ No newline at end of file diff --git a/.rubocop.yml b/.rubocop.yml index 33c419c..874ad9b 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,12 +1,13 @@ AllCops: - Excludes: - - vendor - - test/support/** - Includes: - - Rakefile - - Gemfile - - Vagrantfile - - Berksfile + Exclude: + - 'vendor/**/*' + - 'test/support/**' + - 'bin/**/*' + Include: + - '**/Rakefile' + - '**/Gemfile' + - '**/Vagrantfile' + - '**/Berksfile' CyclomaticComplexity: Enabled: false @@ -17,3 +18,24 @@ LineLength: MethodLength: Enabled: true Max: 20 + +AccessorMethodName: + Enabled: false + +ClassAndModuleChildren: + Enabled: false + +UselessAssignment: + Enabled: false + +BlockNesting: + Max: 4 + +Documentation: + Enabled: false + +MethodLength: + Max: 50 + +SingleSpaceBeforeFirstArg: + Enabled: false \ No newline at end of file diff --git a/.tailor b/.tailor deleted file mode 100644 index f54fac6..0000000 --- a/.tailor +++ /dev/null @@ -1,25 +0,0 @@ -Tailor.config do |config| - config.formatters "text" - config.file_set '**/*.rb' do |style| - style.max_line_length 80, level: :off - style.allow_camel_case_methods false, level: :error - style.allow_hard_tabs false, level: :error - style.allow_screaming_snake_case_classes false, level: :error - style.allow_trailing_line_spaces false, level: :error - style.allow_invalid_ruby false, level: :warn - style.indentation_spaces 2, level: :error - style.max_code_lines_in_class 300, level: :error - style.max_code_lines_in_method 30, level: :error - style.spaces_after_comma 1, level: :error - style.spaces_after_lbrace 1, level: :error - style.spaces_after_lbracket 0, level: :error - style.spaces_after_lparen 0, level: :error - style.spaces_before_comma 0, level: :error - style.spaces_before_lbrace 1, level: :error - style.spaces_before_rbrace 1, level: :error - style.spaces_before_rbracket 0, level: :error - style.spaces_before_rparen 0, level: :error - style.spaces_in_empty_braces 0, level: :error - style.trailing_newlines 1, level: :error - end -end \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index e88cf5e..0bb6bdf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,13 @@ language: ruby -gemfile: - - test/support/Gemfile rvm: - 1.9.3 -install: BUNDLE_GEMFILE=$PWD/test/support/Gemfile bundle install -script: BUNDLE_GEMFILE=$PWD/test/support/Gemfile bundle exec rake strainer \ No newline at end of file +env: + - USE_SYSTEM_GECODE=1 +before_install: + - sudo apt-get update + - sudo apt-get install libgecode-dev +install: + - bundle install --path vendor --binstubs +script: + - bin/berks install + - bin/rake style spec diff --git a/Berksfile b/Berksfile index 6544a42..305b85c 100644 --- a/Berksfile +++ b/Berksfile @@ -1,14 +1,17 @@ # Encoding: utf-8 +source 'https://api.berkshelf.com' if Gem::Version.new(Berkshelf::VERSION) > Gem::Version.new('3') + metadata -cookbook 'rabbitmq', git: 'git://github.com/opscode-cookbooks/rabbitmq.git' cookbook 'java' +cookbook 'curl' +cookbook 'ark' + +cookbook 'pleaserun', git: 'https://github.com/paulczar/chef-pleaserun.git' group :test do - cookbook 'minitest-handler', git: 'git://github.com/btm/minitest-handler-cookbook.git' - cookbook 'elasticsearch', git: 'git://github.com/elasticsearch/cookbook-elasticsearch.git' - cookbook 'kibana', git: 'git://github.com/lusis/chef-kibana.git' - cookbook 'curl' - cookbook 'ark' + cookbook 'minitest-handler', git: 'https://github.com/btm/minitest-handler-cookbook.git' + cookbook 'elasticsearch', git: 'https://github.com/elasticsearch/cookbook-elasticsearch.git' + cookbook 'kibana', git: 'https://github.com/lusis/chef-kibana.git' end diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ef4b1e..686801e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,31 @@ This file is used to list changes made in each version of chef-logstash. -## 0.7.7 + +## 0.9.1: + +* curator LWRP + +## 0.9.0: + +_this will almost certainly break backwards compatibility_ + +* support for Logstash 1.4 +* major refactor towards being a library cookbook + * instance LWRP + * service LWRP + * pattern LWP + * config LWP + +## 0.7.7: + * Support for new beaver config [#239](https://github.com/lusis/chef-logstash/pull/239) * Support for multiline codec [#240](https://github.com/lusis/chef-logstash/pull/240) * Parameterize /var/lib/logstash [#242](https://github.com/lusis/chef-logstash/pull/242) * Fix parameter spacing option [#244](https://github.com/lusis/chef-logstash/pull/244) ## 0.7.6: + * introduced more testing * Strainer: rubocop, knife test, foodcritic, chefspec * lots of style fixes for rubocop @@ -16,6 +34,7 @@ This file is used to list changes made in each version of chef-logstash. * testkitchen + server spec ## 0.7.5: + * added fedora systemd support * moved zeromq repos to own recipe diff --git a/Gemfile b/Gemfile index 79c2e71..e0def1b 100644 --- a/Gemfile +++ b/Gemfile @@ -2,4 +2,28 @@ source 'https://rubygems.org' -gem 'berkshelf' +gem 'berkshelf', '> 3' + +# Uncomment these lines if you want to live on the Edge: +# +# group :development do +# gem "berkshelf", github: "berkshelf/berkshelf" +# gem "vagrant", github: "mitchellh/vagrant", tag: "v1.5.2" +# end +# +# group :plugins do +# gem "vagrant-berkshelf", github: "berkshelf/vagrant-berkshelf" +# gem "vagrant-omnibus", github: "schisamo/vagrant-omnibus" +# end + +gem 'chef', '>= 11.8' +gem 'rake', '>= 10.2' +gem 'rubocop', '= 0.23' +gem 'foodcritic', '< 4.0' +gem 'chefspec', '>= 3.4' +gem 'serverspec', '>= 1.6' +gem 'test-kitchen' +gem 'kitchen-vagrant' +gem 'guard', '>= 2.6' +gem 'guard-rubocop', '>= 1.1' +gem 'guard-foodcritic', '>= 1.0.2' diff --git a/Gemfile.lock b/Gemfile.lock index 2bfa194..f6c303a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,127 +1,248 @@ GEM remote: https://rubygems.org/ specs: - activesupport (3.2.15) - i18n (~> 0.6, >= 0.6.4) - multi_json (~> 1.0) - addressable (2.3.5) - akami (1.2.0) - gyoku (>= 0.4.0) - nokogiri (>= 1.4.0) - berkshelf (2.0.10) - activesupport (~> 3.2.0) + addressable (2.3.6) + ast (2.0.0) + berkshelf (3.1.3) addressable (~> 2.3.4) + berkshelf-api-client (~> 1.2) + buff-config (~> 1.0) + buff-extensions (~> 1.0) buff-shell_out (~> 0.1) - chozo (>= 0.6.1) - faraday (>= 0.8.5) - hashie (>= 2.0.2) + celluloid (~> 0.16.0.pre) + celluloid-io (~> 0.16.0.pre) + faraday (~> 0.9.0) minitar (~> 0.5.4) - rbzip2 (~> 0.2.0) + octokit (~> 3.0) retryable (~> 1.3.3) - ridley (~> 1.5.0) - solve (>= 0.5.0) - thor (~> 0.18.0) - buff-config (0.4.0) - buff-extensions (~> 0.3) - varia_model (~> 0.1) - buff-extensions (0.5.0) - buff-ignore (1.1.0) + ridley (~> 4.0) + solve (~> 1.1) + thor (~> 0.18) + berkshelf-api-client (1.2.0) + faraday (~> 0.9.0) + buff-config (1.0.0) + buff-extensions (~> 1.0) + varia_model (~> 0.4) + buff-extensions (1.0.0) + buff-ignore (1.1.1) buff-ruby_engine (0.1.0) buff-shell_out (0.1.1) buff-ruby_engine (~> 0.1.0) - builder (3.2.2) - celluloid (0.14.1) - timers (>= 1.0.0) - celluloid-io (0.14.1) - celluloid (>= 0.14.1) - nio4r (>= 0.4.5) - chozo (0.6.1) - activesupport (>= 3.2.0) - hashie (>= 2.0.2) - multi_json (>= 1.3.0) - erubis (2.7.0) - faraday (0.8.8) - multipart-post (~> 1.2.0) - ffi (1.9.0) - gssapi (1.0.3) - ffi (>= 1.0.1) - gyoku (1.1.0) - builder (>= 2.1.2) - hashie (2.0.5) - httpclient (2.3.4.1) - httpi (0.9.7) + celluloid (0.16.0.pre2) + timers (~> 3.0.0) + celluloid-io (0.16.0.pre2) + celluloid (>= 0.16.0.pre) + nio4r (>= 1.0.0) + chef (11.12.8) + chef-zero (>= 2.0.2, < 2.1) + diff-lcs (~> 1.2, >= 1.2.4) + erubis (~> 2.7) + highline (~> 1.6, >= 1.6.9) + json (>= 1.4.4, <= 1.8.1) + mime-types (~> 1.16) + mixlib-authentication (~> 1.3) + mixlib-cli (~> 1.4) + mixlib-config (~> 2.0) + mixlib-log (~> 1.3) + mixlib-shellout (~> 1.4) + net-ssh (~> 2.6) + net-ssh-multi (~> 1.1) + ohai (~> 7.0.4) + pry (~> 0.9) + rest-client (>= 1.0.4, < 1.7.0) + yajl-ruby (~> 1.1) + chef-zero (2.0.2) + hashie (~> 2.0) + json + mixlib-log (~> 1.3) rack - i18n (0.6.5) - json (1.7.7) - little-plugger (1.1.3) - logging (1.8.1) - little-plugger (>= 1.1.3) - multi_json (>= 1.3.6) - mini_portile (0.5.2) + chefspec (3.4.0) + chef (~> 11.0) + fauxhai (~> 2.0) + rspec (~> 2.14) + coderay (1.1.0) + dep-selector-libgecode (1.0.2) + dep_selector (1.0.3) + dep-selector-libgecode (~> 1.0) + ffi (~> 1.9) + diff-lcs (1.2.5) + erubis (2.7.0) + faraday (0.9.0) + multipart-post (>= 1.2, < 3) + fauxhai (2.1.2) + net-ssh + ohai + ffi (1.9.3) + foodcritic (3.0.3) + erubis + gherkin (~> 2.11.7) + nokogiri (~> 1.5.4) + rake + treetop (~> 1.4.10) + yajl-ruby (~> 1.1.0) + formatador (0.2.5) + gherkin (2.11.8) + multi_json (~> 1.3) + guard (2.6.1) + formatador (>= 0.2.4) + listen (~> 2.7) + lumberjack (~> 1.0) + pry (>= 0.9.12) + thor (>= 0.18.1) + guard-foodcritic (1.0.2) + foodcritic (>= 1.3, < 4.0) + guard (>= 1.0, < 3.0) + guard-rubocop (1.1.0) + guard (~> 2.0) + rubocop (~> 0.20) + hashie (2.1.2) + highline (1.6.21) + hitimes (1.2.1) + ipaddress (0.8.0) + json (1.8.1) + kitchen-vagrant (0.15.0) + test-kitchen (~> 1.0) + listen (2.7.9) + celluloid (>= 0.15.2) + rb-fsevent (>= 0.9.3) + rb-inotify (>= 0.9) + lumberjack (1.0.9) + method_source (0.8.2) + mime-types (1.25.1) minitar (0.5.4) mixlib-authentication (1.3.0) mixlib-log + mixlib-cli (1.5.0) + mixlib-config (2.1.0) mixlib-log (1.6.0) - multi_json (1.8.2) - multipart-post (1.2.0) - net-http-persistent (2.9) - net-ssh (2.7.0) - nio4r (0.5.0) - nokogiri (1.6.0) - mini_portile (~> 0.5.0) - nori (1.1.5) + mixlib-shellout (1.4.0) + multi_json (1.10.1) + multipart-post (2.0.0) + net-http-persistent (2.9.4) + net-scp (1.2.1) + net-ssh (>= 2.6.5) + net-ssh (2.9.1) + net-ssh-gateway (1.2.0) + net-ssh (>= 2.6.5) + net-ssh-multi (1.2.0) + net-ssh (>= 2.6.5) + net-ssh-gateway (>= 1.2.0) + nio4r (1.0.0) + nokogiri (1.5.11) + octokit (3.2.0) + sawyer (~> 0.5.3) + ohai (7.0.4) + ipaddress + mime-types (~> 1.16) + mixlib-cli + mixlib-config (~> 2.0) + mixlib-log + mixlib-shellout (~> 1.2) + systemu (~> 2.5.2) + yajl-ruby + parser (2.1.9) + ast (>= 1.1, < 3.0) + slop (~> 3.4, >= 3.4.5) + polyglot (0.3.5) + powerpack (0.0.9) + pry (0.10.0) + coderay (~> 1.1.0) + method_source (~> 0.8.1) + slop (~> 3.4) rack (1.5.2) - rbzip2 (0.2.0) - retryable (1.3.3) - ridley (1.5.3) + rainbow (2.0.0) + rake (10.3.2) + rb-fsevent (0.9.4) + rb-inotify (0.9.5) + ffi (>= 0.5.0) + rdoc (4.1.1) + json (~> 1.4) + rest-client (1.6.8) + mime-types (~> 1.16) + rdoc (>= 2.4.2) + retryable (1.3.5) + ridley (4.0.0) addressable - buff-config (~> 0.2) - buff-extensions (~> 0.3) + buff-config (~> 1.0) + buff-extensions (~> 1.0) buff-ignore (~> 1.1) buff-shell_out (~> 0.1) - celluloid (~> 0.14.0) - celluloid-io (~> 0.14.0) + celluloid (~> 0.16.0.pre) + celluloid-io (~> 0.16.0.pre) erubis - faraday (>= 0.8.4) - hashie (>= 2.0.2) + faraday (~> 0.9.0) + hashie (>= 2.0.2, < 3.0.0) json (>= 1.7.7) mixlib-authentication (>= 1.3.0) net-http-persistent (>= 2.8) - net-ssh - nio4r (>= 0.5.0) retryable - solve (>= 0.4.4) - varia_model (~> 0.1) - winrm (~> 1.1.0) - rubyntlm (0.1.1) - savon (0.9.5) - akami (~> 1.0) - builder (>= 2.1.2) - gyoku (>= 0.4.0) - httpi (~> 0.9) - nokogiri (>= 1.4.0) - nori (~> 1.0) - wasabi (~> 1.0) - solve (0.8.1) - thor (0.18.1) - timers (1.1.0) - uuidtools (2.1.4) - varia_model (0.2.0) - buff-extensions (~> 0.2) - hashie (>= 2.0.2) - wasabi (1.0.0) - nokogiri (>= 1.4.0) - winrm (1.1.3) - gssapi (~> 1.0.0) - httpclient (~> 2.2, >= 2.2.0.2) - logging (~> 1.6, >= 1.6.1) - nokogiri (~> 1.5) - rubyntlm (~> 0.1.1) - savon (= 0.9.5) - uuidtools (~> 2.1.2) + semverse (~> 1.1) + varia_model (~> 0.4) + rspec (2.99.0) + rspec-core (~> 2.99.0) + rspec-expectations (~> 2.99.0) + rspec-mocks (~> 2.99.0) + rspec-core (2.99.1) + rspec-expectations (2.99.1) + diff-lcs (>= 1.1.3, < 2.0) + rspec-its (1.0.1) + rspec-core (>= 2.99.0.beta1) + rspec-expectations (>= 2.99.0.beta1) + rspec-mocks (2.99.1) + rubocop (0.23.0) + json (>= 1.7.7, < 2) + parser (~> 2.1.9) + powerpack (~> 0.0.6) + rainbow (>= 1.99.1, < 3.0) + ruby-progressbar (~> 1.4) + ruby-progressbar (1.5.1) + safe_yaml (1.0.3) + sawyer (0.5.4) + addressable (~> 2.3.5) + faraday (~> 0.8, < 0.10) + semverse (1.1.0) + serverspec (1.9.1) + highline + net-ssh + rspec (~> 2.13) + rspec-its + specinfra (~> 1.18) + slop (3.5.0) + solve (1.2.0) + dep_selector (~> 1.0) + semverse (~> 1.1) + specinfra (1.19.0) + systemu (2.5.2) + test-kitchen (1.2.1) + mixlib-shellout (~> 1.2) + net-scp (~> 1.1) + net-ssh (~> 2.7) + safe_yaml (~> 1.0) + thor (~> 0.18) + thor (0.19.1) + timers (3.0.1) + hitimes + treetop (1.4.15) + polyglot + polyglot (>= 0.3.1) + varia_model (0.4.0) + buff-extensions (~> 1.0) + hashie (>= 2.0.2, < 3.0.0) + yajl-ruby (1.1.0) PLATFORMS ruby DEPENDENCIES - berkshelf + berkshelf (> 3) + chef (>= 11.8) + chefspec (>= 3.4) + foodcritic (< 4.0) + guard (>= 2.6) + guard-foodcritic (>= 1.0.2) + guard-rubocop (>= 1.1) + kitchen-vagrant + rake (>= 10.2) + rubocop (= 0.23) + serverspec (>= 1.6) + test-kitchen diff --git a/Guardfile b/Guardfile new file mode 100644 index 0000000..f7b7ef8 --- /dev/null +++ b/Guardfile @@ -0,0 +1,14 @@ +# A sample Guardfile +# More info at https://github.com/guard/guard#readme + +guard :rubocop do + watch(%r{.+\.rb$}) + watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) } +end + +guard "foodcritic" do + watch(%r{attributes/.+\.rb$}) + watch(%r{providers/.+\.rb$}) + watch(%r{recipes/.+\.rb$}) + watch(%r{resources/.+\.rb$}) +end diff --git a/README.md b/README.md index 819c49f..50f2dfb 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,12 @@ Description This is the semi-official 'all-in-one' Logstash cookbook. -If you are using logstash < 1.2 you might want to use the 0.6.1 branch. This -cookbook is best for logstash 1.2 and 1.3. +This cookbook is in transition from being a regular cookbook to following the Library Cookbook pattern. +While you can still use the `agent` and `server` recipes, the power of this cookbook now comes from the +`LWRPs`. -*Looking for a logstash >= 1.4 cookbook?* See the `logstash_1.4` branch. Now -with LWRPs, but still in heavy development, so use with caution. +If you are using logstash < 1.2 you might want to use the 0.6.x branch. +If you are using logstash < 1.4 you might want to use the 0.7.x branch. Requirements ============ @@ -27,156 +28,19 @@ see the Berksfile for more details * [Heavywater Graphite Cookbook](https://github.com/hw-cookbooks/graphite) - This is the one I use * [Karmi's ElasticSearch Cookbook](https://github.com/elasticsearch/cookbook-elasticsearch) * [RiotGames RBENV cookbook](https://github.com/RiotGames/rbenv-cookbook) - - +* [@lusis Kibana cookbook](https://github.com/lusis/chef-kibana) Attributes ========== ## Default -* `node['logstash']['homedir']` - the home directory of the logstash user -* `node['logstash']['basedir']` - the base directory for all the - Logstash components -* `node['logstash']['user']` - the owner for all Logstash components -* `node['logstash']['group']` - the group for all Logstash components -* `node['logstash']['supervisor_gid']` - set gid to run logstash as in supervisor ( runit, upstart ). - Useful for Ubuntu where logstash or beaver needs to run as group `adm` to read syslog -* `node['logstash']['graphite_role']` - the Chef role to search for - discovering your preexisting Graphite server -* `node['logstash']['graphite_query']` - the search query used for - discovering your preexisting Graphite server. Defaults to - node['logstash']['graphite_role'] in the current node environment -* `node['logstash']['elasticsearch_role']` - the Chef role to search - for discovering your preexisting ElasticSearch cluster. -* `node['logstash']['elasticsearch_query']` - the search query used for - discovering your preexisting ElasticSearch cluster. Defaults to - node['logstash']['elasticsearch_role'] in the current node environment -* `node['logstash']['elasticsearch_cluster']` - the cluster name - assigned to your preexisting ElasticSearch cluster. Only applies to - external ES clusters. -* `node['logstash']['elasticsearch_ip']` - the IP address that will be - used for your elasticsearch server in case you are using Chef-solo -* `node['logstash']['graphite_ip']` - the IP address that will be used - for your graphite server in case you are using Chef-solo -* `node['logstash']['join_groups']` - An array of Operating System - groups to join. Usefull to gain read privileges on some logfiles. -* `node['logstash']['patterns']` - A hash with grok patterns to be - used on grok and multiline filters. -* `node['logstash']['create_account']` - create the account info from - `user` and `group`; this is `true` by default. Disable it to use an - existing account! -* `node['logstash']['install_zeromq']` - Should this - recipe install zeromq packages? -* `node['logstash']['install_rabbitmq']` - Should this - recipe install rabbitmq packages? -* `node['logstash']['zeromq_packages']` - zeromq_packages to install - if you use zeromq - -## Agent - -* `node['logstash']['agent']['install_method']` - The method to - install logstash - either `jar` or `source`, defaults to `jar` -* `node['logstash']['agent']['version']` - The version of Logstash to - install. Only applies to `jar` install method. -* `node['logstash']['agent']['source_url']` - The URL of the Logstash - jar to download. Only applies to `jar` install method. -* `node['logstash']['agent']['checksum']` - The checksum of the jar - file. Only applies to `jar` install method. -* `node['logstash']['agent']['base_config']` - The name of the - template to use for `logstash.conf` as a base config. -* `node['logstash']['agent']['base_config_cookbook']` - Where to find - the base\_config template. -* `node['logstash']['agent']['workers']` - Number of workers for filter processing. -* `node['logstash']['agent']['xms']` - The minimum memory to assign - the JVM. -* `node['logstash']['agent']['xmx']` - The maximum memory to assign - the JVM. -* `node['logstash']['agent']['java_opts']` - Additional params you - want to pass to the JVM -* `node['logstash']['agent']['gc_opts']` - Specify your garbage - collection options to pass to the JVM -* `node['logstash']['agent']['ipv4_only']` - Add jvm option - preferIPv4Stack? -* `node['logstash']['agent']['debug']` - Run logstash with `-v` - option? -* `node['logstash']['agent']['server_role']` - The role of the node - behaving as a Logstash `server`/`indexer` -* `node['logstash']['agent']['inputs']` - Array of input plugins - configuration. -* `node['logstash']['agent']['filters']` - Array of filter plugins - configuration. -* `node['logstash']['agent']['outputs']` - Array of output plugins - configuration. -* `node['logstash']['agent']['patterns_dir']` - The patterns directory - where pattern files will be generated. Relative to the basedir or - absolute. -* `node['logstash']['agent']['home']` - home dir of logstash agent -* `node['logstash']['agent']['config_dir']` - location of conf.d style config dir -* `node['logstash']['agent']['config_file']` - name for base config file ( in conf.d dir ) -* `node['logstash']['agent']['upstart_with_sudo']` - use sudo with upstart - - -## Server - -* `node['logstash']['server']['install_method']` - The method to - install logstash - either `jar` or `source` -* `node['logstash']['server']['version']` - The version of Logstash to - install. Only applies to `jar` install method. -* `node['logstash']['server']['source_url']` - The URL of the Logstash - jar to download. Only applies to `jar` install method. -* `node['logstash']['server']['checksum']` - The checksum of the jar - file. Only applies to `jar` install method. -* `node['logstash']['server']['base_config']` - The name of the - template to use for `logstash.conf` as a base config. -* `node['logstash']['server']['base_config_cookbook']` - Where to find - the base config template. -* `node['logstash']['server']['xms']` - The minimum memory to assign - the JVM. -* `node['logstash']['server']['xmx']` - The maximum memory to assign - the JVM. -* `node['logstash']['server']['java_opts']` - Additional params you - want to pass to the JVM -* `node['logstash']['server']['gc_opts']` - Specify your garbage - collection options to pass to the JVM -* `node['logstash']['server']['ipv4_only']` - Add jvm option - preferIPv4Stack? -* `node['logstash']['server']['debug']` - Run logstash with `-v` - option? -* `node['logstash']['server']['enable_embedded_es']` - Should Logstash - run with the embedded ElasticSearch server or not? -* `node['logstash']['server']['inputs']` - Array of input plugins - configuration. -* `node['logstash']['server']['filters']` - Array of filter plugins - configuration. -* `node['logstash']['server']['outputs']` - Array of output plugins - configuration. -* `node['logstash']['server']['patterns_dir']` - The patterns - directory where pattern files will be generated. Relative to the - basedir or absolute. -* `node['logstash']['server']['home']` - home dir of logstash agent -* `node['logstash']['server']['config_dir']` - location of conf.d style config dir -* `node['logstash']['server']['config_file']` - name for base config file ( in conf.d dir ) -* `node['logstash']['server']['workers']` - Number of workers for filter processing. -* `node['logstash']['server']['web']['enable']` - true to enable embedded kibana ( may be behind in features ) -* `node['logstash']['server']['web']['address']` - IP Address to listen on -* `node['logstash']['server']['web']['port']` - port to listen on. -* `node['logstash']['server']['upstart_with_sudo']` - use sudo with upstart - -## Kibana - -Kibana can be run from the embedded version in elasticsearch. -It is not recommended that you use this outside of basic testing. This is for several reasons: - -- Kibana is a fast moving target -- It violates SRP -- It's not very secure when run this way -- There are two solid cookbooks for using Kibana now - - Kibana2 (Ruby version): https://github.com/realityforge/chef-kibana - - Kibana3 (HTML/JS version): https://github.com/lusis/chef-kibana +see [attributes/default.rb](attributes/default.rb) ## Beaver (alternative to Logstash Agent) +_This will be depreciated soon in favor of an external library cookbook._ + * `node['logstash']['beaver']['repo']` - URL or repository to install beaver from (using pip). * `node['logstash']['beaver']['server_role']` - The role of the node @@ -240,6 +104,57 @@ It is not recommended that you use this outside of basic testing. This is for se * `node['logstash']['index_cleaner']['cron']['log_file']` - Path to direct the index_cleaner cron job's stdout and stderr +Lightweight Resource Providers +=================== + +These now do all the heavy lifting. + +## logstash_instance + +This will install a logstash instance. It will take defaults from attributes for most attributes. + +see [resources/instance.rb](resources/instance.rb) + +## logstash_service + +This will create system init scripts for managing logstash instance. It will take defaults from attributes for most attributes. + +see [resources/service.rb](resources/service.rb) + +_experimental support for pleaserun has been added. Only `native` for `Ubuntu 12.04` has been thoroughly tested._ + +## logstash_config + +This will create logstash config files. It will take defaults from attributes for most attributes. + +see [resources/config.rb](resources/config.rb) + +## logstash_pattern + +This will install custom grok patterns for logstash. It will take defaults from attributes for most attributes: + +see [resources/pattern.rb](resources/pattern.rb) + +## logstash_plugns + +This will install the logstash community plugins: + +see [resources/plugins.rb](resources/plugins.rb) + +## attribute precidence in logstash LWRPs + +We've done our best to make this intuitive and easy to use. + +1. the value directly in the resource call. +2. the value from the hash node['logstash']['instance'][name] +3. the value from the hash node['logstash']['instance']['default'] + +Searching +====== + +There is a search helper library `libraries/search.rb` which will help you search for values such as `elasticsearch_ip`. see the `server` recipe for an example of its usage. + + Testing ======= @@ -249,16 +164,10 @@ Testing vagrant up precise64 ``` -## Strainer - -``` -rake strainer -``` - -## Test-Kitchen + ServerSpec +## Rubocop, FoodCritic, Rspec, Test-Kitchen ``` -rake kitchen +bundle exec rake ``` Contributing @@ -273,10 +182,10 @@ Usage A proper readme is forthcoming but in the interim.... -There are 2 recipes you need to concern yourself with: +These two recipes show how to install and configure logstash instances via the provided `LWRPs` -* server - This would be your indexer node -* agent - This would be a local host's agent for collection +* [recipe/server.rb](recipe/server.rb) - This would be your indexer node +* [recipe/agent.rb](recipe/agent.rb) - This would be a local host's agent for collection Every attempt (and I mean this) was made to ensure that the following @@ -379,6 +288,11 @@ node['logstash']['agent']['config_templates_variables'] = { apache: { type: 'apa ## Letting data drive your templates +*DEPRECIATED!* + +While this may work ... it is no longer being actively supported by the maintainers of this cookbook. +We will accept `PRs`. + The current templates for the agent and server are written so that you can provide ruby hashes in your roles that map to inputs, filters, and outputs. Here is a role for logstash_server. @@ -634,11 +548,13 @@ Logstash will listen for syslog messages on tcp/5140 - Author: Richard Clamp (@richardc) - Author: Juanje Ojeda (@juanje) - Author: @benattar +- Author: Paul Czarkowski (@pczarkowski) - Copyright: 2012, John E. Vincent - Copyright: 2012, Bryan W. Berry - Copyright: 2012, Richard Clamp - Copyright: 2012, Juanje Ojeda - Copyright: 2012, @benattar +- Copyright: 2014, Paul Czarkowski Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Rakefile b/Rakefile index 6452486..c649ea3 100644 --- a/Rakefile +++ b/Rakefile @@ -1,73 +1,44 @@ -#!/usr/bin/env rake # Encoding: utf-8 +require 'bundler/setup' -@cookbook_path = '/tmp/logstash-cookbooks' -@cookbook = 'logstash' -@gemfile = "#{File.dirname(__FILE__)}/test/support/Gemfile" +namespace :style do + require 'rubocop/rake_task' + desc 'Run Ruby style checks' + RuboCop::RakeTask.new(:ruby) -desc 'install dependencies using Berkshelf' -task :install_deps do - install_deps + require 'foodcritic' + desc 'Run Chef style checks' + FoodCritic::Rake::LintTask.new(:chef) end -desc 'Runs foodcritic linter' -task :foodcritic do - if Gem::Version.new('1.9.2') <= Gem::Version.new(RUBY_VERSION.dup) - sandbox = File.join(File.dirname(__FILE__), %w{tmp foodcritic}, @cookbook) - prepare_test_sandbox(sandbox) - - sh "foodcritic --epic-fail any #{File.dirname(sandbox)}" - else - puts "WARN: foodcritic run is skipped as Ruby #{RUBY_VERSION} is < 1.9.2." +desc 'Run all style checks' +task style: ['style:chef', 'style:ruby'] + +require 'kitchen' +desc 'Run Test Kitchen integration tests' +task :integration do + unless ENV['CI'] + Kitchen.logger = Kitchen.default_file_logger + Kitchen::Config.new.instances.each do |instance| + instance.test(:always) + end end end -desc 'Runs Strainer' -task :strainer do - if Gem::Version.new('1.9.2') <= Gem::Version.new(RUBY_VERSION.dup) - # sandbox = '/tmp/cookbook_logstash' - # prepare_test_sandbox(sandbox) - # rm_rf '/tmp/strainer' - install_deps - # puts gemfile - puts "BUNDLE_GEMFILE=#{@gemfile} bundle exec strainer test --sandbox=/tmp/strainer --cookbooks-path=#{@cookbook_path}" - system({ 'BUNDLE_GEMFILE' => @gemfile }, "bundle exec strainer test --sandbox=/tmp/strainer --cookbooks-path=#{@cookbook_path}") - else - puts "WARN: strainer run is skipped as Ruby #{RUBY_VERSION} is < 1.9.2." - end +require 'rspec/core/rake_task' +desc 'Run ChefSpec unit tests' +RSpec::Core::RakeTask.new(:spec) do |t, _args| + t.rspec_opts = 'test/unit/spec' end -desc 'Runs Test Kitchen' -task :kitchen do +# The default rake task should just run it all +task default: %w(style spec integration) + +unless ENV['CI'] begin require 'kitchen/rake_tasks' Kitchen::RakeTasks.new rescue LoadError puts '>>>>> Kitchen gem not loaded, omitting tasks' unless ENV['CI'] end - if Gem::Version.new('1.9.2') <= Gem::Version.new(RUBY_VERSION.dup) - install_deps - system({ 'BUNDLE_GEMFILE' => @gemfile }, 'bundle exec kitchen test --destroy=always') - end -end - -task default: 'strainer' - -private - -def install_deps - puts "BUNDLE_GEMFILE=#{@gemfile} bundle exec berks install --path=#{@cookbook_path}" - system({ 'BUNDLE_GEMFILE' => @gemfile }, "bundle exec berks install --path=#{@cookbook_path}") -end - -private - -def prepare_test_sandbox(sandbox) - files = %w{ *.md *.rb attributes definitions files providers Strainerfile .rubocop* - recipes resources templates test/integration test/serverspec } - - rm_rf sandbox - mkdir_p sandbox - cp_r Dir.glob("{#{files.join(',')}}"), sandbox - puts "\n\n" end diff --git a/Vagrantfile b/Vagrantfile index a9b6d34..6ce58ed 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -4,73 +4,27 @@ log_level = :info -chef_run_list = %w[ - java::default - logstash::server - logstash::agent -] -# curl::default -# minitest-handler::default -# logstash::server -# logstash::agent -# ark::default -# kibana::default -# ] +chef_run_list = %w( + java::default + curl::default + logstash::server +) +# logstash::agent +# minitest-handler::default +# kibana::default chef_json = { - java: { - jdk_version: '7' - }, - kibana: { - webserver_listen: '0.0.0.0', - webserver: 'nginx', - install_type: 'file' - }, - logstash: { - supervisor_gid: 'adm', - agent: { - server_ipaddress: '127.0.0.1', - xms: '128m', - xmx: '128m', - enable_embedded_es: false, - inputs: [ - file: { - type: 'syslog', - path: ['/var/log/syslog', '/var/log/messages'], - start_position: 'beginning' - } - ], - filters: [ - { - condition: 'if [type] == "syslog"', - block: { - grok: { - match: [ - 'message', - "%{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:host} (?:%{PROG:program}(?:\[%{POSINT:pid}\])?: )?%{GREEDYDATA:message}" - ] - }, - date: { - match: [ - 'timestamp', - 'MMM d HH:mm:ss', - 'MMM dd HH:mm:ss', - 'ISO8601' - ] - } - } - } - ] - }, - server: { - xms: '128m', - xmx: '128m', - enable_embedded_es: true, - config_templates: ['apache'], - config_templates_variables: { apache: { type: 'apache' } }, - web: { enable: true } - } - } + java: { + jdk_version: '7' + }, + kibana: { + webserver_listen: '0.0.0.0', + webserver: 'nginx', + install_type: 'file' + }, + logstash: { + instance: { server: {} } + } } Vagrant.configure('2') do |config| @@ -88,6 +42,7 @@ Vagrant.configure('2') do |config| end config.vm.define :precise64 do |dist_config| + dist_config.cache.scope = :box if Vagrant.has_plugin?('vagrant-cachier') dist_config.vm.box = 'opscode-ubuntu-12.04' dist_config.vm.box_url = 'https://opscode-vm-bento.s3.amazonaws.com/vagrant/opscode_ubuntu-12.04_provisionerless.box' @@ -98,7 +53,6 @@ Vagrant.configure('2') do |config| chef.run_list = chef_run_list chef.json = chef_json chef.run_list.unshift('apt') - chef.json[:logstash][:server][:init_method] = 'runit' end end @@ -113,7 +67,6 @@ Vagrant.configure('2') do |config| chef.run_list = chef_run_list chef.json = chef_json chef.run_list.unshift('apt') - chef.json[:logstash][:server][:init_method] = 'runit' end end config.vm.define :lucid32 do |dist_config| @@ -126,7 +79,6 @@ Vagrant.configure('2') do |config| chef.run_list = chef_run_list chef.json = chef_json chef.run_list.unshift('apt') - chef.json[:logstash][:server][:init_method] = 'runit' end end diff --git a/attributes/agent.rb b/attributes/agent.rb deleted file mode 100644 index 5f33809..0000000 --- a/attributes/agent.rb +++ /dev/null @@ -1,46 +0,0 @@ -# Encoding: utf-8 -default['logstash']['agent']['version'] = '1.3.3' -default['logstash']['agent']['log_file'] = '/var/log/logstash/agent.log' -default['logstash']['agent']['source_url'] = 'https://download.elasticsearch.org/logstash/logstash/logstash-1.3.3-flatjar.jar' -default['logstash']['agent']['checksum'] = 'a83503bd2aa32e1554b98f812d0b411fbc5f7b6b21cebb48b7d344474f2dfc6d' -default['logstash']['agent']['install_method'] = 'jar' # Either `source` or `jar` -default['logstash']['agent']['home'] = "#{node['logstash']['basedir']}/agent" -default['logstash']['agent']['patterns_dir'] = 'etc/patterns' -default['logstash']['agent']['config_dir'] = 'etc/conf.d' -default['logstash']['agent']['config_file'] = 'logstash.conf' -default['logstash']['agent']['config_templates'] = [] -default['logstash']['agent']['config_templates_cookbook'] = 'logstash' -default['logstash']['agent']['config_templates_variables'] = {} -default['logstash']['agent']['base_config'] = 'agent.conf.erb' -default['logstash']['agent']['base_config_cookbook'] = 'logstash' -default['logstash']['agent']['xms'] = '384M' -default['logstash']['agent']['xmx'] = '384M' -default['logstash']['agent']['java_opts'] = '' -default['logstash']['agent']['gc_opts'] = '-XX:+UseParallelOldGC' -default['logstash']['agent']['ipv4_only'] = false -default['logstash']['agent']['debug'] = false -# allow control over the upstart config -default['logstash']['agent']['upstart_with_sudo'] = false -default['logstash']['agent']['upstart_respawn_count'] = 5 -default['logstash']['agent']['upstart_respawn_timeout'] = 30 -default['logstash']['agent']['init_method'] = 'native' # native or runit -default['logstash']['agent']['init_method'] = 'native' -default['logstash']['agent']['workers'] = 1 - -# allow control over the upstart config -default['logstash']['server']['upstart_with_sudo'] = false - -# logrotate options for logstash agent -default['logstash']['agent']['logrotate']['options'] = %w{ missingok notifempty } -# stop/start on logrotate? -default['logstash']['agent']['logrotate']['stopstartprepost'] = false - -# roles/flasgs for various autoconfig/discovery components -default['logstash']['agent']['server_role'] = 'logstash_server' - -# for use in case recipe used w/ chef-solo, default to self -default['logstash']['agent']['server_ipaddress'] = '' - -default['logstash']['agent']['inputs'] = [] -default['logstash']['agent']['filters'] = [] -default['logstash']['agent']['outputs'] = [] diff --git a/attributes/beaver.rb b/attributes/beaver.rb index 773fb8a..f1dbcce 100644 --- a/attributes/beaver.rb +++ b/attributes/beaver.rb @@ -9,5 +9,5 @@ default['logstash']['beaver']['outputs'] = [] default['logstash']['beaver']['format'] = 'json' -default['logstash']['beaver']['logrotate']['options'] = %w{ missingok notifempty compress copytruncate } +default['logstash']['beaver']['logrotate']['options'] = %w(missingok notifempty compress copytruncate) default['logstash']['beaver']['logrotate']['postrotate'] = 'invoke-rc.d logstash_beaver force-reload >/dev/null 2>&1 || true' diff --git a/attributes/default.rb b/attributes/default.rb index a07580e..ee54563 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -1,40 +1,87 @@ # Encoding: utf-8 -default['logstash']['basedir'] = '/opt/logstash' -default['logstash']['user'] = 'logstash' -default['logstash']['uid'] = nil # set to nil to let system pick -default['logstash']['group'] = 'logstash' -default['logstash']['gid'] = nil # set to nil to let system pick -default['logstash']['supervisor_gid'] = node['logstash']['group'] -default['logstash']['pid_dir'] = '/var/run/logstash' -default['logstash']['create_account'] = true -default['logstash']['join_groups'] = [] -default['logstash']['homedir'] = '/var/lib/logstash' # roles/flags for various search/discovery -default['logstash']['graphite_role'] = 'graphite_server' -default['logstash']['graphite_query'] = "roles:#{node['logstash']['graphite_role']} AND chef_environment:#{node.chef_environment}" -default['logstash']['elasticsearch_role'] = 'elasticsearch_server' -default['logstash']['elasticsearch_query'] = "roles:#{node['logstash']['elasticsearch_role']} AND chef_environment:#{node.chef_environment}" -default['logstash']['elasticsearch_cluster'] = 'logstash' -default['logstash']['elasticsearch_ip'] = '' -default['logstash']['elasticsearch_port'] = '' -default['logstash']['graphite_ip'] = '' - -default['logstash']['patterns'] = {} -default['logstash']['install_zeromq'] = false -default['logstash']['install_rabbitmq'] = false - -case node['platform_family'] -when 'rhel' - default['logstash']['zeromq_packages'] = %w{ zeromq zeromq-devel } -when 'fedora' - default['logstash']['zeromq_packages'] = %w{ zeromq zeromq-devel } -when 'debian' - default['logstash']['zeromq_packages'] = %w{ libzmq3-dbg libzmq3-dev libzmq3 } -end +default['logstash']['instance']['default']['graphite_role'] = 'graphite_server' +default['logstash']['instance']['default']['graphite_query'] = "roles:#{node['logstash']['instance']['default']['graphite_role']} AND chef_environment:#{node.chef_environment}" +default['logstash']['instance']['default']['elasticsearch_role'] = 'elasticsearch_server' +default['logstash']['instance']['default']['elasticsearch_query'] = "roles:#{node['logstash']['instance']['default']['elasticsearch_role']} AND chef_environment:#{node.chef_environment}" +default['logstash']['instance']['default']['elasticsearch_cluster'] = 'logstash' +default['logstash']['instance']['default']['elasticsearch_ip'] = '' +default['logstash']['instance']['default']['elasticsearch_port'] = '' +default['logstash']['instance']['default']['graphite_ip'] = '' + +# Default logstash instance variables +default['logstash']['instance']['default']['basedir'] = '/opt/logstash' +default['logstash']['instance']['default']['user'] = 'logstash' +default['logstash']['instance']['default']['group'] = 'logstash' +default['logstash']['instance']['default']['user_opts'] = { homedir: '/var/lib/logstash', uid: nil, gid: nil } +default['logstash']['instance']['default']['supervisor_gid'] = node['logstash']['instance']['default']['group'] +default['logstash']['instance']['default']['pid_dir'] = '/var/run/logstash' +default['logstash']['instance']['default']['create_account'] = true +default['logstash']['instance']['default']['join_groups'] = [] +default['logstash']['instance']['default']['homedir'] = '/var/lib/logstash' + +default['logstash']['instance']['default']['version'] = '1.4.1' +default['logstash']['instance']['default']['source_url'] = 'https://download.elasticsearch.org/logstash/logstash/logstash-1.4.1.tar.gz' +default['logstash']['instance']['default']['checksum'] = 'a1db8eda3d8bf441430066c384578386601ae308ccabf5d723df33cee27304b4' +default['logstash']['instance']['default']['install_type'] = 'tarball' + +default['logstash']['instance']['default']['plugins_version'] = '1.4.1' +default['logstash']['instance']['default']['plugins_source_url'] = 'https://download.elasticsearch.org/logstash/logstash/logstash-contrib-1.4.1.tar.gz' +default['logstash']['instance']['default']['plugins_checksum'] = 'beb0927351a3c298cd346225950c8d4fbda984ba54252d8a2f244329207c31e2' +default['logstash']['instance']['default']['plugins_install_type'] = 'tarball' # native|tarball +default['logstash']['instance']['default']['plugins_check_if_installed'] = 'lib/logstash/filters/translate.rb' + +default['logstash']['instance']['default']['log_file'] = 'logstash.log' +default['logstash']['instance']['default']['xms'] = '1024M' +default['logstash']['instance']['default']['xmx'] = "#{(node['memory']['total'].to_i * 0.6).floor / 1024}M" +default['logstash']['instance']['default']['java_opts'] = '' +default['logstash']['instance']['default']['gc_opts'] = '-XX:+UseParallelOldGC' +default['logstash']['instance']['default']['ipv4_only'] = false +default['logstash']['instance']['default']['debug'] = false +default['logstash']['instance']['default']['workers'] = 1 + +default['logstash']['instance']['default']['pattern_templates_cookbook'] = 'logstash' +default['logstash']['instance']['default']['pattern_templates'] = {} +default['logstash']['instance']['default']['pattern_templates_variables'] = {} + +default['logstash']['instance']['default']['base_config_cookbook'] = 'logstash' +default['logstash']['instance']['default']['base_config'] = '' # set if want data driven + +default['logstash']['instance']['default']['config_file'] = '' +default['logstash']['instance']['default']['config_templates'] = {} +default['logstash']['instance']['default']['config_templates_cookbook'] = 'logstash' +default['logstash']['instance']['default']['config_templates_variables'] = {} + +# allow control over the upstart config +default['logstash']['instance']['default']['upstart_with_sudo'] = false + +default['logstash']['instance']['default']['init_method'] = 'native' # pleaserun or native or runit +default['logstash']['instance']['default']['service_templates_cookbook'] = 'logstash' + +# roles/flags for various autoconfig/discovery components +default['logstash']['instance']['default']['enable_embedded_es'] = false +default['logstash']['instance']['default']['bind_host_interface'] = '' +default['logstash']['instance']['default']['es_index'] = nil + +default['logstash']['instance']['default']['inputs'] = [] +default['logstash']['instance']['default']['filters'] = [] +default['logstash']['instance']['default']['outputs'] = [] + +default['logstash']['instance']['default']['web']['enable'] = false +default['logstash']['instance']['default']['web']['address'] = '0.0.0.0' +default['logstash']['instance']['default']['web']['port'] = '9292' # Logging features -default['logstash']['logging']['rotateFrequency'] = 'daily' -default['logstash']['logging']['maxBackup'] = 10 -default['logstash']['logging']['maxSize'] = '10M' -default['logstash']['logging']['useFileSize'] = false +default['logstash']['instance']['default']['logrotate_enable'] = true +default['logstash']['instance']['default']['logrotate_options'] = %w(missingok notifempty compress copytruncate) +default['logstash']['instance']['default']['logrotate_frequency'] = 'daily' +default['logstash']['instance']['default']['logrotate_max_backup'] = 10 +default['logstash']['instance']['default']['logrotate_max_size'] = '10M' +default['logstash']['instance']['default']['logrotate_use_filesize'] = false + +# Curator +default['logstash']['instance']['default']['curator_days_to_keep'] = 31 +default['logstash']['instance']['default']['curator_cron_minute'] = '0' +default['logstash']['instance']['default']['curator_cron_hour'] = '*' +default['logstash']['instance']['default']['curator_cron_log_file'] = '/dev/null' diff --git a/attributes/index_cleaner.rb b/attributes/index_cleaner.rb deleted file mode 100644 index 6943c7b..0000000 --- a/attributes/index_cleaner.rb +++ /dev/null @@ -1,7 +0,0 @@ -# Encoding: utf-8 -default['logstash']['index_cleaner']['days_to_keep'] = 31 -default['logstash']['index_cleaner']['cron'] = { - 'minute' => '0', - 'hour' => '*', - 'log_file' => '/dev/null' -} diff --git a/attributes/server.rb b/attributes/server.rb deleted file mode 100644 index fbbded9..0000000 --- a/attributes/server.rb +++ /dev/null @@ -1,39 +0,0 @@ -# Encoding: utf-8 -default['logstash']['server']['version'] = '1.3.3' -default['logstash']['server']['home'] = "#{node['logstash']['basedir']}/server" -default['logstash']['server']['log_file'] = '/var/log/logstash/server.log' -default['logstash']['server']['source_url'] = 'https://download.elasticsearch.org/logstash/logstash/logstash-1.3.3-flatjar.jar' -default['logstash']['server']['checksum'] = 'a83503bd2aa32e1554b98f812d0b411fbc5f7b6b21cebb48b7d344474f2dfc6d' -default['logstash']['server']['install_method'] = 'jar' # Either `source` or `jar` -default['logstash']['server']['patterns_dir'] = 'etc/patterns' -default['logstash']['server']['config_dir'] = 'etc/conf.d' -default['logstash']['server']['config_file'] = 'logstash.conf' -default['logstash']['server']['config_templates'] = [] -default['logstash']['server']['config_templates_cookbook'] = 'logstash' -default['logstash']['server']['config_templates_variables'] = {} -default['logstash']['server']['base_config'] = 'server.conf.erb' # set blank if don't want data driven config -default['logstash']['server']['base_config_cookbook'] = 'logstash' -default['logstash']['server']['xms'] = '1024M' -default['logstash']['server']['xmx'] = '1024M' -default['logstash']['server']['java_opts'] = '' -default['logstash']['server']['gc_opts'] = '-XX:+UseParallelOldGC' -default['logstash']['server']['ipv4_only'] = false -default['logstash']['server']['debug'] = false -default['logstash']['server']['workers'] = 1 - -# allow control over the upstart config -default['logstash']['server']['upstart_with_sudo'] = false - -default['logstash']['server']['init_method'] = 'native' # native or runit -# roles/flags for various autoconfig/discovery components -default['logstash']['server']['enable_embedded_es'] = true - -default['logstash']['server']['inputs'] = [] -default['logstash']['server']['filters'] = [] -default['logstash']['server']['outputs'] = [] - -default['logstash']['server']['web']['enable'] = false -default['logstash']['server']['web']['address'] = '0.0.0.0' -default['logstash']['server']['web']['port'] = '9292' - -default['logstash']['server']['logrotate']['options'] = %w{ missingok notifempty compress copytruncate } diff --git a/libraries/logstash_conf.rb b/libraries/logstash_conf.rb index 95a7619..d35d402 100644 --- a/libraries/logstash_conf.rb +++ b/libraries/logstash_conf.rb @@ -4,8 +4,6 @@ # Evaluate objects for logstash config file. class Erubis::RubyEvaluator::LogstashConf - private - def self.key_to_str(k) case k.class.to_s when 'String' @@ -23,7 +21,7 @@ def self.hash_to_str(h, indent = 0) h.each do |k, v| case v when Hash, Mash - result << k + " {" + result << k + ' {' result << hash_to_str(v, indent) else indent += 4 @@ -39,9 +37,9 @@ def self.value_to_str(v, indent = 0) when String, Symbol, Fixnum, Float "\"#{v}\"" when Array - "[#{v.map { |e| value_to_str(e,indent) }.join(", ")}]" + "[#{v.map { |e| value_to_str(e, indent) }.join(', ')}]" when Hash, Mash - hash_to_str(v, indent) + "\n" + indent(indent + 4) + "}" + hash_to_str(v, indent) + "\n" + indent(indent + 4) + '}' when TrueClass, FalseClass v.to_s else @@ -50,42 +48,35 @@ def self.value_to_str(v, indent = 0) end def self.key_value_to_str(k, v, indent = 0) - if !v.nil? - #k.inspect + " => " + v.inspect - key_to_str(k) + ' => ' + value_to_str(v, indent) - else + if v.nil? key_to_str(k) + else + # k.inspect + " => " + v.inspect + key_to_str(k) + ' => ' + value_to_str(v, indent) end end def self.plugin_to_arr(plugin, patterns_dir_plugins = nil, patterns_dir = nil, indent = 0) # , type_to_condition) result = [] plugin.each do |name, hash| -# result << '' -# result << " if [type] == \"#{hash['type']}\" {" if hash.has_key?('type') and type_to_condition indent += 4 result << indent(indent) + name.to_s + ' {' - result << indent(indent) + key_value_to_str('patterns_dir', patterns_dir, indent) if patterns_dir_plugins.include?(name.to_s) && !patterns_dir.nil? && !hash.key?('patterns_dir') + result << indent(indent) + key_value_to_str('patterns_dir', patterns_dir, indent) if patterns_dir_plugins.include?(name.to_s) && patterns_dir && !hash.key?('patterns_dir') hash.sort.each do |k, v| -# next if k == 'type' and type_to_condition indent += 4 result << indent(indent) + key_value_to_str(k, v, indent) indent -= 4 end result << indent(indent) + '}' indent -= 4 -# result << ' }' if hash.has_key?('type') and type_to_condition end return result.join("\n") end - public - def self.section_to_str(section, version = nil, patterns_dir = nil, indent = 0) result = [] patterns_dir_plugins = ['grok'] patterns_dir_plugins << 'multiline' if Gem::Version.new(version) >= Gem::Version.new('1.1.2') unless version.nil? -# type_to_condition = Gem::Version.new(version) >= Gem::Version.new('1.2.0') section.each do |segment| result << '' if segment.key?('condition') || segment.key?('block') @@ -105,7 +96,7 @@ def self.section_to_str(section, version = nil, patterns_dir = nil, indent = 0) end def indent(i) - res = String.new - i.times { res << " " } + res = '' + i.times { res << ' ' } res end diff --git a/libraries/logstash_util.rb b/libraries/logstash_util.rb new file mode 100644 index 0000000..42c5809 --- /dev/null +++ b/libraries/logstash_util.rb @@ -0,0 +1,35 @@ +# Encoding: utf-8 +# rubocop:disable RedundantReturn + +module Logstash + def self.service_ip(node, instance = 'default', service = 'elasticsearch', interface = nil) + if node['logstash']['instance'].key?(instance) + attributes = node['logstash']['instance'][instance] + defaults = node['logstash']['instance']['default'] + else + attributes = node['logstash']['instance']['default'] + end + if Chef::Config[:solo] + service_ip = attributes["#{service}_ip"] || defaults["#{service}_ip"] + else + results = [] + service_query = attributes["#{service}_query"] || defaults["#{service}_query"] + Chef::Search::Query.new.search(:node, service_query) { |o| results << o } + if !results.empty? + service_ip = get_ip_for_node(results[0], interface) + else + service_ip = attributes["#{service}_ip"] || defaults["#{service}_ip"] + end + end + end + + def self.get_ip_for_node(node, interface) + if interface.nil? + service_ip = node['ipaddress'] + else + service_ip = node['network']['interfaces'][interface]['addresses'].to_hash.find do + |_, addr_info| addr_info['family'] == 'inet' + end.first + end + end +end diff --git a/libraries/matchers.rb b/libraries/matchers.rb new file mode 100644 index 0000000..4f4a82a --- /dev/null +++ b/libraries/matchers.rb @@ -0,0 +1,54 @@ +# Encoding: utf-8 +# used by ChefSpec for LWRPs + +if defined?(ChefSpec) + # LWRP - Instance + def create_logstash_instance(name) + ChefSpec::Matchers::ResourceMatcher.new(:logstash_instance, :create, name) + end + + def delete_logstash_instance(name) + ChefSpec::Matchers::ResourceMatcher.new(:logstash_instance, :delete, name) + end + + # LWRP - Config + def create_logstash_config(name) + ChefSpec::Matchers::ResourceMatcher.new(:logstash_config, :create, name) + end + + # LWRP - Pattern + def create_logstash_pattern(name) + ChefSpec::Matchers::ResourceMatcher.new(:logstash_pattern, :create, name) + end + + # LWRP - Plugins + def create_logstash_plugins(name) + ChefSpec::Matchers::ResourceMatcher.new(:logstash_plugins, :create, name) + end + + # LWRP - Service + def enable_logstash_service(name) + ChefSpec::Matchers::ResourceMatcher.new(:logstash_service, :enable, name) + end + + def restart_logstash_service(name) + ChefSpec::Matchers::ResourceMatcher.new(:logstash_service, :restart, name) + end + + def start_logstash_service(name) + ChefSpec::Matchers::ResourceMatcher.new(:logstash_service, :start, name) + end + + def reload_logstash_service(name) + ChefSpec::Matchers::ResourceMatcher.new(:logstash_service, :reload, name) + end + + def stop_logstash_service(name) + ChefSpec::Matchers::ResourceMatcher.new(:logstash_service, :stop, name) + end + + def create_logstash_curator(name) + ChefSpec::Matchers::ResourceMatcher.new(:logstash_curator, :create, name) + end + +end diff --git a/metadata.rb b/metadata.rb index defdb9b..4b327f6 100644 --- a/metadata.rb +++ b/metadata.rb @@ -5,16 +5,16 @@ license 'Apache 2.0' description 'Installs/Configures logstash' long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version '0.8.1' +version '0.9.1' -%w{ ubuntu debian redhat centos scientific amazon fedora }.each do |os| +%w(ubuntu debian redhat centos scientific amazon fedora).each do |os| supports os end -%w{ build-essential runit git ant java logrotate rabbitmq yum python }.each do |ckbk| +%w(build-essential runit git ant java logrotate yum python ark).each do |ckbk| depends ckbk end -%w{ yum apt }.each do |ckbk| +%w(yum apt).each do |ckbk| recommends ckbk end diff --git a/providers/config.rb b/providers/config.rb new file mode 100644 index 0000000..9f81d31 --- /dev/null +++ b/providers/config.rb @@ -0,0 +1,65 @@ +# Encoding: utf-8 +# Cookbook Name:: logstash +# Provider:: config +# Author:: John E. Vincent +# License:: Apache 2.0 +# +# Copyright 2014, John E. Vincent + +require 'chef/mixin/shell_out' +require 'chef/mixin/language' +include Chef::Mixin::ShellOut + +def load_current_resource + @instance = new_resource.instance + if node['logstash']['instance'].key?(@instance) + attributes = node['logstash']['instance'][@instance] + defaults = node['logstash']['instance']['default'] + else + attributes = node['logstash']['instance']['default'] + end + @basedir = attributes['basedir'] || defaults['basedir'] + @templates = new_resource.templates || attributes['config_templates'] || defaults['config_templates'] + @templates_cookbook = new_resource.templates_cookbook || attributes['config_templates_cookbook'] || defaults['config_templates_cookbook'] + @variables = new_resource.variables || attributes['config_templates_variables'] || defaults['config_templates_variables'] + @owner = new_resource.owner || attributes['user'] || defaults['user'] + @group = new_resource.group || attributes['group'] || defaults['group'] + @mode = new_resource.mode || '0644' + @path = new_resource.path || "#{@basedir}/#{@instance}/etc/conf.d" + @service_name = new_resource.service_name || @instance +end + +action :create do + conf = conf_vars + # Chef::Log.info("config vars: #{conf.inspect}") + conf[:templates].each do |_template, file| + tp = template "#{conf[:path]}/#{::File.basename(file).chomp(::File.extname(file))}" do + source file + cookbook conf[:templates_cookbook] + owner conf[:owner] + group conf[:group] + mode conf[:mode] + variables conf[:variables] + notifies :restart, "logstash_service[#{conf[:service_name]}]" + action :create + end + new_resource.updated_by_last_action(tp.updated_by_last_action?) + end +end + +private + +def conf_vars + conf = { + instance: @instance, + templates: @templates, + variables: @variables, + path: @path, + owner: @owner, + group: @group, + mode: @mode, + service_name: @service_name, + templates_cookbook: @templates_cookbook + } + conf +end diff --git a/providers/curator.rb b/providers/curator.rb new file mode 100644 index 0000000..2ea5e2c --- /dev/null +++ b/providers/curator.rb @@ -0,0 +1,76 @@ +# Encoding: utf-8 +# Cookbook Name:: logstash +# Provider:: curator +# Author:: John E. Vincent +# License:: Apache 2.0 +# +# Copyright 2014, John E. Vincent + +require 'chef/mixin/shell_out' +require 'chef/mixin/language' +include Chef::Mixin::ShellOut + +def load_current_resource + @instance = new_resource.instance || 'default' + if node['logstash']['instance'].key?(@instance) + attributes = node['logstash']['instance'][@instance] + defaults = node['logstash']['instance']['default'] + else + attributes = node['logstash']['instance']['default'] + end + @days_to_keep = new_resource.days_to_keep || attributes['curator_days_to_keep'] || defaults['curator_days_to_keep'] + @minute = new_resource.minute || attributes['curator_cron_minute'] || defaults['curator_cron_minute'] + @hour = new_resource.hour || attributes['curator_cron_hour'] || defaults['curator_cron_hour'] + @log_file = new_resource.log_file || attributes['curator_cron_log_file'] || defaults['curator_cron_log_file'] + @user = new_resource.user || attributes['user'] || defaults['user'] +end + +action :create do + cur_instance = @instance + cur_days_to_keep = @days_to_keep + cur_log_file = @log_file + cur_hour = @hour + cur_minute = @minute + cur_user = @user + + @run_context.include_recipe 'python::pip' + + pi = python_pip 'elasticsearch-curator' do + action :install + end + new_resource.updated_by_last_action(pi.updated_by_last_action?) + + cr = cron "curator-#{cur_instance}" do + command "curator --host #{::Logstash.service_ip(node, cur_instance, 'elasticsearch')} -d #{cur_days_to_keep} &> #{cur_log_file}" + user cur_user + minute cur_minute + hour cur_hour + action [:create] + end + new_resource.updated_by_last_action(cr.updated_by_last_action?) +end + +action :delete do + cur_instance = @instance + cur_days_to_keep = @days_to_keep + cur_log_file = @log_file + cur_hour = @hour + cur_minute = @minute + cur_user = @user + + @run_context.include_recipe 'python::pip' + + pi = python_pip 'elasticsearch-curator' do + action :install + end + new_resource.updated_by_last_action(pi.updated_by_last_action?) + + cr = cron "curator-#{cur_instance}" do + command "curator --host #{::Logstash.service_ip(node, cur_instance, 'elasticsearch')} -d #{cur_days_to_keep} &> #{cur_log_file}" + user cur_user + minute cur_minute + hour cur_hour + action [:delete] + end + new_resource.updated_by_last_action(cr.updated_by_last_action?) +end diff --git a/providers/instance.rb b/providers/instance.rb new file mode 100644 index 0000000..09ec0de --- /dev/null +++ b/providers/instance.rb @@ -0,0 +1,244 @@ +# Encoding: utf-8 +# Cookbook Name:: logstash +# Provider:: instance +# Author:: John E. Vincent +# License:: Apache 2.0 +# +# Copyright 2014, John E. Vincent + +require 'chef/mixin/shell_out' +require 'chef/mixin/language' +include Chef::Mixin::ShellOut + +def load_current_resource + @name = new_resource.name || 'default' + if node['logstash']['instance'].key?(@name) + attributes = node['logstash']['instance'][@name] + defaults = node['logstash']['instance']['default'] + else + attributes = node['logstash']['instance']['default'] + end + @base_directory = new_resource.base_directory || attributes['basedir'] || defaults['basedir'] + @install_type = new_resource.install_type || attributes['install_type'] || defaults['install_type'] + @version = new_resource.version || attributes['version'] || defaults['version'] + @checksum = new_resource.checksum || attributes['checksum'] || defaults['checksum'] + @source_url = new_resource.source_url || attributes['source_url'] || defaults['source_url'] + @repo = new_resource.repo + @sha = new_resource.sha + @java_home = new_resource.java_home + @user = new_resource.user || attributes['user'] || defaults['user'] + @group = new_resource.group || attributes['group'] || defaults['group'] + @useropts = new_resource.user_opts || attributes['user_opts'] || defaults['user_opts'] + @instance_dir = "#{@base_directory}/#{new_resource.name}".clone + @logrotate_size = new_resource.user_opts || attributes['logrotate_max_size'] || defaults['logrotate_max_size'] + @logrotate_use_filesize = new_resource.logrotate_use_filesize || attributes['logrotate_use_filesize'] || defaults['logrotate_use_filesize'] + @logrotate_frequency = new_resource.logrotate_frequency || attributes['logrotate_frequency'] || defaults['logrotate_frequency'] + @logrotate_max_backup = new_resource.logrotate_max_backup || attributes['logrotate_max_backup'] || defaults['logrotate_max_backup'] + @logrotate_options = new_resource.logrotate_options || attributes['logrotate_options'] || defaults['logrotate_options'] + @logrotate_enable = new_resource.logrotate_enable || attributes['logrotate_enable'] || defaults['logrotate_enable'] +end + +action :delete do + ls = ls_vars + + idr = directory ls[:instance_dir] do + recursive true + action :delete + end + new_resource.updated_by_last_action(idr.updated_by_last_action?) +end + +action :create do + ls = ls_vars + + ur = user ls[:user] do + home ls[:homedir] + system true + action :create + manage_home true + uid ls[:uid] + end + new_resource.updated_by_last_action(ur.updated_by_last_action?) + + gr = group ls[:group] do + gid ls[:gid] + members ls[:user] + append true + system true + end + new_resource.updated_by_last_action(gr.updated_by_last_action?) + + case @install_type + when 'tarball' + @run_context.include_recipe 'ark::default' + arkit = ark ls[:name] do + url ls[:source_url] + checksum ls[:checksum] + owner ls[:user] + group ls[:group] + mode 0755 + version ls[:version] + path ls[:basedir] + action :put + end + new_resource.updated_by_last_action(arkit.updated_by_last_action?) + + %w(bin etc lib log tmp etc/conf.d patterns).each do |ldir| + r = directory "#{ls[:instance_dir]}/#{ldir}" do + action :create + mode '0755' + owner ls[:user] + group ls[:group] + end + new_resource.updated_by_last_action(r.updated_by_last_action?) + end + + when 'jar' + bdr = directory @base_directory do + action :create + mode '0755' + owner ls[:user] + group ls[:group] + end + new_resource.updated_by_last_action(bdr.updated_by_last_action?) + + idr = directory ls[:instance_dir] do + action :create + mode '0755' + owner ls[:user] + group ls[:group] + end + new_resource.updated_by_last_action(idr.updated_by_last_action?) + + %w(bin etc lib log tmp etc/conf.d patterns).each do |ldir| + r = directory "#{ls[:instance_dir]}/#{ldir}" do + action :create + mode '0755' + owner ls[:user] + group ls[:group] + end + new_resource.updated_by_last_action(r.updated_by_last_action?) + end + + rfr = remote_file "#{ls[:instance_dir]}/lib/logstash-#{ls[:version]}.jar" do + owner ls[:user] + group ls[:group] + mode '0755' + source ls[:source_url] + checksum ls[:checksum] + end + new_resource.updated_by_last_action(rfr.updated_by_last_action?) + + lr = link "#{ls[:instance_dir]}/lib/logstash.jar" do + to "#{ls[:instance_dir]}/lib/logstash-#{ls[:version]}.jar" + only_if { new_resource.auto_symlink } + end + new_resource.updated_by_last_action(lr.updated_by_last_action?) + + when 'source' + bdr = directory @base_directory do + action :create + mode '0755' + owner ls[:user] + group ls[:group] + end + new_resource.updated_by_last_action(bdr.updated_by_last_action?) + + idr = directory ls[:instance_dir] do + action :create + mode '0755' + owner ls[:user] + group ls[:group] + end + new_resource.updated_by_last_action(idr.updated_by_last_action?) + + %w(bin etc lib log tmp etc/conf.d patterns).each do |ldir| + r = directory "#{ls[:instance_dir]}/#{ldir}" do + action :create + mode '0755' + owner ls[:user] + group ls[:group] + end + new_resource.updated_by_last_action(r.updated_by_last_action?) + end + + sd = directory "#{ls[:instance_dir]}/source" do + action :create + owner ls[:user] + group ls[:group] + mode '0755' + end + new_resource.updated_by_last_action(sd.updated_by_last_action?) + + gr = git "#{ls[:instance_dir]}/source" do + repository @repo + reference @sha + action :sync + user ls[:user] + group ls[:group] + end + new_resource.updated_by_last_action(gr.updated_by_last_action?) + + source_version = @sha || "v#{@version}" + er = execute 'build-logstash' do + cwd "#{ls[:instance_dir]}/source" + environment(JAVA_HOME: @java_home) + user ls_user # Changed from root cause building as root...WHA? + command "make clean && make VERSION=#{source_version} jar" + action :run + creates "#{ls[:instance_dir]}/source/build/logstash-#{source_version}--monolithic.jar" + not_if "test -f #{ls[:instance_dir]}/source/build/logstash-#{source_version}--monolithic.jar" + end + new_resource.updated_by_last_action(er.updated_by_last_action?) + lr = link "#{ls[:instance_dir]}/lib/logstash.jar" do + to "#{ls[:instance_dir]}/source/build/logstash-#{source_version}--monolithic.jar" + only_if { new_resource.auto_symlink } + end + new_resource.updated_by_last_action(lr.updated_by_last_action?) + else + Chef::Application.fatal!("Unknown install type: #{@install_type}") + end + logrotate(ls) if ls[:logrotate_enable] + +end + +private + +def logrotate(ls) + name = ls[:name] + + @run_context.include_recipe 'logrotate::default' + + logrotate_app "logstash_#{name}" do + path "#{ls[:homedir]}/log/*.log" + size ls[:logrotate_size] if ls[:logrotate_use_filesize] + frequency ls[:logrotate_frequency] + rotate ls[:logrotate_max_backup] + options ls[:logrotate_options] + create "664 #{ls[:user]} #{ls[:group]}" + end +end + +def ls_vars + ls = { + homedir: @useropts[:homedir], + uid: @useropts[:uid], + gid: @useropts[:gid], + source_url: @source_url, + version: @version, + checksum: @checksum, + basedir: @base_directory, + user: @user, + group: @group, + name: @name, + instance_dir: @instance_dir, + enable_logrotate: @enable_logrotate, + logrotate_size: @logrotate_size, + logrotate_use_filesize: @logrotate_use_filesize, + logrotate_frequency: @logrotate_frequency, + logrotate_max_backup: @logrotate_max_backup, + logrotate_options: @logrotate_options, + logrotate_enable: @logrotate_enable + } + ls +end diff --git a/providers/pattern.rb b/providers/pattern.rb new file mode 100644 index 0000000..829d306 --- /dev/null +++ b/providers/pattern.rb @@ -0,0 +1,63 @@ +# Encoding: utf-8 +# Cookbook Name:: logstash +# Provider:: patterns +# Author:: John E. Vincent +# License:: Apache 2.0 +# +# Copyright 2014, John E. Vincent + +require 'chef/mixin/shell_out' +require 'chef/mixin/language' +include Chef::Mixin::ShellOut + +def load_current_resource + @instance = new_resource.instance + if node['logstash']['instance'].key?(@instance) + attributes = node['logstash']['instance'][@instance] + defaults = node['logstash']['instance']['default'] + else + attributes = node['logstash']['instance']['default'] + end + @basedir = attributes['basedir'] || defaults['basedir'] + @templates = new_resource.templates || attributes['pattern_templates'] || defaults['pattern_templates'] + @variables = new_resource.variables || attributes['pattern_templates_variables'] || defaults['pattern_templates_variables'] + @owner = new_resource.owner || attributes['user'] || defaults['user'] + @group = new_resource.group || attributes['group'] || defaults['group'] + @templates_cookbook = new_resource.templates_cookbook || attributes['pattern_templates_cookbook'] || defaults['pattern_templates_cookbook'] + @mode = new_resource.mode || '0644' + @path = new_resource.path || "#{@basedir}/#{@instance}/patterns" +end + +action :create do + pattern = pattern_vars + # Chef::Log.info("config vars: #{pattern.inspect}") + pattern[:templates].each do |_template, file| + tp = template "#{pattern[:path]}/#{::File.basename(file).chomp(::File.extname(file))}" do + source file + cookbook pattern[:templates_cookbook] + owner pattern[:owner] + group pattern[:group] + mode pattern[:mode] + variables pattern[:variables] + notifies :restart, "logstash_service[#{pattern[:instance]}]" + action :create + end + new_resource.updated_by_last_action(tp.updated_by_last_action?) + end +end + +private + +def pattern_vars + pattern = { + instance: @instance, + templates: @templates, + variables: @variables, + path: @path, + owner: @owner, + group: @group, + mode: @mode, + templates_cookbook: @templates_cookbook + } + pattern +end diff --git a/providers/plugins.rb b/providers/plugins.rb new file mode 100644 index 0000000..167cc09 --- /dev/null +++ b/providers/plugins.rb @@ -0,0 +1,77 @@ +# Encoding: utf-8 +# Cookbook Name:: logstash +# Provider:: plugins +# Author:: John E. Vincent +# License:: Apache 2.0 +# +# Copyright 2014, John E. Vincent + +require 'chef/mixin/shell_out' +require 'chef/mixin/language' +include Chef::Mixin::ShellOut + +def load_current_resource + @name = new_resource.name || 'contrib' + @instance = new_resource.instance || 'default' + if node['logstash']['instance'].key?(@instance) + attributes = node['logstash']['instance'][@instance] + defaults = node['logstash']['instance']['default'] + else + attributes = node['logstash']['instance']['default'] + end + @base_directory = new_resource.base_directory || attributes['basedir'] || defaults['basedir'] + @version = new_resource.version || attributes['plugins_version'] || defaults['plugins_version'] + @checksum = new_resource.checksum || attributes['plugins_checksum'] || defaults['plugins_checksum'] + @source_url = new_resource.source_url || attributes['plugins_source_url'] || defaults['plugins_source_url'] + @user = new_resource.user || attributes['user'] || defaults['user'] + @group = new_resource.group || attributes['group'] || defaults['group'] + @instance_dir = "#{@base_directory}/#{@instance}" + @install_type = new_resource.install_type || attributes['plugins_install_type'] || defaults['plugins_install_type'] + @install_check = new_resource.install_check || attributes['plugins_check_if_installed'] || defaults['plugins_check_if_installed'] +end + +action :create do + ls_version = @version + ls_checksum = @checksum + ls_source_url = @source_url + ls_basedir = @base_directory + ls_user = @user + ls_group = @group + ls_name = @name + ls_instance = @instance + ls_instance_dir = @instance_dir + ls_install_check = @install_check + + case @install_type + when 'native' + ex = execute "bin/plugin install #{ls_name}" do + command "bin/plugin install #{ls_name}" + user ls_user + group ls_group + cwd ls_instance_dir + notifies :restart, "logstash_service[#{ls_instance}]" + # this is a temp workaround to make the plugin command idempotent. + not_if { ::File.exist?("#{ls_instance_dir}/#{ls_install_check}") } + end + new_resource.updated_by_last_action(ex.updated_by_last_action?) + when 'tarball' + @run_context.include_recipe 'ark::default' + arkit = ark "#{ls_instance}_contrib" do + name ls_instance + url ls_source_url + checksum ls_checksum + owner ls_user + group ls_group + mode 0755 + version ls_version + path ls_basedir + action [:put] + notifies :restart, "logstash_service[#{ls_instance}]" + # this is a temp workaround to ensure idempotent. + not_if { ::File.exist?("#{ls_instance_dir}/#{ls_install_check}") } + end + new_resource.updated_by_last_action(arkit.updated_by_last_action?) + else + Chef::Application.fatal!("Unknown install type: #{@install_type}") + end +end diff --git a/providers/service.rb b/providers/service.rb new file mode 100644 index 0000000..825924c --- /dev/null +++ b/providers/service.rb @@ -0,0 +1,270 @@ +# Encoding: utf-8 +# Cookbook Name:: logstash +# Provider:: service +# Author:: John E. Vincent +# License:: Apache 2.0 +# +# Copyright 2014, John E. Vincent + +require 'chef/mixin/shell_out' +require 'chef/mixin/language' +include Chef::Mixin::ShellOut + +def load_current_resource + @instance = new_resource.instance + if node['logstash']['instance'].key?(@instance) + attributes = node['logstash']['instance'][@instance] + defaults = node['logstash']['instance']['default'] + else + attributes = node['logstash']['instance']['default'] + end + + @basedir = attributes['basedir'] || defaults['basedir'] + @templates_cookbook = new_resource.templates_cookbook || attributes['service_templates_cookbook'] || defaults['service_templates_cookbook'] + @service_name = new_resource.service_name || "logstash_#{@instance}" + @home = "#{@basedir}/#{@instance}" + @method = new_resource.method || attributes['init_method'] || defaults['init_method'] + @command = new_resource.command || "#{@home}/bin/logstash" + @user = new_resource.user || attributes['user'] || defaults['user'] + @group = new_resource.group || attributes['group'] || defaults['group'] + @log_file = attributes['log_file'] || defaults['log_file'] + @max_heap = attributes['xmx'] || defaults['xmx'] + @min_heap = attributes['xms'] || defaults['xms'] + @gc_opts = attributes['gc_opts'] || defaults['gc_opts'] + @ipv4_only = attributes['ipv4_only'] || defaults['ipv4_only'] + @java_opts = attributes['java_opts'] || defaults['java_opts'] + @description = new_resource.description || @service_name + @chdir = @home + @workers = attributes['workers'] || defaults['workers'] + @debug = attributes['debug'] || defaults['debug'] + @install_type = attributes['install_type'] || defaults['install_type'] + @supervisor_gid = attributes['supervisor_gid'] || defaults['supervisor_gid'] +end + +action :restart do + svc = svc_vars + case svc[:method] + when 'native' + sv = service svc[:service_name] do + # provider Chef::Provider::Service::Upstart + action [:restart] + end + new_resource.updated_by_last_action(sv.updated_by_last_action?) + end +end + +action :start do + svc = svc_vars + case svc[:method] + when 'native' + sv = service svc[:service_name] do + # provider Chef::Provider::Service::Upstart + action [:start] + end + new_resource.updated_by_last_action(sv.updated_by_last_action?) + end +end + +action :stop do + svc = svc_vars + case svc[:method] + when 'native' + sv = service svc[:service_name] do + # provider Chef::Provider::Service::Upstart + action [:stop] + end + new_resource.updated_by_last_action(sv.updated_by_last_action?) + end +end + +action :reload do + svc = svc_vars + case svc[:method] + when 'native' + sv = service svc[:service_name] do + # provider Chef::Provider::Service::Upstart + action [:reload] + end + new_resource.updated_by_last_action(sv.updated_by_last_action?) + end +end + +action :enable do + svc = svc_vars + Chef::Log.info("Using init method #{svc[:method]} for #{svc[:service_name]}") + case svc[:method] + when 'pleaserun' + @run_context.include_recipe 'pleaserun::default' + pr = pleaserun svc[:service_name] do + name svc[:service_name] + program svc[:command] + args default_args + description svc[:description] + chdir svc[:chdir] + user svc[:user] + group svc[:group] + action :create + not_if { ::File.exist?("/etc/init.d/#{svc[:service_name]}") } + end + new_resource.updated_by_last_action(pr.updated_by_last_action?) + when 'runit' + @run_context.include_recipe 'runit::default' + ri = runit_service svc[:service_name] do + options( + name: svc[:name], + home: svc[:home], + max_heap: svc[:max_heap], + min_heap: svc[:min_heap], + gc_opts: svc[:gc_opts], + java_opts: svc[:java_opts], + ipv4_only: svc[:ipv4_only], + debug: svc[:debug], + log_file: svc[:log_file], + workers: svc[:workers], + install_type: svc[:install_type], + supervisor_gid: svc[:supervisor_gid], + user: svc[:user], + web_address: svc[:web_address], + web_port: svc[:web_port] + ) + cookbook svc[:templates_cookbook] + end + new_resource.updated_by_last_action(ri.updated_by_last_action?) + when 'native' + if platform_family? 'debian' + if node['platform_version'] >= '12.04' + args = default_args + tp = template "/etc/init/#{svc[:service_name]}.conf" do + mode '0644' + source "init/#{svc[:install_type]}_upstart.erb" + cookbook svc[:templates_cookbook] + variables( + home: svc[:home], + name: svc[:name], + command: svc[:command], + args: args, + user: svc[:user], + group: svc[:group], + description: svc[:description], + max_heap: svc[:max_heap], + min_heap: svc[:min_heap], + gc_opts: svc[:gc_opts], + java_opts: svc[:java_opts], + ipv4_only: svc[:ipv4_only], + debug: svc[:debug], + log_file: svc[:log_file], + workers: svc[:workers], + supervisor_gid: svc[:supervisor_gid] + ) + end + new_resource.updated_by_last_action(tp.updated_by_last_action?) + sv = service svc[:service_name] do + provider Chef::Provider::Service::Upstart + supports restart: true, reload: true, start: true, stop: true + action [:enable] + end + new_resource.updated_by_last_action(sv.updated_by_last_action?) + else + Chef::Log.fatal("Please set node['logstash']['instance']['server']['init_method'] to 'runit' for #{node['platform_version']}") + end + elsif (platform_family? 'fedora') && (node['platform_version'] >= '15') + ex = execute 'reload-systemd' do + command 'systemctl --system daemon-reload' + action :nothing + end + new_resource.updated_by_last_action(ex.updated_by_last_action?) + template '/etc/systemd/system/logstash_server.service' do + tp = source 'logstash_server.service.erb' + cookbook svc[:templates_cookbook] + owner 'root' + group 'root' + mode '0755' + notifies :run, 'execute[reload-systemd]', :immediately + notifies :restart, 'service[logstash_server]', :delayed + end + new_resource.updated_by_last_action(tp.updated_by_last_action?) + + sv = service 'logstash_server' do + service_name 'logstash_server.service' + provider Chef::Provider::Service::Systemd + action [:enable, :start] + end + new_resource.updated_by_last_action(sv.updated_by_last_action?) + + elsif platform_family? 'rhel', 'fedora' + args = default_args + tp = template "/etc/init.d/#{svc[:service_name]}" do + source "init/#{svc[:install_type]}_init.logstash.erb" + cookbook svc[:templates_cookbook] + owner 'root' + group 'root' + mode '0774' + variables( + home: svc[:home], + name: svc[:name], + command: svc[:command], + args: args, + user: svc[:user], + group: svc[:group], + description: svc[:description], + max_heap: svc[:max_heap], + min_heap: svc[:min_heap], + gc_opts: svc[:gc_opts], + java_opts: svc[:java_opts], + ipv4_only: svc[:ipv4_only], + debug: svc[:debug], + log_file: svc[:log_file], + workers: svc[:workers], + supervisor_gid: svc[:supervisor_gid], + config_file: "#{svc[:home]}/etc/conf.d", + group: svc[:group] + ) + end + new_resource.updated_by_last_action(tp.updated_by_last_action?) + + sv = service svc[:service_name] do + supports restart: true, reload: true, status: true + action [:enable, :start] + end + new_resource.updated_by_last_action(sv.updated_by_last_action?) + end + else + Chef::Log.fatal("Unsupported init method: #{@svc[:method]}") + end +end + +private + +def default_args + svc = svc_vars + args = ['agent', '-f', "#{svc[:home]}/etc/conf.d/"] + args.concat ['-vv'] if svc[:debug] + args.concat ['-l', "#{svc[:home]}/log/#{svc[:log_file]}"] if svc[:log_file] + args.concat ['-w', svc[:workers].to_s] if svc[:workers] + args +end + +def svc_vars + svc = { + name: @instance, + service_name: @service_name, + home: @home, + method: @method, + command: @command, + description: @description, + chdir: @chdir, + user: @user, + group: @group, + log_file: @log_file, + max_heap: @max_heap, + min_heap: @min_heap, + java_opts: @java_opts, + ipv4_only: @ipv4_only, + workers: @workers, + debug: @debug, + install_type: @install_type, + supervisor_gid: @supervisor_gid, + templates_cookbook: @templates_cookbook + } + svc +end diff --git a/recipes/agent.rb b/recipes/agent.rb index 000bded..0157648 100644 --- a/recipes/agent.rb +++ b/recipes/agent.rb @@ -4,228 +4,29 @@ # Recipe:: agent # # -include_recipe 'logstash::default' -include_recipe 'yum::default' -if node['logstash']['agent']['init_method'] == 'runit' - include_recipe 'runit' - service_resource = 'runit_service[logstash_agent]' -else - service_resource = 'service[logstash_agent]' -end - -if node['logstash']['agent']['patterns_dir'][0] == '/' - patterns_dir = node['logstash']['agent']['patterns_dir'] -else - patterns_dir = node['logstash']['agent']['home'] + '/' + node['logstash']['agent']['patterns_dir'] -end - -if node['logstash']['install_zeromq'] - include_recipe 'logstash::zero_mq_repo' - node['logstash']['zeromq_packages'].each { |p| package p } -end - -# check if running chef-solo. If not, detect the logstash server/ip by role. If I can't do that, fall back to using ['logstash']['agent']['server_ipaddress'] -if Chef::Config[:solo] - logstash_server_ip = node['logstash']['agent']['server_ipaddress'] -else - logstash_server_results = search(:node, "roles:#{node['logstash']['agent']['server_role']}") - if !logstash_server_results.empty? - logstash_server_ip = logstash_server_results[0]['ipaddress'] - else - logstash_server_ip = node['logstash']['agent']['server_ipaddress'] - end -end +name = 'agent' -directory node['logstash']['agent']['home'] do - action :create - mode '0755' - owner node['logstash']['user'] - group node['logstash']['group'] -end - -%w{bin etc lib tmp log}.each do |ldir| - directory "#{node['logstash']['agent']['home']}/#{ldir}" do - action :create - mode '0755' - owner node['logstash']['user'] - group node['logstash']['group'] - end - - link "#{node['logstash']['homedir']}/#{ldir}" do - to "#{node['logstash']['agent']['home']}/#{ldir}" - end -end +Chef::Application.fatal!("attribute hash node['logstash']['instance']['#{name}'] must exist.") if node['logstash']['instance'][name].nil? -directory "#{node['logstash']['agent']['home']}/etc/conf.d" do - action :create - mode '0755' - owner node['logstash']['user'] - group node['logstash']['group'] +# these should all default correctly. listing out for example. +logstash_instance name do + action :create end -directory patterns_dir do - action :create - mode '0755' - owner node['logstash']['user'] - group node['logstash']['group'] +# services are hard! Let's go LWRP'ing. FIREBALL! FIREBALL! FIREBALL! +logstash_service name do + action [:enable, :start] end -node['logstash']['patterns'].each do |file, hash| - template_name = patterns_dir + '/' + file - template template_name do - source 'patterns.erb' - owner node['logstash']['user'] - group node['logstash']['group'] - variables(:patterns => hash) - mode '0644' - notifies :restart, service_resource - end -end - -if node['logstash']['agent']['install_method'] == 'jar' - remote_file "#{node['logstash']['agent']['home']}/lib/logstash-#{node['logstash']['agent']['version']}.jar" do - owner 'root' - group 'root' - mode '0755' - source node['logstash']['agent']['source_url'] - checksum node['logstash']['agent']['checksum'] - action :create_if_missing - end - - link "#{node['logstash']['agent']['home']}/lib/logstash.jar" do - to "#{node['logstash']['agent']['home']}/lib/logstash-#{node['logstash']['agent']['version']}.jar" - notifies :restart, service_resource - end -else - include_recipe 'logstash::source' - - logstash_version = node['logstash']['source']['sha'] || "v#{node['logstash']['server']['version']}" - link "#{node['logstash']['agent']['home']}/lib/logstash.jar" do - to "#{node['logstash']['basedir']}/source/build/logstash-#{logstash_version}-monolithic.jar" - notifies :restart, service_resource - end -end - -template "#{node['logstash']['agent']['home']}/#{node['logstash']['agent']['config_dir']}/#{node['logstash']['agent']['config_file']}" do - source node['logstash']['agent']['base_config'] - cookbook node['logstash']['agent']['base_config_cookbook'] - owner node['logstash']['user'] - group node['logstash']['group'] - mode '0644' +logstash_config name do variables( - :logstash_server_ip => logstash_server_ip, - :patterns_dir => patterns_dir) - notifies :restart, service_resource - only_if { node['logstash']['agent']['config_file'] } -end - -unless node['logstash']['agent']['config_templates'].empty? || node['logstash']['agent']['config_templates'].nil? - node['logstash']['agent']['config_templates'].each do |config_template| - template "#{node['logstash']['agent']['home']}/#{node['logstash']['agent']['config_dir']}/#{config_template}.conf" do - source "#{config_template}.conf.erb" - cookbook node['logstash']['agent']['config_templates_cookbook'] - owner node['logstash']['user'] - group node['logstash']['group'] - mode '0644' - variables node['logstash']['agent']['config_templates_variables'][config_template] - notifies :restart, service_resource - action :create - end - end -end - -log_dir = ::File.dirname node['logstash']['agent']['log_file'] -directory log_dir do - action :create - mode '0755' - owner node['logstash']['user'] - group node['logstash']['group'] - recursive true -end - -if node['logstash']['agent']['init_method'] == 'runit' - runit_service 'logstash_agent' -elsif node['logstash']['agent']['init_method'] == 'native' - if platform_family? 'debian' - if node['platform_version'] >= '12.04' - template '/etc/init/logstash_agent.conf' do - mode '0644' - source 'logstash_agent.conf.erb' - notifies :restart, service_resource - end - - service 'logstash_agent' do - provider Chef::Provider::Service::Upstart - action [:enable, :start] - end - else - Chef::Log.fatal("Please set node['logstash']['agent']['init_method'] to 'runit' for #{node['platform_version']}") - end - elsif platform_family? 'fedora' && node['platform_version'] >= '15' - execute 'reload-systemd' do - command 'systemctl --system daemon-reload' - action :nothing - end - - template '/etc/systemd/system/logstash_agent.service' do - source 'logstash_agent.service.erb' - owner 'root' - group 'root' - mode '0755' - notifies :run, 'execute[reload-systemd]', :immediately - notifies :restart, 'service[logstash_agent]', :delayed - end - - service 'logstash_agent' do - service_name 'logstash_agent.service' - provider Chef::Provider::Service::Systemd - action [:enable, :start] - end - elsif platform_family? 'rhel', 'fedora' - template '/etc/init.d/logstash_agent' do - source 'init.logstash_server.erb' - owner 'root' - group 'root' - mode '0774' - variables( - :config_file => node['logstash']['agent']['config_dir'], - :home => node['logstash']['agent']['home'], - :log_file => node['logstash']['agent']['log_file'], - :name => 'agent', - :max_heap => node['logstash']['agent']['xmx'], - :min_heap => node['logstash']['agent']['xms'] - ) - end - - service 'logstash_agent' do - supports :restart => true, :reload => true, :status => true - action :enable - end - end -else - Chef::Log.fatal("Unsupported init method: #{node['logstash']['server']['init_method']}") + input_file_name: '/var/log/syslog', + input_file_type: 'syslog' + ) + action [:create] end -logrotate_app 'logstash' do - path "#{log_dir}/*.log" - if node['logstash']['logging']['useFileSize'] - size node['logstash']['logging']['maxSize'] - else - frequency node['logstash']['logging']['rotateFrequency'] - end - rotate node['logstash']['logging']['maxBackup'] - options node['logstash']['agent']['logrotate']['options'] - create "664 #{node['logstash']['user']} #{node['logstash']['group']}" - notifies :restart, 'service[rsyslog]' - if node['logstash']['agent']['logrotate']['stopstartprepost'] - prerotate <<-EOF - service logstash_agent stop - logger stopped logstash_agent service for log rotation - EOF - postrotate <<-EOF - service logstash_agent start - logger started logstash_agent service after log rotation - EOF - end +logstash_pattern name do + action [:create] end diff --git a/recipes/beaver.rb b/recipes/beaver.rb index 08dc735..4b1a5bb 100644 --- a/recipes/beaver.rb +++ b/recipes/beaver.rb @@ -28,7 +28,7 @@ logstash_server_ip = nil if Chef::Config[:solo] logstash_server_ip = node['logstash']['beaver']['server_ipaddress'] if node['logstash']['beaver']['server_ipaddress'] -elsif !node['logstash']['beaver']['server_ipaddress'].nil? +elsif node['logstash']['beaver']['server_ipaddress'] logstash_server_ip = node['logstash']['beaver']['server_ipaddress'] elsif node['logstash']['beaver']['server_role'] logstash_server_results = search(:node, "roles:#{node['logstash']['beaver']['server_role']}") @@ -45,13 +45,13 @@ [ File.dirname(conf_file), File.dirname(log_file), - File.dirname(pid_file), + File.dirname(pid_file) ].each do |dir| directory dir do owner node['logstash']['user'] group node['logstash']['group'] recursive true - not_if { ::File.exists?(dir) } + not_if { ::File.exist?(dir) } end end @@ -139,8 +139,8 @@ owner node['logstash']['user'] group node['logstash']['group'] variables( - :conf => conf, - :files => files + conf: conf, + files: files ) notifies :restart, 'service[logstash_beaver]' end @@ -163,17 +163,17 @@ mode '0644' source 'logstash_beaver.conf.erb' variables( - :cmd => cmd, - :group => node['logstash']['supervisor_gid'], - :user => node['logstash']['user'], - :log => log_file, - :supports_setuid => supports_setuid + cmd: cmd, + group: node['logstash']['supervisor_gid'], + user: node['logstash']['user'], + log: log_file, + supports_setuid: supports_setuid ) notifies :restart, 'service[logstash_beaver]' end service 'logstash_beaver' do - supports :restart => true, :reload => false + supports restart: true, reload: false action [:enable, :start] provider Chef::Provider::Service::Upstart end @@ -182,17 +182,17 @@ mode '0755' source 'init-beaver.erb' variables( - :cmd => cmd, - :pid_file => pid_file, - :user => node['logstash']['user'], - :log => log_file, - :platform => node['platform'] + cmd: cmd, + pid_file: pid_file, + user: node['logstash']['user'], + log: log_file, + platform: node['platform'] ) notifies :restart, 'service[logstash_beaver]' end service 'logstash_beaver' do - supports :restart => true, :reload => false, :status => true + supports restart: true, reload: false, status: true action [:enable, :start] end end diff --git a/recipes/default.rb b/recipes/default.rb index cce024f..d5d94ae 100644 --- a/recipes/default.rb +++ b/recipes/default.rb @@ -3,46 +3,3 @@ # Cookbook Name:: logstash # Recipe:: default # -include_recipe 'runit' unless node['platform_version'] >= '12.04' - -if node['logstash']['create_account'] - - group node['logstash']['group'] do - system true - gid node['logstash']['gid'] - end - - user node['logstash']['user'] do - group node['logstash']['group'] - home node['logstash']['homedir'] - system true - action :create - manage_home true - uid node['logstash']['uid'] - end - -else - directory node['logstash']['homedir'] do - recursive true - action :create - group node['logstash']['group'] - owner node['logstash']['user'] - mode '0755' - end -end - -directory node['logstash']['basedir'] do - action :create - owner 'root' - group 'root' - mode '0755' -end - -node['logstash']['join_groups'].each do |grp| - group grp do - members node['logstash']['user'] - action :modify - append true - only_if "grep -q '^#{grp}:' /etc/group" - end -end diff --git a/recipes/haproxy.rb b/recipes/haproxy.rb deleted file mode 100644 index 84e38c2..0000000 --- a/recipes/haproxy.rb +++ /dev/null @@ -1,45 +0,0 @@ -# Encoding: utf-8 -# this recipe lets output haproxy logs to file as if they were apache -# virtual logs, in order to interface with legacy traffic measuring -# applications like AWstats -# Also requires changes to your haproxy configuration -# and a file output on your logstash_server -# I have no idea if it meets anyone's needs other than my own -# only for those crazy enough to replace apache or Nginx as their -# main front-end server - Bryan W. Berry 28 June 2012 - -include_recipe 'logrotate' - -directory "#{node['logstash']['server']['home']}/apache_logs" do - action :create - mode '0755' - owner node['logstash']['user'] - group node['logstash']['group'] -end - -apache_logs = "#{node['logstash']['homedir']}/apache_logs" -link apache_logs do - to "#{node['logstash']['server']['home']}/apache_logs" -end - -directory "#{node['logstash']['server']['home']}/etc/patterns" do - owner node['logstash']['user'] - group node['logstash']['group'] - mode '0774' -end - -# create pattern_file for haproxy -cookbook_file "#{node['logstash']['server']['home']}/etc/patterns/haproxy" do - source 'haproxy' - owner node['logstash']['user'] - group node['logstash']['group'] - mode '0774' -end - -# set logrotate for /opt/logstash/server/apache_logs -logrotate_app 'apache_logs' do - path node['logstash']['server']['logrotate_target'] - frequency 'daily' - create "664 #{node['logstash']['user']} #{node['logstash']['user']}" - rotate '30' -end diff --git a/recipes/index_cleaner.rb b/recipes/index_cleaner.rb deleted file mode 100644 index afdba6b..0000000 --- a/recipes/index_cleaner.rb +++ /dev/null @@ -1,38 +0,0 @@ -# Encoding: utf-8 -include_recipe 'python::pip' - -base_dir = File.join(node['logstash']['basedir'], 'index_cleaner') -index_cleaner_bin = File.join(base_dir, 'logstash_index_cleaner.py') -days_to_keep = node['logstash']['index_cleaner']['days_to_keep'] -log_file = node['logstash']['index_cleaner']['cron']['log_file'] - -python_pip 'pyes' do - action :install -end - -directory base_dir do - action :create - mode '0755' - owner node['logstash']['user'] - group node['logstash']['group'] -end - -cookbook_file index_cleaner_bin do - source 'logstash_index_cleaner.py' - owner node['logstash']['user'] - group node['logstash']['group'] - mode '0774' -end - -# FIXME: http://tickets.opscode.com/browse/CHEF-3547 -file index_cleaner_bin do - mode '0744' - action :touch - only_if { Chef::VERSION == '10.16.0' } -end - -cron 'logstash_index_cleaner' do - command "#{index_cleaner_bin} -d #{days_to_keep} &> #{log_file}" - minute node['logstash']['index_cleaner']['cron']['minute'] - hour node['logstash']['index_cleaner']['cron']['hour'] -end diff --git a/recipes/pyshipper.rb b/recipes/pyshipper.rb index 50516ba..d5a9fe8 100644 --- a/recipes/pyshipper.rb +++ b/recipes/pyshipper.rb @@ -20,7 +20,7 @@ action :sync end -%w{pyzmq-static simplejson argparse}.each do |ppkg| +%w(pyzmq-static simplejson argparse).each do |ppkg| python_pip ppkg do action :install end diff --git a/recipes/server.rb b/recipes/server.rb index f6310f3..b3cd0d8 100644 --- a/recipes/server.rb +++ b/recipes/server.rb @@ -10,232 +10,54 @@ # # -include_recipe 'logstash::default' -include_recipe 'logrotate' +# install logstash 'server' -include_recipe 'rabbitmq' if node['logstash']['install_rabbitmq'] +name = 'server' -if node['logstash']['install_zeromq'] - include_recipe 'logstash::zero_mq_repo' - node['logstash']['zeromq_packages'].each { |p| package p } -end +Chef::Application.fatal!("attribute hash node['logstash']['instance']['#{name}'] must exist.") if node['logstash']['instance'][name].nil? -if node['logstash']['server']['init_method'] == 'runit' - include_recipe 'runit' - service_resource = 'runit_service[logstash_server]' -else - service_resource = 'service[logstash_server]' +# these should all default correctly. listing out for example. +logstash_instance name do + action :create end -if node['logstash']['server']['patterns_dir'][0] == '/' - patterns_dir = node['logstash']['server']['patterns_dir'] -else - patterns_dir = node['logstash']['server']['home'] + '/' + node['logstash']['server']['patterns_dir'] +# services are hard! Let's go LWRP'ing. FIREBALL! FIREBALL! FIREBALL! +logstash_service name do + action [:enable] end -if Chef::Config[:solo] - es_server_ip = node['logstash']['elasticsearch_ip'] - graphite_server_ip = node['logstash']['graphite_ip'] -else - es_results = search(:node, node['logstash']['elasticsearch_query']) - graphite_results = search(:node, node['logstash']['graphite_query']) - - if !es_results.empty? - es_server_ip = es_results[0]['ipaddress'] - else - es_server_ip = node['logstash']['elasticsearch_ip'] - end - - if !graphite_results.empty? - graphite_server_ip = graphite_results[0]['ipaddress'] - else - graphite_server_ip = node['logstash']['graphite_ip'] - end -end +my_templates = node['logstash']['instance']['default']['config_templates'] -# Create directory for logstash -directory node['logstash']['server']['home'] do - action :create - mode '0755' - owner node['logstash']['user'] - group node['logstash']['group'] +if my_templates.empty? + my_templates = { + 'input_syslog' => 'config/input_syslog.conf.erb', + 'output_stdout' => 'config/output_stdout.conf.erb', + 'output_elasticsearch' => 'config/output_elasticsearch.conf.erb' + } end -%w{bin etc lib log tmp }.each do |ldir| - directory "#{node['logstash']['server']['home']}/#{ldir}" do - action :create - mode '0755' - owner node['logstash']['user'] - group node['logstash']['group'] - end -end - -# installation -if node['logstash']['server']['install_method'] == 'jar' - remote_file "#{node['logstash']['server']['home']}/lib/logstash-#{node['logstash']['server']['version']}.jar" do - owner 'root' - group 'root' - mode '0755' - source node['logstash']['server']['source_url'] - checksum node['logstash']['server']['checksum'] - action :create_if_missing - end - - link "#{node['logstash']['server']['home']}/lib/logstash.jar" do - to "#{node['logstash']['server']['home']}/lib/logstash-#{node['logstash']['server']['version']}.jar" - notifies :restart, service_resource - end -else - include_recipe 'logstash::source' - - logstash_version = node['logstash']['source']['sha'] || "v#{node['logstash']['server']['version']}" - link "#{node['logstash']['server']['home']}/lib/logstash.jar" do - to "#{node['logstash']['basedir']}/source/build/logstash-#{logstash_version}-monolithic.jar" - notifies :restart, service_resource - end -end - -directory "#{node['logstash']['server']['home']}/etc/conf.d" do - action :create - mode '0755' - owner node['logstash']['user'] - group node['logstash']['group'] -end - -directory patterns_dir do - action :create - mode '0755' - owner node['logstash']['user'] - group node['logstash']['group'] -end - -node['logstash']['patterns'].each do |file, hash| - template_name = patterns_dir + '/' + file - template template_name do - source 'patterns.erb' - owner node['logstash']['user'] - group node['logstash']['group'] - variables(:patterns => hash) - mode '0644' - notifies :restart, service_resource - end -end - -log_dir = ::File.dirname node['logstash']['server']['log_file'] -directory log_dir do - action :create - mode '0755' - owner node['logstash']['user'] - group node['logstash']['group'] - recursive true -end - -template "#{node['logstash']['server']['home']}/#{node['logstash']['server']['config_dir']}/#{node['logstash']['server']['config_file']}" do - source node['logstash']['server']['base_config'] - cookbook node['logstash']['server']['base_config_cookbook'] - owner node['logstash']['user'] - group node['logstash']['group'] - mode '0644' +logstash_config name do + templates my_templates + action [:create] variables( - :graphite_server_ip => graphite_server_ip, - :es_server_ip => es_server_ip, - :enable_embedded_es => node['logstash']['server']['enable_embedded_es'], - :es_cluster => node['logstash']['elasticsearch_cluster'], - :patterns_dir => patterns_dir - ) - notifies :restart, service_resource - action :create - only_if { node['logstash']['server']['config_file'] } + elasticsearch_embedded: true + ) end +# ^ see `.kitchen.yml` for example attributes to configure templates. -unless node['logstash']['server']['config_templates'].empty? || node['logstash']['server']['config_templates'].nil? - node['logstash']['server']['config_templates'].each do |config_template| - template "#{node['logstash']['server']['home']}/#{node['logstash']['server']['config_dir']}/#{config_template}.conf" do - source "#{config_template}.conf.erb" - cookbook node['logstash']['server']['config_templates_cookbook'] - owner node['logstash']['user'] - group node['logstash']['group'] - mode '0644' - variables node['logstash']['server']['config_templates_variables'][config_template] - notifies :restart, service_resource - action :create - end - end +logstash_plugins 'contrib' do + instance name + action [:create] end -services = ['server'] -services << 'web' if node['logstash']['server']['web']['enable'] - -services.each do |type| - if node['logstash']['server']['init_method'] == 'runit' - runit_service("logstash_#{type}") - elsif node['logstash']['server']['init_method'] == 'native' - if platform_family? 'debian' - if node['platform_version'] >= '12.04' - template "/etc/init/logstash_#{type}.conf" do - mode '0644' - source "logstash_#{type}.conf.erb" - end - - service "logstash_#{type}" do - provider Chef::Provider::Service::Upstart - action [:enable, :start] - end - else - Chef::Log.fatal("Please set node['logstash']['server']['init_method'] to 'runit' for #{node['platform_version']}") - end - - elsif platform_family? 'fedora' && node['platform_version'] >= '15' - execute 'reload-systemd' do - command 'systemctl --system daemon-reload' - action :nothing - end - - template '/etc/systemd/system/logstash_server.service' do - source 'logstash_server.service.erb' - owner 'root' - group 'root' - mode '0755' - notifies :run, 'execute[reload-systemd]', :immediately - notifies :restart, 'service[logstash_server]', :delayed - end - - service 'logstash_server' do - service_name 'logstash_server.service' - provider Chef::Provider::Service::Systemd - action [:enable, :start] - end - - elsif platform_family? 'rhel', 'fedora' - template "/etc/init.d/logstash_#{type}" do - source "init.logstash_#{type}.erb" - owner 'root' - group 'root' - mode '0774' - variables(:config_file => node['logstash']['server']['config_dir'], - :home => node['logstash']['server']['home'], - :name => type, - :log_file => node['logstash']['server']['log_file'], - :max_heap => node['logstash']['server']['xmx'], - :min_heap => node['logstash']['server']['xms'] - ) - end +logstash_pattern name do + action [:create] +end - service "logstash_#{type}" do - supports :restart => true, :reload => true, :status => true - action [:enable, :start] - end - end - else - Chef::Log.fatal("Unsupported init method: #{node['logstash']['server']['init_method']}") - end +logstash_service name do + action [:start] end -logrotate_app 'logstash_server' do - path "#{log_dir}/*.log" - size node['logstash']['logging']['maxSize'] if node['logstash']['logging']['useFileSize'] - frequency node['logstash']['logging']['rotateFrequency'] - rotate node['logstash']['logging']['maxBackup'] - options node['logstash']['server']['logrotate']['options'] - create "664 #{node['logstash']['user']} #{node['logstash']['group']}" +logstash_curator 'server' do + action [:create] end diff --git a/recipes/source.rb b/recipes/source.rb index aa395ee..d8828c8 100644 --- a/recipes/source.rb +++ b/recipes/source.rb @@ -26,7 +26,7 @@ execute 'build-logstash' do cwd "#{node['logstash']['basedir']}/source" environment( - :JAVA_HOME => node['logstash']['source']['java_home'] + JAVA_HOME: node['logstash']['source']['java_home'] ) user 'root' # This variant is useful for troubleshooting stupid environment problems diff --git a/recipes/test.rb b/recipes/test.rb new file mode 100644 index 0000000..665bee2 --- /dev/null +++ b/recipes/test.rb @@ -0,0 +1,12 @@ +# Encoding: utf-8 +# +# Cookbook Name:: logstash +# Recipe:: default +# + +logstash_instance 'tarball' do + version '1.4.0.rc1' + checksum 'b015fa130d589af957c9a48e6f59754f5c0954835abf44bd013547a6b6520e59' + source_url 'https://download.elasticsearch.org/logstash/logstash/logstash-1.4.0.rc1.tar.gz' + install_type 'tarball' +end diff --git a/recipes/zero_mq_repo.rb b/recipes/zero_mq_repo.rb deleted file mode 100644 index f5247e9..0000000 --- a/recipes/zero_mq_repo.rb +++ /dev/null @@ -1,37 +0,0 @@ -# Encoding: utf-8 -# -# Cookbook Name:: logstash -# Recipe:: zero_mq_repo -# -# -include_recipe 'yum::default' - -major_version = node['platform_version'].split('.').first.to_i - -case -when platform_family?('rhel') - yum_repository 'zeromq' do - description 'zeromq repo' - baseurl "http://download.opensuse.org/repositories/home:/fengshuo:/zeromq/CentOS_CentOS-#{major_version}/" - gpgkey "http://download.opensuse.org/repositories/home:/fengshuo:/zeromq/CentOS_CentOS-#{major_version}/repodata/repomd.xml.key" - action :create - end -when platform_family?('debian') - apt_repository 'zeromq-ppa' do - uri 'http://ppa.launchpad.net/chris-lea/zeromq/ubuntu' - distribution node['lsb']['codename'] - components ['main'] - keyserver 'keyserver.ubuntu.com' - key 'C7917B12' - action :add - end - apt_repository 'libpgm-ppa' do - uri 'http://ppa.launchpad.net/chris-lea/libpgm/ubuntu' - distribution node['lsb']['codename'] - components ['main'] - keyserver 'keyserver.ubuntu.com' - key 'C7917B12' - action :add - notifies :run, 'execute[apt-get update]', :immediately - end -end diff --git a/resources/config.rb b/resources/config.rb new file mode 100644 index 0000000..78b77f0 --- /dev/null +++ b/resources/config.rb @@ -0,0 +1,20 @@ +# Encoding: utf-8 +# Cookbook Name:: logstash +# Resource:: config +# Author:: John E. Vincent +# Copyright 2014, John E. Vincent +# License:: Apache 2.0 + +actions :create + +default_action :create if defined?(default_action) + +attribute :instance, kind_of: String, name_attribute: true +attribute :service_name, kind_of: String +attribute :templates, kind_of: Hash +attribute :variables, kind_of: Hash +attribute :owner, kind_of: String +attribute :group, kind_of: String +attribute :mode, kind_of: String +attribute :path, kind_of: String +attribute :templates_cookbook, kind_of: String diff --git a/resources/curator.rb b/resources/curator.rb new file mode 100644 index 0000000..9d3bf70 --- /dev/null +++ b/resources/curator.rb @@ -0,0 +1,17 @@ +# Encoding: utf-8 +# Cookbook Name:: logstash +# Resource:: config +# Author:: John E. Vincent +# Copyright 2014, John E. Vincent +# License:: Apache 2.0 + +actions :create, :delete + +default_action :create if defined?(default_action) + +attribute :instance, kind_of: String, name_attribute: true +attribute :days_to_keep, kind_of: String +attribute :minute, kind_of: String +attribute :hour, kind_of: String +attribute :log_file, kind_of: String +attribute :user, kind_of: String diff --git a/resources/instance.rb b/resources/instance.rb new file mode 100644 index 0000000..5405330 --- /dev/null +++ b/resources/instance.rb @@ -0,0 +1,32 @@ +# Encoding: utf-8 +# Cookbook Name:: logstash +# Resource:: instance +# Author:: John E. Vincent +# Copyright 2014, John E. Vincent +# License:: Apache 2.0 + +actions :create, :delete + +default_action :create if defined?(default_action) + +attribute :name, kind_of: String, name_attribute: true +attribute :base_directory, kind_of: String +attribute :install_type, kind_of: String +attribute :auto_symlink, kind_of: [TrueClass, FalseClass], default: true +# version/checksum/source_url used by `jar`, `tarball` install_type +attribute :version, kind_of: String +attribute :checksum, kind_of: String +attribute :source_url, kind_of: String +# sha/repo/java_home used by `source` install_type +attribute :sha, kind_of: String, default: 'HEAD' +attribute :repo, kind_of: String, default: 'git://github.com/logstash/logstash.git' +attribute :java_home, kind_of: String, default: '/usr/lib/jvm/java-6-openjdk' # openjdk6 on ubuntu +attribute :user, kind_of: String +attribute :group, kind_of: String +attribute :logrotate_enable, kind_of: [TrueClass, FalseClass] +attribute :user_opts, kind_of: [Hash] +attribute :logrotate_size, kind_of: [String] +attribute :logrotate_use_filesize, kind_of: [TrueClass, FalseClass] +attribute :logrotate_frequency, kind_of: [String] +attribute :logrotate_max_backup, kind_of: [Integer] +attribute :logrotate_options, kind_of: [String] diff --git a/resources/pattern.rb b/resources/pattern.rb new file mode 100644 index 0000000..4d3361f --- /dev/null +++ b/resources/pattern.rb @@ -0,0 +1,19 @@ +# Encoding: utf-8 +# Cookbook Name:: logstash +# Resource:: patterns +# Author:: John E. Vincent +# Copyright 2014, John E. Vincent +# License:: Apache 2.0 + +actions :create + +default_action :create if defined?(default_action) + +attribute :instance, kind_of: String, name_attribute: true +attribute :templates, kind_of: Hash +attribute :variables, kind_of: Hash +attribute :path, kind_of: String +attribute :owner, kind_of: String +attribute :group, kind_of: String +attribute :mode, kind_of: String +attribute :templates_cookbook, kind_of: String diff --git a/resources/plugins.rb b/resources/plugins.rb new file mode 100644 index 0000000..b6e9d3e --- /dev/null +++ b/resources/plugins.rb @@ -0,0 +1,21 @@ +# Encoding: utf-8 +# Cookbook Name:: logstash +# Resource:: instance +# Author:: John E. Vincent +# Copyright 2014, John E. Vincent +# License:: Apache 2.0 + +actions :create + +default_action :create if defined?(default_action) + +attribute :name, kind_of: String, name_attribute: true +attribute :instance, kind_of: String +attribute :version, kind_of: String +attribute :checksum, kind_of: String +attribute :source_url, kind_of: String +attribute :user, kind_of: String +attribute :group, kind_of: String +attribute :base_directory, kind_of: String +attribute :install_type, kind_of: String, default: 'native' +attribute :install_check, kind_of: String diff --git a/resources/service.rb b/resources/service.rb new file mode 100644 index 0000000..b335a32 --- /dev/null +++ b/resources/service.rb @@ -0,0 +1,20 @@ +# Encoding: utf-8 +# Cookbook Name:: logstash +# Resource:: instance +# Author:: John E. Vincent +# Copyright 2014, John E. Vincent +# License:: Apache 2.0 + +actions :enable, :start, :restart, :reload, :stop + +default_action :enable if defined?(default_action) + +attribute :instance, kind_of: String, name_attribute: true +attribute :service_name, kind_of: String +attribute :method, kind_of: String +attribute :command, kind_of: String +attribute :args, kind_of: Array +attribute :description, kind_of: String +attribute :user, kind_of: String +attribute :group, kind_of: String +attribute :templates_cookbook, kind_of: String diff --git a/templates/default/apache.conf.erb b/templates/default/config/filter_apache.conf.erb similarity index 84% rename from templates/default/apache.conf.erb rename to templates/default/config/filter_apache.conf.erb index 4fc9b95..01e628a 100644 --- a/templates/default/apache.conf.erb +++ b/templates/default/config/filter_apache.conf.erb @@ -7,8 +7,8 @@ filter { date { match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss" ] } - geoip { - source => "clientip" - } +# geoip { +# source => "clientip" +# } } } \ No newline at end of file diff --git a/templates/default/config/input_file.conf.erb b/templates/default/config/input_file.conf.erb new file mode 100644 index 0000000..9a5df6d --- /dev/null +++ b/templates/default/config/input_file.conf.erb @@ -0,0 +1,6 @@ +input { + file { + path => ["<%= @input_file_name %>"] + type => ["<%= @input_file_type %>"] + } +} diff --git a/templates/default/syslog.conf.erb b/templates/default/config/input_syslog.conf.erb similarity index 60% rename from templates/default/syslog.conf.erb rename to templates/default/config/input_syslog.conf.erb index 0fb5220..1ae8174 100644 --- a/templates/default/syslog.conf.erb +++ b/templates/default/config/input_syslog.conf.erb @@ -1,10 +1,17 @@ +input { + tcp { + port => "5959" + type => "syslog" + } +} + filter { - if [type] == "<%= @type %>" { + if [type] == "syslog" { grok { overwrite => "message" match => [ "message", - "?%{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:host} (?:%{PROG:program}(?:\[%{POSINT:pid}\])?: )?%{GREEDYDATA:message}" + "%{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:host} (?:%{PROG:program}(?:\[%{POSINT:pid}\])?: )?%{GREEDYDATA:message}" ] } syslog_pri { } diff --git a/templates/default/config/output_elasticsearch.conf.erb b/templates/default/config/output_elasticsearch.conf.erb new file mode 100644 index 0000000..e72c6b6 --- /dev/null +++ b/templates/default/config/output_elasticsearch.conf.erb @@ -0,0 +1,19 @@ +output { + elasticsearch { +<% if @elasticsearch_ip -%> + host => "<%= @elasticsearch_ip %>" +<% end -%> +<% if @elasticsearch_cluster -%> + cluster => "<%= @elasticsearch_cluster %>" +<% end -%> +<% if @elasticsearch_embedded -%> + embedded => true +<% end -%> +<% if @bind_host -%> + bind_host => "<%= @bind_host %>" +<% end -%> +<% if @es_index -%> + index => "<%= @es_index %>" +<% end -%> + } +} diff --git a/templates/default/config/output_stdout.conf.erb b/templates/default/config/output_stdout.conf.erb new file mode 100644 index 0000000..ae89df0 --- /dev/null +++ b/templates/default/config/output_stdout.conf.erb @@ -0,0 +1,5 @@ +output { + stdout { + codec => rubydebug + } +} \ No newline at end of file diff --git a/templates/default/init/java_init.logstash.erb b/templates/default/init/java_init.logstash.erb new file mode 100755 index 0000000..b0ed39b --- /dev/null +++ b/templates/default/init/java_init.logstash.erb @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# +# logstash +# +# chkconfig: - 57 47 +# description: logstash +# processname: logstash + + +PIDDIR="/var/run" +export PIDFILE="/var/run/logstash-<%= @name %>.pid" +export LS_HOME="<%= @home %>" +export LS_CONFIG="<%= @config_file %>" +LS_USER="<%= @user %>" +LS_LOG="<%= @log_file %>" +LOGDIR="<%= ::File.dirname @log_file %>" +export JAVA_OPTS="-server -Xms<%= @min_heap %> -Xmx<%= @max_heap %> -Djava.io.tmpdir=$LS_HOME/tmp/ <%= @java_opts %> <%= '-Djava.net.preferIPv4Stack=true' if @ipv4_only %>" +BIN_SCRIPT="/usr/bin/env java $JAVA_OPTS -jar $LS_HOME/lib/logstash.jar agent -f $LS_CONFIG > $LS_LOG 2>&1 & echo \$! > $PIDFILE" + +if [ -f /etc/init.d/functions ] ; then + . /etc/init.d/functions +fi + +start() { + + if [ ! -d "$PIDDIR" ] ; then + mkdir "$PIDDIR" + chown -R $LS_USER:$LS_USER $PIDDIR + fi + + if [ ! -d "$LOGDIR" ] ; then + mkdir "$LOGDIR" + fi + + chown -R $LS_USER:$LS_USER $LOGDIR $PIDDIR + + + if [ -f $PIDFILE ]; then + echo -e "\033[31;1mPID file found in $PIDFILE, already running?\033[0m" + ls_pid="$(cat $PIDFILE)" + pid_running="$( ps ax | grep 'java' | grep $ls_pid )" + + if [ ! -z "$pid_running" ] ; then + echo -e "\033[31;1mPID $ls_pid still alive, logstash is already running. Doing nothing\033[0m" + return 1 + fi + fi + + echo -e "\033[1mStarting logstash...\033[0m" + pushd $LS_HOME > /dev/null 2>&1 + su $LS_USER -c "$BIN_SCRIPT" > /dev/null 2>&1 + ls_pid=$! + result=$? + popd > /dev/null 2>&1 + + if [ $result -ne 0 ] ; then + failure + echo -e "Logstash did not start successfully" + exit 1 + else + success + echo -e "Logstash started successfully" + fi + +} + + + +function stop() { + echo -n -e "\033[1mStopping logstash...\033[0m" + + if [ -z "$SHUTDOWN_WAIT" ]; then + SHUTDOWN_WAIT=5 + fi + + if [ ! -z "$PIDFILE" ]; then + if [ -f "$PIDFILE" ]; then + kill -0 `cat $PIDFILE` >/dev/null 2>&1 + if [ $? -gt 0 ]; then + echo "PID file ($PIDFILE) found but no matching process was found. Nothing to do." + return 0 + fi + else + echo "\$PIDFILE was set ($PIDFILE) but the specified file does not exist. Is Logstash running? Assuming it has stopped and pro\ + ceeding." + return 0 + fi + fi + + kill `cat $PIDFILE` >/dev/null 2>&1 + + if [ ! -z "$PIDFILE" ]; then + if [ -f "$PIDFILE" ]; then + while [ $SHUTDOWN_WAIT -ge 0 ]; do + kill -0 `cat $PIDFILE` >/dev/null 2>&1 + if [ $? -gt 0 ]; then + rm $PIDFILE + break + fi + if [ $SHUTDOWN_WAIT -gt 0 ]; then + sleep 1 + fi + SHUTDOWN_WAIT=`expr $SHUTDOWN_WAIT - 1 ` + done + # still not dead, we may need to resort to drastic measures + if [ -f "$PIDFILE" ]; then + kill -0 `cat $PIDFILE` >/dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "Application still alive, sleeping for 20 seconds before sending SIGKILL" + sleep 20 + kill -0 `cat $PIDFILE` >/dev/null 2>&1 + if [ $? -eq 0 ]; then + kill -9 `cat $PIDFILE` >/dev/null 2>&1 + echo "Killed with extreme prejudice" + else + echo "Application stopped, no need to use SIGKILL" + fi + rm $PIDFILE + fi + fi + fi + fi +} + +restart() { + stop + start +} + +status() { + # GOT PIDFILE? + [ -f $PIDFILE ] && pid=$(cat $PIDFILE) + + # RUNNING + if [[ $pid && -d "/proc/$pid" ]]; then + success + echo -e "Logstash is running with pid $pid" + fi + + # NOT RUNNING + if [[ ! $pid || ! -d "/proc/$pid" ]]; then + echo "Logstash not running" + fi + + # STALE PID FOUND + if [[ ! -d "/proc/$pid" && -f $PIDFILE ]]; then + echo -e "\033[1;31;40m[!] Stale PID found in $PIDFILE\033[0m" + fi +} + + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + restart + ;; + status) + status $2 + ;; + *) + echo $"Usage: $0 {start|stop|restart|status [-v]|}" + exit 1 +esac + +exit $? diff --git a/templates/default/init/java_upstart.erb b/templates/default/init/java_upstart.erb new file mode 100644 index 0000000..cdfdbfe --- /dev/null +++ b/templates/default/init/java_upstart.erb @@ -0,0 +1,42 @@ +description "Logstash" +author "Chef" + +start on (filesystem and net-device-up) +stop on runlevel [!2345] + +respawn +respawn limit 5 30 +limit nofile 65550 65550 + +chdir <%= @home %> + +<% unless node['logstash']['instance'][@name]['upstart_with_sudo'] -%> +setuid <%= @user %> + <% unless @supervisor_gid.to_s.empty? -%> +setgid <%= @supervisor_gid %> + <% end -%> +<% end -%> + +script + export LOGSTASH_HOME="<%= @home %>" + export HOME=$LOGSTASH_HOME + + export GC_OPTS="<%= @gc_opts %>" + export JAVA_OPTS="-server -Xms<%= @min_heap %> -Xmx<%= @max_heap %> -Djava.io.tmpdir=$LOGSTASH_HOME/tmp/ <%= @java_opts %> <%= '-Djava.net.preferIPv4Stack=true' if @ipv4_only %>" + export LOGSTASH_OPTS="agent -f $LOGSTASH_HOME/etc/conf.d" + export LOGSTASH_OPTS="$LOGSTASH_OPTS --pluginpath $LOGSTASH_HOME/lib" + export LOGSTASH_OPTS="$LOGSTASH_OPTS -w <%= @workers %>" + export LOGSTASH_OPTS="$LOGSTASH_OPTS -l $LOGSTASH_HOME/log/<%= @log_file %>" + <% if @debug -%> + export LOGSTASH_OPTS="$LOGSTASH_OPTS -vv" + <% end -%> + export OPTS="$JAVA_OPTS $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS" + + <% if node['logstash']['instance'][@name]['upstart_with_sudo'] -%> + exec sudo -u <%= @user %> /usr/bin/java $OPTS + <% else -%> + exec /usr/bin/java $OPTS + <% end -%> +end script + +emits logstash-server-running diff --git a/templates/default/init/tarball_init.logstash.erb b/templates/default/init/tarball_init.logstash.erb new file mode 100644 index 0000000..7991ec5 --- /dev/null +++ b/templates/default/init/tarball_init.logstash.erb @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# +# logstash +# +# chkconfig: - 57 47 +# description: logstash +# processname: logstash + + +PIDDIR="/var/run/logstash" +export PIDFILE="$PIDDIR/logstash-<%= @name %>.pid" +export LS_HOME="<%= @home %>" +export LS_HEAP_SIZE="<%= @max_heap %>" +export LOGSTASH_OPTS="<%= @args.join(' ') %>" +LS_USER="<%= @user %>" +LS_LOG="<%= @log_file %>" +LOGDIR="<%= ::File.dirname @log_file %>" +export JAVA_OPTS="-server -Xms<%= @min_heap %> -Xmx<%= @max_heap %> -Djava.io.tmpdir=$LS_HOME/tmp/ <%= @java_opts %> <%= '-Djava.net.preferIPv4Stack=true' if @ipv4_only %>" +BIN_SCRIPT="/usr/bin/env $LS_HOME/bin/logstash $LOGSTASH_OPTS > $LS_LOG 2>&1 & echo \$! > $PIDFILE" + +if [ -f /etc/init.d/functions ] ; then + . /etc/init.d/functions +fi + +start() { + + if [ ! -d "$PIDDIR" ] ; then + mkdir "$PIDDIR" + chown $LS_USER:$LS_USER $PIDDIR + fi + + if [ -f $PIDFILE ]; then + echo -e "\033[31;1mPID file found in $PIDFILE, already running?\033[0m" + ls_pid="$(cat $PIDFILE)" + pid_running="$( ps ax | grep 'java' | grep $ls_pid )" + + if [ ! -z "$pid_running" ] ; then + echo -e "\033[31;1mPID $ls_pid still alive, logstash is already running. Doing nothing\033[0m" + return 1 + fi + fi + + echo -e "\033[1mStarting logstash...\033[0m" + pushd $LS_HOME > /dev/null 2>&1 + su $LS_USER -c "$BIN_SCRIPT" > /dev/null 2>&1 + ls_pid=$! + result=$? + popd > /dev/null 2>&1 + + if [ $result -ne 0 ] ; then + failure + echo -e "Logstash did not start successfully" + exit 1 + else + success + echo -e "Logstash started successfully" + fi + +} + + + +function stop() { + echo -n -e "\033[1mStopping logstash...\033[0m" + + if [ -z "$SHUTDOWN_WAIT" ]; then + SHUTDOWN_WAIT=5 + fi + + if [ ! -z "$PIDFILE" ]; then + if [ -f "$PIDFILE" ]; then + kill -0 `cat $PIDFILE` >/dev/null 2>&1 + if [ $? -gt 0 ]; then + echo "PID file ($PIDFILE) found but no matching process was found. Nothing to do." + return 0 + fi + else + echo "\$PIDFILE was set ($PIDFILE) but the specified file does not exist. Is Logstash running? Assuming it has stopped and pro\ + ceeding." + return 0 + fi + fi + + kill `cat $PIDFILE` >/dev/null 2>&1 + + if [ ! -z "$PIDFILE" ]; then + if [ -f "$PIDFILE" ]; then + while [ $SHUTDOWN_WAIT -ge 0 ]; do + kill -0 `cat $PIDFILE` >/dev/null 2>&1 + if [ $? -gt 0 ]; then + rm $PIDFILE + break + fi + if [ $SHUTDOWN_WAIT -gt 0 ]; then + sleep 1 + fi + SHUTDOWN_WAIT=`expr $SHUTDOWN_WAIT - 1 ` + done + # still not dead, we may need to resort to drastic measures + if [ -f "$PIDFILE" ]; then + kill -0 `cat $PIDFILE` >/dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "Application still alive, sleeping for 20 seconds before sending SIGKILL" + sleep 20 + kill -0 `cat $PIDFILE` >/dev/null 2>&1 + if [ $? -eq 0 ]; then + kill -9 `cat $PIDFILE` >/dev/null 2>&1 + echo "Killed with extreme prejudice" + else + echo "Application stopped, no need to use SIGKILL" + fi + rm $PIDFILE + fi + fi + fi + fi +} + +restart() { + stop + start +} + +status() { + # GOT PIDFILE? + [ -f $PIDFILE ] && pid=$(cat $PIDFILE) + + # RUNNING + if [[ $pid && -d "/proc/$pid" ]]; then + success + echo -e "Logstash is running with pid $pid" + fi + + # NOT RUNNING + if [[ ! $pid || ! -d "/proc/$pid" ]]; then + echo "Logstash not running" + fi + + # STALE PID FOUND + if [[ ! -d "/proc/$pid" && -f $PIDFILE ]]; then + echo -e "\033[1;31;40m[!] Stale PID found in $PIDFILE\033[0m" + fi +} + + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + restart + ;; + status) + status $2 + ;; + *) + echo $"Usage: $0 {start|stop|restart|status [-v]|}" + exit 1 +esac + +exit $? diff --git a/templates/default/init/tarball_upstart.erb b/templates/default/init/tarball_upstart.erb new file mode 100644 index 0000000..6a95057 --- /dev/null +++ b/templates/default/init/tarball_upstart.erb @@ -0,0 +1,33 @@ +description "Logstash" +author "Chef" + +start on (filesystem and net-device-up) +stop on runlevel [!2345] + +respawn +respawn limit 5 30 +limit nofile 65550 65550 + +chdir <%= @home %> + +<% unless @upstart_with_sudo -%> +setuid <%= @user %> + <% unless @supervisor_gid.to_s.empty? -%> +setgid <%= @supervisor_gid %> + <% end -%> +<% end -%> + +script + export LS_HEAP_SIZE="<%= @max_heap %>" + export LOGSTASH_HOME="<%= @home %>" + export HOME=$LOGSTASH_HOME + export LOGSTASH_OPTS="<%= @args.join(' ') %>" + + <% if @upstart_with_sudo -%> + exec sudo -u <%= @user %> $LOGSTASH_HOME/bin/logstash $LOGSTASH_OPTS + <% else -%> + exec $LOGSTASH_HOME/bin/logstash $LOGSTASH_OPTS + <% end -%> +end script + +emits logstash-server-running diff --git a/templates/default/init/upstart.erb b/templates/default/init/upstart.erb new file mode 100644 index 0000000..cdfdbfe --- /dev/null +++ b/templates/default/init/upstart.erb @@ -0,0 +1,42 @@ +description "Logstash" +author "Chef" + +start on (filesystem and net-device-up) +stop on runlevel [!2345] + +respawn +respawn limit 5 30 +limit nofile 65550 65550 + +chdir <%= @home %> + +<% unless node['logstash']['instance'][@name]['upstart_with_sudo'] -%> +setuid <%= @user %> + <% unless @supervisor_gid.to_s.empty? -%> +setgid <%= @supervisor_gid %> + <% end -%> +<% end -%> + +script + export LOGSTASH_HOME="<%= @home %>" + export HOME=$LOGSTASH_HOME + + export GC_OPTS="<%= @gc_opts %>" + export JAVA_OPTS="-server -Xms<%= @min_heap %> -Xmx<%= @max_heap %> -Djava.io.tmpdir=$LOGSTASH_HOME/tmp/ <%= @java_opts %> <%= '-Djava.net.preferIPv4Stack=true' if @ipv4_only %>" + export LOGSTASH_OPTS="agent -f $LOGSTASH_HOME/etc/conf.d" + export LOGSTASH_OPTS="$LOGSTASH_OPTS --pluginpath $LOGSTASH_HOME/lib" + export LOGSTASH_OPTS="$LOGSTASH_OPTS -w <%= @workers %>" + export LOGSTASH_OPTS="$LOGSTASH_OPTS -l $LOGSTASH_HOME/log/<%= @log_file %>" + <% if @debug -%> + export LOGSTASH_OPTS="$LOGSTASH_OPTS -vv" + <% end -%> + export OPTS="$JAVA_OPTS $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS" + + <% if node['logstash']['instance'][@name]['upstart_with_sudo'] -%> + exec sudo -u <%= @user %> /usr/bin/java $OPTS + <% else -%> + exec /usr/bin/java $OPTS + <% end -%> +end script + +emits logstash-server-running diff --git a/templates/default/logstash_server.conf.erb b/templates/default/logstash_server.conf.erb index 94dc3c2..ba4291f 100644 --- a/templates/default/logstash_server.conf.erb +++ b/templates/default/logstash_server.conf.erb @@ -22,7 +22,7 @@ script export HOME=$LOGSTASH_HOME export GC_OPTS="<%= node['logstash']['server']['gc_opts'] %>" export JAVA_OPTS="-server -Xms<%= node['logstash']['server']['xms'] %> -Xmx<%= node['logstash']['server']['xmx'] %> -Djava.io.tmpdir=$LOGSTASH_HOME/tmp/ <%= node['logstash']['server']['java_opts'] %> <%= '-Djava.net.preferIPv4Stack=true' if node['logstash']['agent']['ipv4_only'] %>" - export LOGSTASH_OPTS="agent -f <%= node['logstash']['server']['config_dir'] %>" + export LOGSTASH_OPTS="agent -f <%= node['logstash']['server']['home'] %>/etc/conf.d" <% if node['logstash']['server']['pluginpath'] -%> export LOGSTASH_OPTS="$LOGSTASH_OPTS --pluginpath <%= node['logstash']['server']['pluginpath'] %>" <% end -%> @@ -30,7 +30,7 @@ script export LOGSTASH_OPTS="$LOGSTASH_OPTS -w <%= node['logstash']['server']['workers'] %>" <% end %> <% if node['logstash']['server']['log_file'] -%> - export LOGSTASH_OPTS="$LOGSTASH_OPTS -l <%= node['logstash']['server']['log_file'] %>" + export LOGSTASH_OPTS="$LOGSTASH_OPTS -l <%= node['logstash']['server']['home'] %>/log/<%= node['logstash']['server']['log_file'] %>" <% end -%> <% if node['logstash']['server']['debug'] -%> export LOGSTASH_OPTS="$LOGSTASH_OPTS -vv" diff --git a/templates/default/patterns.erb b/templates/default/patterns.erb deleted file mode 100644 index 448ee7c..0000000 --- a/templates/default/patterns.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% @patterns.sort.each do |name, pattern| -%> -<%= name %> <%= pattern %> -<% end -%> diff --git a/templates/default/patterns/custom_patterns.erb b/templates/default/patterns/custom_patterns.erb new file mode 100644 index 0000000..93ee935 --- /dev/null +++ b/templates/default/patterns/custom_patterns.erb @@ -0,0 +1,3 @@ +# this file was created and managed by chef. +# see https://github.com/elasticsearch/logstash/tree/master/patterns +# for pattern examples. diff --git a/templates/default/sv-logstash_agent-run.erb b/templates/default/sv-logstash_agent-run.erb index 770f435..a624c0d 100644 --- a/templates/default/sv-logstash_agent-run.erb +++ b/templates/default/sv-logstash_agent-run.erb @@ -1,23 +1,19 @@ #!/bin/sh -cd /<%= node['logstash']['agent']['home'] %> +cd /<%= @options[:home] %> exec 2>&1 # Need to set LOGSTASH_HOME and HOME so sincedb will work -LOGSTASH_HOME="<%= node['logstash']['agent']['home'] %>" +LOGSTASH_HOME="<%= @options[:home] %>" GC_OPTS="-XX:+UseParallelOldGC" -JAVA_OPTS="-server -Xms<%= node['logstash']['agent']['xms'] %> -Xmx<%= node['logstash']['agent']['xmx'] %> -Djava.io.tmpdir=$LOGSTASH_HOME/tmp/" -LOGSTASH_OPTS="agent -f <%= node['logstash']['agent']['config_dir'] %>" -<% if node['logstash']['agent']['debug'] -%> +JAVA_OPTS="-server -Xms<%= @options[:min_heap] %> -Xmx<%= @options[:max_heap] %> -Djava.io.tmpdir=$LOGSTASH_HOME/tmp/" +LOGSTASH_OPTS="agent -f $LOGSTASH_HOME/etc/conf.d" +<% if @options[:debug] -%> LOGSTASH_OPTS="$LOGSTASH_OPTS -vv" <% end -%> -<% if node['logstash']['agent']['log_file'] -%> -LOGSTASH_OPTS="$LOGSTASH_OPTS -l <%= node['logstash']['agent']['log_file'] %>" -<% end -%> -<% if node['logstash']['agent']['workers'] %> -export LOGSTASH_OPTS="$LOGSTASH_OPTS -w <%= node['logstash']['agent']['workers'] %>" -<% end %> -<% if node['logstash']['supervisor_gid'] -%> -HOME=$LOGSTASH_HOME exec chpst -u <%= node["logstash"]["user"] %>:<%= node["logstash"]["supervisor_gid"] %> java $JAVA_OPTS $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS +LOGSTASH_OPTS="$LOGSTASH_OPTS -l $LOGSTASH_HOME/log/<%= @options[:log_file] %>" +export LOGSTASH_OPTS="$LOGSTASH_OPTS -w <%= @options[:workers] %>" +<% if @options[:supervisor_gid] -%> +HOME=$LOGSTASH_HOME exec chpst -u <%= @options[:user] %>:<%= @options[:supervisor_gid] %> java $JAVA_OPTS $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS <% else -%> -HOME=$LOGSTASH_HOME exec chpst -u <%= node["logstash"]["user"] %> java $JAVA_OPTS $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS -<% end -%> \ No newline at end of file +HOME=$LOGSTASH_HOME exec chpst -u <%= @options[:user] %> java $JAVA_OPTS $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS +<% end -%> diff --git a/templates/default/sv-logstash_server-run.erb b/templates/default/sv-logstash_server-run.erb index 77173e2..0fc077a 100644 --- a/templates/default/sv-logstash_server-run.erb +++ b/templates/default/sv-logstash_server-run.erb @@ -3,27 +3,29 @@ ulimit -Hn 65550 ulimit -Sn 65550 -cd /<%= node['logstash']['server']['home'] %> +cd /<%= @options[:home] %> exec 2>&1 # Need to set LOGSTASH_HOME and HOME so sincedb will work -LOGSTASH_HOME="<%= node['logstash']['server']['home'] %>" -GC_OPTS="<%= node['logstash']['server']['gc_opts'] %>" -JAVA_OPTS="-server -Xms<%= node['logstash']['server']['xms'] %> -Xmx<%= node['logstash']['server']['xmx'] %> -Djava.io.tmpdir=$LOGSTASH_HOME/tmp/ <%= node['logstash']['server']['java_opts'] %> <%= '-Djava.net.preferIPv4Stack=true' if node['logstash']['agent']['ipv4_only'] %>" -LOGSTASH_OPTS="agent -f <%= node['logstash']['server']['config_dir'] %>" -<% if node['logstash']['server']['pluginpath'] -%> -LOGSTASH_OPTS="$LOGSTASH_OPTS --pluginpath <%= node['logstash']['server']['pluginpath'] %>" -<% end -%> -<% if node['logstash']['server']['debug'] -%> +LOGSTASH_HOME="<%= @options[:home] %>" +GC_OPTS="<%= @options[:gc_opts] %>" +JAVA_OPTS="-server -Xms<%= @options[:min_heap] %> -Xmx<%= @options[:max_heap] %> -Djava.io.tmpdir=$LOGSTASH_HOME/tmp/ <%= @options[:java_opts] %> <%= '-Djava.net.preferIPv4Stack=true' if @options[:ipv4_only] %>" +LOGSTASH_OPTS="agent -f $LOGSTASH_HOME/etc/conf.d" +LOGSTASH_OPTS="$LOGSTASH_OPTS --pluginpath $LOGSTASH_HOME/lib" +<% if @options[:debug] -%> LOGSTASH_OPTS="$LOGSTASH_OPTS -vv" <% end -%> -<% if node['logstash']['server']['log_file'] -%> -LOGSTASH_OPTS="$LOGSTASH_OPTS -l <%= node['logstash']['server']['log_file'] %>" -<% end -%> -<% if node['logstash']['server']['workers'] %> -export LOGSTASH_OPTS="$LOGSTASH_OPTS -w <%= node['logstash']['server']['workers'] %>" -<% end %> -<% if node['logstash']['supervisor_gid'] -%> -HOME=$LOGSTASH_HOME exec chpst -u <%= node["logstash"]["user"] %>:<%= node["logstash"]["supervisor_gid"] %> java $JAVA_OPTS $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS +LOGSTASH_OPTS="$LOGSTASH_OPTS -l $LOGSTASH_HOME/log/<%= @options[:log_file] %>" +export LOGSTASH_OPTS="$LOGSTASH_OPTS -w <%= @options[:workers] %>" +<% if @options[:install_type] == 'tarball' -%> +<% if @options[:supervisor_gid] -%> +HOME=$LOGSTASH_HOME exec chpst -u <%= @options[:user] %>:<%= @options[:supervisor_gid] %> $LOGSTASH_HOME/bin/logstash $LOGSTASH_OPTS +<% else -%> +HOME=$LOGSTASH_HOME exec chpst -u <%= @options[:user] %> $LOGSTASH_HOME/bin/logstash $LOGSTASH_OPTS +<% end -%> <% else -%> -HOME=$LOGSTASH_HOME exec chpst -u <%= node["logstash"]["user"] %> java $JAVA_OPTS $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS +<% if @options[:supervisor_gid] -%> +HOME=$LOGSTASH_HOME exec chpst -u <%= @options[:user] %>:<%= @options[:supervisor_gid] %> java $JAVA_OPTS $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS +<% else -%> +HOME=$LOGSTASH_HOME exec chpst -u <%= @options[:user] %> java $JAVA_OPTS $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS +<% end -%> <% end -%> diff --git a/templates/default/sv-logstash_web-run.erb b/templates/default/sv-logstash_web-run.erb index 3ade8b6..bd8d05c 100644 --- a/templates/default/sv-logstash_web-run.erb +++ b/templates/default/sv-logstash_web-run.erb @@ -3,17 +3,24 @@ ulimit -Hn 65550 ulimit -Sn 65550 -cd /<%= node['logstash']['server']['home'] %> +cd /<%= @options[:home] %> exec 2>&1 -LOGSTASH_HOME="<%= node['logstash']['server']['home'] %>" -GC_OPTS="<%= node['logstash']['server']['gc_opts'] %>" +LOGSTASH_HOME="<%= @options[:home] %>" +GC_OPTS="<%= @options[:gc_opts] %>" -LOGSTASH_OPTS="web -a <%= node['logstash']['server']['web']['address'] %> -p <%= node['logstash']['server']['web']['port'] %>" +LOGSTASH_OPTS="web -a <%= node[:logstash][:instance][@name][:web][:address] %> -p <%= node[:logstash][:instance][@name][:web][:port] %>" - -<% if ! node['logstash']['supervisor_gid'].empty? -%> -HOME=$LOGSTASH_HOME exec chpst -u <%= node['logstash']['user'] %>:<%= node['logstash']['supervisor_gid'] %> java $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS +<% if @options[:install_type] == 'tarball' -%> +<% if ! @options[:supervisor_gid].empty? -%> +HOME=$LOGSTASH_HOME exec chpst -u <%= @options[:user] %>:<%= @options[:supervisor_gid] %> $LOGSTASH_HOME/bin/logstash $LOGSTASH_OPTS +<% else -%> +HOME=$LOGSTASH_HOME exec chpst -u <%= @options[:user] %> $LOGSTASH_HOME/bin/logstash $LOGSTASH_OPTS +<% end -%> <% else -%> -HOME=$LOGSTASH_HOME exec chpst -u <%= node['logstash']['user'] %> java $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS +<% if ! @options[:supervisor_gid].empty? -%> +HOME=$LOGSTASH_HOME exec chpst -u <%= @options[:user] %>:<%= @options[:supervisor_gid] %> java $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS +<% else -%> +HOME=$LOGSTASH_HOME exec chpst -u <%= @options[:user] %> java $GC_OPTS -jar $LOGSTASH_HOME/lib/logstash.jar $LOGSTASH_OPTS +<% end -%> <% end -%> diff --git a/test/integration/server/serverspec/server_spec.rb b/test/integration/server/serverspec/server_spec.rb index c4c74e4..4168d4c 100644 --- a/test/integration/server/serverspec/server_spec.rb +++ b/test/integration/server/serverspec/server_spec.rb @@ -1,11 +1,43 @@ # Encoding: utf-8 require 'spec_helper' -describe 'logstash server' do - it { pending 'it writes the tests for the code or else it gets the hose again' } +# Java 1.6 +describe command('java -version') do + it { should return_stdout(/java version "1.7.\d+_\d+"/) } end +# Logstash Instance describe service('logstash_server') do it { should be_enabled } it { should be_running } end + +describe user('logstash') do + it { should exist } +end + +# Logstash Config +describe file('/opt/logstash/server/etc/conf.d/input_syslog.conf') do + it { should be_file } +end + +describe file('/opt/logstash/server/etc/conf.d/output_elasticsearch.conf') do + it { should be_file } +end + +describe file('/opt/logstash/server/etc/conf.d/output_stdout.conf') do + it { should be_file } +end + +describe port(9200) do + it { should be_listening } +end + +describe port(5959) do + it { should be_listening } +end + +# Logstash Curator +describe cron do + it { should have_entry('0 * * * * curator --host 127.0.0.1 -d 31 &> /dev/null').with_user('logstash') } +end diff --git a/test/unit/spec/agent_spec.rb b/test/unit/spec/agent_spec.rb index b122224..4648138 100644 --- a/test/unit/spec/agent_spec.rb +++ b/test/unit/spec/agent_spec.rb @@ -2,12 +2,37 @@ require_relative 'spec_helper' describe 'logstash::agent' do - before { logstash_stubs } describe 'ubuntu' do - let(:chef_run) { ChefSpec::Runner.new.converge(described_recipe) } + let(:runner) { ChefSpec::Runner.new(::UBUNTU_OPTS) } + let(:node) { runner.node } + let(:chef_run) do + # runner.node.set['logstash'] ... + runner.node.automatic['memory']['total'] = '1024kB' + runner.node.set['logstash']['instance']['agent']['basedir'] = '/opt/logstash' + runner.node.set['logstash']['instance']['agent']['user'] = 'logstash' + runner.node.set['logstash']['instance']['agent']['group'] = 'logstash' + runner.node.set['logstash']['instance']['agent']['config_templates_cookbook'] = 'logstash' + runner.node.set['logstash']['instance']['agent']['elasticsearch_ip'] = '127.0.0.1' + runner.node.set['logstash']['instance']['agent']['enable_embedded_es'] = false + runner.converge(described_recipe) + end + include_context 'stubs-common' + + it 'calls the logstash_instance LWRP' do + expect(chef_run).to create_logstash_instance('agent') + end + + it 'calls the logstash_config LWRP' do + expect(chef_run).to create_logstash_config('agent') + end + + it 'calls the logstash_pattern LWRP' do + expect(chef_run).to create_logstash_pattern('agent') + end - it 'writes some chefspec code' do - pending 'todo' + it 'calls the logstash_instance LWRP' do + expect(chef_run).to enable_logstash_service('agent') + expect(chef_run).to start_logstash_service('agent') end end diff --git a/test/unit/spec/beaver_spec.rb b/test/unit/spec/beaver_spec.rb index 1088fe1..c234d53 100644 --- a/test/unit/spec/beaver_spec.rb +++ b/test/unit/spec/beaver_spec.rb @@ -2,12 +2,17 @@ require_relative 'spec_helper' describe 'logstash::beaver' do - before { logstash_stubs } describe 'ubuntu' do - let(:chef_run) { ChefSpec::Runner.new.converge(described_recipe) } + let(:runner) { ChefSpec::Runner.new(::UBUNTU_OPTS) } + let(:node) { runner.node } + let(:chef_run) do + # runner.node.set['logstash'] ... + runner.converge(described_recipe) + end + include_context 'stubs-common' it 'writes some chefspec code' do - pending 'todo' + skip 'todo' end end diff --git a/test/unit/spec/default_spec.rb b/test/unit/spec/default_spec.rb index b995568..b226dcf 100644 --- a/test/unit/spec/default_spec.rb +++ b/test/unit/spec/default_spec.rb @@ -2,12 +2,17 @@ require_relative 'spec_helper' describe 'logstash::default' do - before { logstash_stubs } describe 'ubuntu' do - let(:chef_run) { ChefSpec::Runner.new.converge(described_recipe) } + let(:runner) { ChefSpec::Runner.new(::UBUNTU_OPTS) } + let(:node) { runner.node } + let(:chef_run) do + runner.node.set['memory']['total'] = '1024000kb' + runner.converge(described_recipe) + end + include_context 'stubs-common' it 'writes some chefspec code' do - pending 'todo' + skip 'todo' end end diff --git a/test/unit/spec/haproxy_spec.rb b/test/unit/spec/haproxy_spec.rb index c9cfe80..51d1d53 100644 --- a/test/unit/spec/haproxy_spec.rb +++ b/test/unit/spec/haproxy_spec.rb @@ -2,12 +2,17 @@ require_relative 'spec_helper' describe 'logstash::haproxy' do - before { logstash_stubs } describe 'ubuntu' do - let(:chef_run) { ChefSpec::Runner.new.converge(described_recipe) } + let(:runner) { ChefSpec::Runner.new(::UBUNTU_OPTS) } + let(:node) { runner.node } + let(:chef_run) do + # runner.node.set['logstash'] ... + runner.converge(described_recipe) + end + include_context 'stubs-common' it 'writes some chefspec code' do - pending 'todo' + skip 'todo' end end diff --git a/test/unit/spec/index_cleaer_spec.rb b/test/unit/spec/index_cleaer_spec.rb deleted file mode 100644 index c4ae32e..0000000 --- a/test/unit/spec/index_cleaer_spec.rb +++ /dev/null @@ -1,14 +0,0 @@ -# Encoding: utf-8 -require_relative 'spec_helper' - -describe 'logstash::index_cleaner' do - before { logstash_stubs } - describe 'ubuntu' do - let(:chef_run) { ChefSpec::Runner.new.converge(described_recipe) } - - it 'writes some chefspec code' do - pending 'todo' - end - - end -end diff --git a/test/unit/spec/lwrp_config_spec.rb b/test/unit/spec/lwrp_config_spec.rb new file mode 100644 index 0000000..29979e4 --- /dev/null +++ b/test/unit/spec/lwrp_config_spec.rb @@ -0,0 +1,65 @@ +# Encoding: utf-8 +require_relative 'spec_helper' +require_relative 'server_spec' + +::LWRP = { + step_into: ['logstash_config'] +}.merge(::UBUNTU_OPTS) + +describe 'logstash::server' do + describe 'ubuntu' do + let(:runner) { ChefSpec::Runner.new(::LWRP) } + let(:node) { runner.node } + let(:chef_run) do + runner.node.set['memory']['total'] = '1024000kb' + runner.node.set['logstash']['instance']['server']['config_templates'] = { + output_stdout: 'config/output_stdout.conf.erb' + } + runner.node.set['logstash']['instance']['server']['basedir'] = '/opt/logstash' + runner.node.set['logstash']['instance']['server']['user'] = 'logstash' + runner.node.set['logstash']['instance']['server']['group'] = 'logstash' + runner.node.set['logstash']['instance']['server']['config_templates_cookbook'] = 'logstash' + runner.node.set['logstash']['instance']['server']['elasticsearch_ip'] = '127.0.0.1' + runner.node.set['logstash']['instance']['server']['enable_embedded_es'] = true + runner.converge(described_recipe) + end + include_context 'stubs-common' + + it 'installs the output_stdout template' do + expect(chef_run).to create_template('/opt/logstash/server/etc/conf.d/output_stdout.conf').with( + source: 'config/output_stdout.conf.erb', + cookbook: 'logstash', + owner: 'logstash', + group: 'logstash', + mode: '0644', + action: [:create] + ) + end + + it 'installs the input_syslog template' do + expect(chef_run).to create_template('/opt/logstash/server/etc/conf.d/input_syslog.conf').with( + source: 'config/input_syslog.conf.erb', + cookbook: 'logstash', + owner: 'logstash', + group: 'logstash', + mode: '0644', + action: [:create] + ) + end + + it 'installs the output_elasticsearch template' do + expect(chef_run).to create_template('/opt/logstash/server/etc/conf.d/output_elasticsearch.conf').with( + source: 'config/output_elasticsearch.conf.erb', + cookbook: 'logstash', + owner: 'logstash', + group: 'logstash', + mode: '0644', + variables: { + elasticsearch_embedded: true + }, + action: [:create] + ) + end + + end +end diff --git a/test/unit/spec/lwrp_pattern_spec.rb b/test/unit/spec/lwrp_pattern_spec.rb new file mode 100644 index 0000000..e1e06ca --- /dev/null +++ b/test/unit/spec/lwrp_pattern_spec.rb @@ -0,0 +1,39 @@ +# Encoding: utf-8 +require_relative 'spec_helper' +require_relative 'server_spec' + +describe 'logstash::server' do + describe 'ubuntu' do + let(:runner) { ChefSpec::Runner.new(step_into: ['logstash_pattern']) } + let(:node) { runner.node } + let(:chef_run) do + runner.node.merge(::UBUNTU_OPTS) + runner.node.set['memory']['total'] = '1024000kb' + runner.node.set['logstash']['instance']['server']['pattern_templates'] = { + 'default' => 'patterns/custom_patterns.erb' + } + runner.node.set['logstash']['instance']['server']['pattern_templates_variables'] = { + 'test' => true + } + runner.node.set['logstash']['instance']['server']['basedir'] = '/opt/logstash' + runner.node.set['logstash']['instance']['server']['user'] = 'logstash' + runner.node.set['logstash']['instance']['server']['group'] = 'logstash' + runner.node.set['logstash']['instance']['server']['pattern_templates_cookbook'] = 'logstash' + runner.converge(described_recipe) + end + include_context 'stubs-common' + + it 'installs the pattern template' do + expect(chef_run).to create_template('/opt/logstash/server/patterns/custom_patterns').with( + source: 'patterns/custom_patterns.erb', + cookbook: 'logstash', + owner: 'logstash', + group: 'logstash', + mode: '0644', + variables: { 'test' => true }, + action: [:create] + ) + end + + end +end diff --git a/test/unit/spec/pyshipper_spec.rb b/test/unit/spec/pyshipper_spec.rb index 68a9721..34091a1 100644 --- a/test/unit/spec/pyshipper_spec.rb +++ b/test/unit/spec/pyshipper_spec.rb @@ -2,15 +2,17 @@ require_relative 'spec_helper' describe 'logstash::pyshipper' do - before { logstash_stubs } describe 'ubuntu' do - before do - @chef_run = ::ChefSpec::Runner.new ::UBUNTU_OPTS - @chef_run.converge 'logstash::pyshipper' + let(:runner) { ChefSpec::Runner.new(::UBUNTU_OPTS) } + let(:node) { runner.node } + let(:chef_run) do + # runner.node.set['logstash'] ... + runner.converge(described_recipe) end + include_context 'stubs-common' it 'writes some chefspec code' do - pending 'todo' + skip 'todo' end end diff --git a/test/unit/spec/server_spec.rb b/test/unit/spec/server_spec.rb index 0934e77..75ca8bf 100644 --- a/test/unit/spec/server_spec.rb +++ b/test/unit/spec/server_spec.rb @@ -2,12 +2,45 @@ require_relative 'spec_helper' describe 'logstash::server' do - before { logstash_stubs } describe 'ubuntu' do - let(:chef_run) { ChefSpec::Runner.new.converge(described_recipe) } + let(:runner) { ChefSpec::Runner.new(::UBUNTU_OPTS) } + let(:node) { runner.node } + let(:chef_run) do + # runner.node.set['logstash'] ... + runner.node.automatic['memory']['total'] = '1024kB' + runner.node.set['logstash']['instance']['server']['basedir'] = '/opt/logstash' + runner.node.set['logstash']['instance']['server']['user'] = 'logstash' + runner.node.set['logstash']['instance']['server']['group'] = 'logstash' + runner.node.set['logstash']['instance']['server']['config_templates_cookbook'] = 'logstash' + runner.node.set['logstash']['instance']['server']['elasticsearch_ip'] = '127.0.0.1' + runner.node.set['logstash']['instance']['server']['enable_embedded_es'] = true + runner.converge(described_recipe) + end + include_context 'stubs-common' + + it 'calls the logstash_instance LWRP' do + expect(chef_run).to create_logstash_instance('server') + end + + it 'calls the logstash_config LWRP' do + expect(chef_run).to create_logstash_config('server') + end + + it 'calls the logstash_pattern LWRP' do + expect(chef_run).to create_logstash_pattern('server') + end + + it 'calls the logstash_service LWRP' do + expect(chef_run).to enable_logstash_service('server') + expect(chef_run).to start_logstash_service('server') + end + + it 'calls the logstash_plugins LWRP' do + expect(chef_run).to create_logstash_plugins('contrib') + end - it 'writes some chefspec code' do - pending 'todo' + it 'calls the logstash_curator LWRP' do + expect(chef_run).to create_logstash_curator('server') end end diff --git a/test/unit/spec/source_spec.rb b/test/unit/spec/source_spec.rb index e520082..e9086bc 100644 --- a/test/unit/spec/source_spec.rb +++ b/test/unit/spec/source_spec.rb @@ -2,12 +2,17 @@ require_relative 'spec_helper' describe 'logstash::source' do - before { logstash_stubs } describe 'ubuntu' do - let(:chef_run) { ChefSpec::Runner.new.converge(described_recipe) } + let(:runner) { ChefSpec::Runner.new(::UBUNTU_OPTS) } + let(:node) { runner.node } + let(:chef_run) do + # runner.node.set['logstash'] ... + runner.converge(described_recipe) + end + include_context 'stubs-common' it 'writes some chefspec code' do - pending 'todo' + skip 'todo' end end diff --git a/test/unit/spec/spec_helper.rb b/test/unit/spec/spec_helper.rb index cee8990..9b95abc 100644 --- a/test/unit/spec/spec_helper.rb +++ b/test/unit/spec/spec_helper.rb @@ -1,18 +1,37 @@ # Encoding: utf-8 +require 'rspec/expectations' require 'chefspec' require 'chefspec/berkshelf' +require 'chefspec/server' require 'chef/application' ::LOG_LEVEL = :fatal +::REDHAT_OPTS = { + platform: 'redhat', + version: '6.4', + log_level: ::LOG_LEVEL +} + ::UBUNTU_OPTS = { - platform: 'ubuntu', - version: '12.04', + platform: 'ubuntu', + version: '12.04', log_level: ::LOG_LEVEL } -def logstash_stubs - stub_command("update-alternatives --display java | grep '/usr/lib/jvm/java-6-openjdk-amd64/jre/bin/java - priority 1061'").and_return(true) - stub_command("/usr/bin/python -c 'import setuptools'").and_return(true) - stub_command('test -f /opt/logstash/source/build/logstash-v1.3.2-monolithic.jar').and_return(true) +shared_context 'stubs-common' do + before do + Chef::Application.stub(:fatal!).and_return('fatal') + stub_command("update-alternatives --display java | grep '/usr/lib/jvm/java-6-openjdk-amd64/jre/bin/java - priority 1061'").and_return(true) + stub_command("/usr/bin/python -c 'import setuptools'").and_return(true) + stub_command('test -f /opt/logstash/source/build/logstash-v1.3.2-monolithic.jar').and_return(true) + end end + +shared_examples 'example' do + # it 'does not include example recipe by default' do + # expect(chef_run).not_to include_recipe('example::default') + # end +end + +at_exit { ChefSpec::Coverage.report! } diff --git a/test/unit/spec/zero_mq_repo.rb b/test/unit/spec/zero_mq_repo.rb index 2b72d78..ac75be2 100644 --- a/test/unit/spec/zero_mq_repo.rb +++ b/test/unit/spec/zero_mq_repo.rb @@ -2,12 +2,17 @@ require_relative 'spec_helper' describe 'logstash::zero_mq_repo' do - before { logstash_stubs } describe 'ubuntu' do - let(:chef_run) { ChefSpec::Runner.new.converge(described_recipe) } + let(:runner) { ChefSpec::Runner.new(::UBUNTU_OPTS) } + let(:node) { runner.node } + let(:chef_run) do + # runner.node.set['logstash'] ... + runner.converge(described_recipe) + end + include_context 'stubs-common' it 'writes some chefspec code' do - pending 'todo' + skip 'todo' end end