ref: refs/heads/jr_ntp commit 43a1dd11716ff71907a9efdedd170b1425324f38 Author: Duncan Mac-Vicar P Date: Fri Sep 4 18:21:51 2009 +0200 more updates. Refactor the Status model in a model class that follows sane semantics, and full test, still not there. --- plugins/status/lib/metric.rb | 232 ++++++++++++++++++++++ plugins/status/test/data/rrdtool-memory-free.txt | 11 + plugins/status/test/data/rrdtool-packets.txt | 8 + plugins/status/test/unit/metric_test.rb | 144 ++++++++++++++ webservice/app/controllers/logs_controller.rb | 16 +- 5 files changed, 403 insertions(+), 8 deletions(-) diff --git a/plugins/status/lib/metric.rb b/plugins/status/lib/metric.rb new file mode 100644 index 0000000..a3b46dc --- /dev/null +++ b/plugins/status/lib/metric.rb @@ -0,0 +1,232 @@ +# +# Model over collectd data +# +# @author Bjoern Geuken +# @author Duncan Mac-Vicar P. +# @author Stefan Schubert +# +require 'scr' +require 'yaml' + +# +# This class acts as a model for metrics provided by collectd +# each round robin database +# +# By default the model operates in the current host, or in the +# first available one, unless overriden by the :host option +# +# metrics = Metric.find(:all, group => /cpu/) +# metrics.each do |metric| +# metric.name +# metric.group +# metric.host +# metric.data(:stop => Time.now - 5000) +# end +# +# +class Metric + + COLLECTD_BASE = "/var/lib/collectd" + + attr_accessor :host, :group, :name + + def path + File.join(Metric.host_directory(host), group, name + '.rrd') + end + + def id + [host, group, name].join('/') + end + + # returns available hosts for which metrics are collected + def self.available_hosts + Dir.glob(File.join(COLLECTD_BASE, "*")).reject{|x| not File.directory?(x) }.map{|x| File.basename(x) } + end + + # whether there is data available for a given host + def self.host_available?(host) + available_hosts.include?(host) + end + + # returns the hostname of the running + # host or nil if not available + def self.this_hostname + Socket.gethostbyname(Socket.gethostname).first rescue nil + end + + # what host should be used if host is not + # specified + # + # we try to use the running system hostname + # or the first hostname with data available + def self.default_host + # try to get the real hostname + hostname = this_hostname + hosts = available_hosts + return case + # if data exists for it, then this is the default + when host_available?(hostname) then hostname + when !hosts.empty? then hosts.first + else raise "Can't determine default host to read metric from" + end + end + + # the directory where we are gathering data from + # normally is the directory corresponding to *this* + # host. If there are more than one, we take the first + # one or the one given + # + # Metrics.host_directory + # Metrics.host_directory("foo.com") + # + def self.host_directory(host=Metric.default_host) + File.join(COLLECTD_BASE, host) + end + + # returns true if collectd daemon is running + def self.collectd_running? + ret = Scr.instance.execute(["/usr/sbin/rccollectd", "status"]) + ret[:exit].zero? + end + + # returns the available metric groups + def self.available_groups(host=Metric.default_host) + Dir.glob(File.join(host_directory(host), '*')).reject{|x| not File.directory?(x) }.map{|x| File.basename(x) } + end + + # returns the available metrics names for a group + # that is the rrd file name without the extension + def self.available_metrics(group, host=Metric.default_host) + Dir.glob(File.join(host_directory(host), group, '*')).reject{|x| not File.file?(x) }.map{|x| File.basename(x).chomp('.rrd') } + end + + # Finds metrics. + # + # Metric.find(:all) + # Metric.find(:all, group => "cpu-1") + # Metric.find(id) + # Where id is host:group:name (whithout rrd extension) + def self.find(what, opts={}) + case what + when :all then find_multiple(opts) + # in this case, the options are the first + # parameter + when Hash + else find_multiple(what.merge(opts)) + end + end + + def self.find_multiple(opts) + ret = [] + current_host = opts[:host] || default_host + available_hosts.each do |host| + # if host is specified, only use that + # host + next if host != current_host + available_groups(current_host).each do |group| + # filter by group + case opts[:group] + when Regexp + next if not group =~ opts[:group] + when String + next if group != opts[:group] + end + available_metrics(group, current_host).each do |metric_name| + # filter by name + case opts[:name] + when Regexp then next if not metric_name =~ opts[:name] + when String then next if metric_name != opts[:name] + end + + # instantiate the object + metric = Metric.new + metric.host = current_host + metric.name = metric_name + metric.group = group + ret << metric + end + end + end + ret + end + + # returns data for a given period of time + # range can be specified with the :start and + # :stop options + # + # default last 5 minutes from now + # + # result is a hash of the type: + # { column1 => { time1 => value1, time2 => value2, ...}, ...} + # + def data(opts={}) + Metric.read_metric_file(path, opts) + end + + # runs the rddtool on file with start time and end time + # + # You can pass start and stop options: + # Metric.run_rrdtool(file, :start => t1, :stop => t2) + # + # default last 5 minutes from now + def self.run_rrdtool(file, opts={}) + stop = opts.has_key?(:stop) ? opts[:stop] : Time.now + start = opts.has_key?(:stop) ? opts[:stop] : stop - 300 + + cmd = IO.popen("rrdtool fetch #{file} AVERAGE --start #{start.strftime("%H:%M,%m/%d/%Y")} --end #{stop.strftime("%H:%M,%m/%d/%Y")}") + output = cmd.read + cmd.close + output + end + + def self.read_metric_file(rrdfile, opts={}) + result = Hash.new + output = run_rrdtool(rrdfile, opts) + + raise "Error running collectd rrdtool" if output =~ /ERROR/ or output.nil? + + labels="" + output = output.gsub(",", ".") # translates eg. 1,234e+07 to 1.234e+07 + lines = output.split "\n" + + # set label names + lines[0].each do |l| + if l =~ /\D*/ + labels = l.split " " + end + end + unless labels.blank? + # set values for each label and time + # evaluates interval and starttime once for each metric (not each label) + nexttime = 9.9e+99 + result["starttime"] = 9.9e+99 + lines.each do |l| # each time + next if l.blank? + if l =~ /\d*:\D*/ + pair = l.split ":" + values = pair[1].split " " + column = 0 + values.each do |v| # each label + unless result["interval"] # already defined? + # get the least distance to evaluate time interval + if pair[0].to_i < result["starttime"].to_i + result["starttime"] = pair[0].to_i + elsif pair[0].to_i < nexttime and pair[0].to_i > result["starttime"].to_i + nexttime = pair[0].to_i + end + end + v = "invalid" if v == "nan" #store valid values only + result[labels[column]] ||= Hash.new + result[labels[column]][pair[0]] = v + column += 1 + end + end + end + result["interval"] = nexttime.to_i - result["starttime"].to_i if result["interval"].nil? + return result + else + raise "error reading data from rrdtool" + end + end + +end diff --git a/plugins/status/test/data/rrdtool-memory-free.txt b/plugins/status/test/data/rrdtool-memory-free.txt new file mode 100644 index 0000000..e36fe36 --- /dev/null +++ b/plugins/status/test/data/rrdtool-memory-free.txt @@ -0,0 +1,11 @@ + value + +1252071660: 6.1664133120e+08 +1252071670: 6.1510287360e+08 +1252071680: 6.1513154560e+08 +1252071690: 6.1518643200e+08 +1252071700: 6.1514301440e+08 +1252071750: 6.1545021440e+08 +1252071760: 6.1678837760e+08 +1252071770: nan +1252071780: nan diff --git a/plugins/status/test/data/rrdtool-packets.txt b/plugins/status/test/data/rrdtool-packets.txt new file mode 100644 index 0000000..ed7f2ed --- /dev/null +++ b/plugins/status/test/data/rrdtool-packets.txt @@ -0,0 +1,8 @@ + rx tx + +1252075500: 2.2150000000e+01 7.7900000000e+00 +1252075760: 2.5814000000e+02 6.1660000000e+01 +1252075770: 3.3962000000e+02 1.5922000000e+02 +1252075780: 2.8069000000e+02 4.2576000000e+02 +1252075790: nan nan +1252075800: nan nan diff --git a/plugins/status/test/unit/metric_test.rb b/plugins/status/test/unit/metric_test.rb new file mode 100644 index 0000000..52fc8b5 --- /dev/null +++ b/plugins/status/test/unit/metric_test.rb @@ -0,0 +1,144 @@ +require File.expand_path(File.dirname(__FILE__) + "/../test_helper") +require 'scr' +require 'mocha' +require 'metric' +require 'test_helper' + +def test_data(name) + File.join(File.dirname(__FILE__), '..', 'data', name) +end + +class MetricTest < ActiveSupport::TestCase + + def setup + # standard setup + Metric.stubs(:this_hostname).returns('myhost.domain.de') + Metric.stubs(:available_hosts).returns(['foo.com', 'myhost.domain.de']) + Metric.stubs(:available_groups).with('myhost.domain.de').returns(['memory', 'cpu-1', 'cpu-2', 'interface']) + Metric.stubs(:available_metrics).with('memory', 'myhost.domain.de').returns(['memory-free']) + Metric.stubs(:available_metrics).with('cpu-1', 'myhost.domain.de').returns(['cpudata1', 'cpudata2']) + Metric.stubs(:available_metrics).with('cpu-2', 'myhost.domain.de').returns(['cpudata1']) + Metric.stubs(:available_metrics).with('interface', 'myhost.domain.de').returns(['packets']) + # in total 5 metrics, 4 groups, 2 hosts + + + end + + def teardown + end + + def test_host_directory + # if only one host data is available, then that one should be + # the one used for the data directory + Metric.stubs(:this_hostname).returns(nil) + assert_equal 'foo.com', Metric.default_host + assert_equal File.join(Metric::COLLECTD_BASE, 'foo.com'), Metric.host_directory + # if our hostname is in the available list, that one should be the default host + Metric.stubs(:this_hostname).returns('myhost.domain.de') + assert_equal 'myhost.domain.de', Metric.default_host + assert_equal File.join(Metric::COLLECTD_BASE, 'myhost.domain.de'), Metric.host_directory + + # if no host data is available, no base_directory + Metric.stubs(:available_hosts).returns([]) + assert_raise RuntimeError do + Metric.host_directory + end + end + + def test_parse_rrdtool_output + stop = Time.now + start = Time.now - 300 + + Metric.stubs(:run_rrdtool).with('/var/lib/collectd/myhost.domain.de/memory/memory-free.rrd', :start => start, :stop => stop).returns(File.open(test_data('rrdtool-memory-free.txt')).read) + Metric.stubs(:run_rrdtool).with('/var/lib/collectd/myhost.domain.de/interface/packets.rrd', :start => start, :stop => stop).returns(File.open(test_data('rrdtool-packets.txt')).read) + + ret = Metric.find(:all, :group => /memory/, :name => 'memory-free') + memory = ret.first + + expected = { "value"=> + {"1252071700"=>"6.1514301440e+08", + "1252071690"=>"6.1518643200e+08", + "1252071680"=>"6.1513154560e+08", + "1252071780"=>"invalid", + "1252071670"=>"6.1510287360e+08", + "1252071770"=>"invalid", + "1252071660"=>"6.1664133120e+08", + "1252071760"=>"6.1678837760e+08", + "1252071750"=>"6.1545021440e+08"}, + "interval"=>10, + "starttime"=>1252071660 } + + assert_equal expected, memory.data(:start => start, :stop => stop) + + ret = Metric.find(:all, :group => /interface/, :name => 'packets') + packets = ret.first + expected = {"tx"=> + {"1252075780"=>"4.2576000000e+02", + "1252075770"=>"1.5922000000e+02", + "1252075760"=>"6.1660000000e+01", + "1252075500"=>"7.7900000000e+00", + "1252075800"=>"invalid", + "1252075790"=>"invalid"}, + "rx"=> + {"1252075780"=>"2.8069000000e+02", + "1252075770"=>"3.3962000000e+02", + "1252075760"=>"2.5814000000e+02", + "1252075500"=>"2.2150000000e+01", + "1252075800"=>"invalid", + "1252075790"=>"invalid"}, + "interval"=>260, + "starttime"=>1252075500} + + assert_equal expected, packets.data(:start => start, :stop => stop) + + end + + def test_collectd_running + Scr.instance.stubs(:execute).with(['/usr/sbin/rccollectd', 'status']).returns({:exit => 0}) + assert Metric.collectd_running? + Scr.instance.stubs(:execute).with(['/usr/sbin/rccollectd', 'status']).returns({:exit => 1}) + assert !Metric.collectd_running? + end + + def test_finders + ret = Metric.find(:all) + assert_equal 5, ret.size + assert ret.map{|x| x.group}.include?('cpu-1') + assert ret.map{|x| x.group}.include?('memory') + assert ret.map{|x| x.group}.include?('interface') + + ret = Metric.find(:all, :group => 'memory') + assert_equal 1, ret.size + assert_equal 'memory', ret.first.group + assert_equal 'memory-free', ret.first.name + assert_equal 'myhost.domain.de', ret.first.host + + # should produce same result + ret = Metric.find(:all, :group => /memory/) + assert_equal 1, ret.size + ret = Metric.find(:all, :group => /memory/, :name => 'boooh') + assert_equal 0, ret.size + ret = Metric.find(:all, :group => /memory/, :name => 'memory-free') + assert_equal 1, ret.size + + # test attributes + assert_equal '/var/lib/collectd/myhost.domain.de/memory/memory-free.rrd', ret.first.path + assert_equal 'myhost.domain.de/memory/memory-free', ret.first.id + + ret = Metric.find(:all, :group => /meemory/) + assert_equal 0, ret.size + + ret = Metric.find(:all, :group => 'meemory') + assert_equal 0, ret.size + + # using a regexp + ret = Metric.find(:all, :group => /cpu/) + assert ret.map{ |x| x.group }.include?('cpu-1') + assert ret.map{ |x| x.group }.include?('cpu-2') + assert ret.map{ |x| x.name }.include?('cpudata1') + assert ret.map{ |x| x.name }.include?('cpudata2') + assert_equal 3, ret.size + end + + +end diff --git a/webservice/app/controllers/logs_controller.rb b/webservice/app/controllers/logs_controller.rb index 7f0122b..d3a975f 100644 --- a/webservice/app/controllers/logs_controller.rb +++ b/webservice/app/controllers/logs_controller.rb @@ -11,12 +11,16 @@ class LogsController < ApplicationController when "messages" then '/var/log/messages' when "apache_access" then '/var/log/apache2/access_log' when "apache_error" then '/var/log/apache2/error_log' - else nil + when "custom" + # if a custom log is requested, then + # we evaluate a filename parameter + params[:filename].blank? ? nil : params[:filename] + else params[:id] end # not found - if log_filename.nil? - render :nothing, :status => 404 and return + if log_filename.nil? or not File.exist?(log_filename) + render :nothing => true, :status => 404 and return end # how many lines to show @@ -28,12 +32,8 @@ class LogsController < ApplicationController output = Scr.instance.execute(['tail', '-n', "#{lines}", log_filename]) respond_to do |format| - format.xml { render :xml => settings.to_xml } - format.json { render :json => VendorSetting } + format.text { render :xml => output[:stdout] } end - - #render :text => output[:stdout] - render :xml => xm.target! end end -- To unsubscribe, e-mail: yast-commit+unsubscribe@opensuse.org For additional commands, e-mail: yast-commit+help@opensuse.org