true Sam Sinensky

sphinx

# Created by Sam Sinensky (http://github.com/samsinensky) at Code and Beats (http://codeandbeats.com/) 12/29/09
class SphinxMonitor < Scout::Plugin
  
  needs 'elif'
  needs 'time'
  
  OPTIONS=<<-EOS
  search_log_path:
    name: Searchd log path
    notes: This is the path to the log file
  query_log_path:
    name: Query log path
    notes: This is the path to the query log
  EOS
  
  def build_report
    
    #test command  = scout test sphinx_monitor.rb query_log_path=/Users/sam/Desktop/query.log.bak search_log_path=/Users/sam/Desktop/searchd.log.bak
     
     search_log_path = option(:search_log_path)
     
     query_log_path =  option(:query_log_path)
     
     unless search_log_path and not search_log_path.empty? and query_log_path and not query_log_path.empty?
       return error("Full paths to the searchd log and/or searchd query file(s) were not provided.","Please provide the full paths to the log files in the plugin settings.")
     end
     
     last_run = memory(:last_request_time) || Time.now

     #in seconds or amount/second
     report_data = {
       :query_rate => 0,
       :average_query_time => 0,
       :average_results_returned => 0,
       :index_rebuilds => 0,
       :average_time_per_rebuild => 0
     }
     
     #calculate the stats based on queries, rate, avg_time and average results returned
     
     #Load each line from the log in if it happened after the last request used in the previous report
     queries = 0
     total_query_time = 0
     total_results_returned = 0
     begin
       Elif.foreach(query_log_path) do |line|
         #extract the date form the line and make sure it occured after last_run
         line_data = parse_query_line(line)
         if line_data.timestamp.to_f <= last_run.to_f
           break
         else
           queries+=1
           total_query_time += line_data.time_spent
           total_results_returned += line_data.results_returned
         end
       end
     
       if queries > 0 
         # calculate the time btw runs in minutes
         interval = (Time.now-last_run)
         interval < 1 ? inteval = 1 : nil # if the interval is less than 1 second (may happen on initial run) set to 1 second
         interval = interval/60 # convert to minutes
         interval = interval.to_f
         # determine the rate of queries in queries/min
         query_rate                             = queries / interval
         report_data[:query_rate]               = sprintf("%.2f", query_rate)
         report_data[:average_query_time]       = sprintf("%.4f", total_query_time/queries)
         report_data[:average_results_returned] = sprintf("%.4f", total_results_returned/queries)
       end
     rescue Errno::ENOENT => error
       return error("Unable to find the query log file", "Could not find the query log at the specified path: #{option(:query_log_path)}.")
     rescue Exception => error
       return error("Error while processing query log:\n#{error.class}: #{error.message}", error.backtrace.join("\n"))
     end
     
     #calculate the index rotation stats, only for index rotations that occur completely in the interval
     total_rotations = 0
     total_length_rotations = 0
     finish_time = nil
     begin
       Elif.foreach(search_log_path) do |line|
         line_data = parse_log_line(line)
         if line_data.timestamp.to_f <= last_run.to_f
           break
         else
           if finish_time
             if line_data.step == :start
               total_rotations += 1
               total_length_rotations += finish_time.to_f - line_data.timestamp.to_f
               finish_time = nil
             end
           else
             finish_time = line_data.timestamp if line_data.step == :finish
           end
         end
       end
       
       if total_rotations > 0
         report_data[:index_rebuilds] = total_rotations
         report_data[:average_time_per_rebuild] = sprintf("%.4f", total_length_rotations/total_rotations)
       end
     rescue Errno::ENOENT => error
       return error("Unable to find the searchd log file", "Could not find the searchd log at the specified path: #{option(:query_log_path)}.")
     rescue Exception => error
       return error("Error while processing searchd log:\n#{error.class}: #{error.message}", error.backtrace.join("\n"))
     end
     # the time
     # should be fixed so that it stores the time of the last log entry from both logs
     remember(:last_request_time, Time.now)
     report(report_data)
  end
private

  QueryData = Struct.new(:timestamp, :time_spent, :results_returned)
  
  LogData = Struct.new(:timestamp, :step)
  
  #based off of http://kobesearch.cpan.org/htdocs/Sphinx-Log-Parser/Sphinx/Log/Parser.pm.html
  def parse_query_line(line)
    time = line.match(/\[(.*?)\]/).captures.first
    time_spent = line.match(/\]\s([\d\.]+).*?\[/).captures.first
    results_returned = line.match(/\s(\d+)\s\(/).captures.first
    QueryData.new(Time.parse(time), time_spent.to_f, results_returned.to_i)
  end
  
  def parse_log_line(line)
    time = line.match(/\[(.*?)\]/).captures.first    
    step = if line.match('rotating finished')
      :finish
    elsif line.match('rotating indices')
      :start
    else
      :intermediate
    end
    LogData.new(Time.parse(time), step)
  end
  
end

2010-01-03T20:52:22-05:00

Monitors "Sphinx":http://www.sphinxsearch.com/, an open-source SQL full-text search engine, reporting the metrics below: * Queries per-minute * Average query time * Average time per rebuild * Index rebuilds * Average results returned h2. Installing Just provide the path to the searchd log files. Note that the first run will not return any data. h2. Example Data !http://img.skitch.com/20100104-1isag92chgygd9mgxsnkfappw5.png! h2. Alerts Two default triggers -- which monitor trends in the _query rate_ and _average query time_ -- are installed with this plugin. These triggers can be adjusted to your system. An example alert: !http://img.skitch.com/20100104-xbkjq182m4ehctf1wk4tinfmw5.png! false 241 --- |- metadata: query_rate: precision: 2 label: Queries units: queries/min average_query_time: precision: 2 units: sec average_time_per_rebuild: precision: 2 units: req/min triggers: - type: trend dname: query_rate direction: UP percentage_change: 300 duration: 60 window_reference: LAST_WEEK min_value: 3 - type: trend dname: average_query_time direction: UP percentage_change: 200 duration: 60 window_reference: LAST_WEEK min_value: 0.01 Sphinx Monitoring

0.0

Monitors Sphinx, an open-source SQL full-text search engine (http://www.sphinxsearch.com/) for the following metrics by looking at the searchd logs: * Queries per-minute * Average query time * Average time per rebuild * Index rebuilds * Average results returned Note that the first run will not return any data. ---- Created by Sam Sinensky (http://github.com/samsinensky) at Code and Beats (http://codeandbeats.com/) 12/29/09

Monitors Sphinx, an open-source SQL full-text search engine.

linux osx

2010-01-04T18:04:15-05:00

http://github.com/highgroove/scout-plugins/raw/master/sphinx_monitor/sphinx_monitor.rb