#!/usr/bin/ruby
# SPDX-License-Identifier: GPL-2.0-only
#
# Observe irq and softirq in top fashion
# (c) 2014 <abc@telekom.ru>
# License: GPL-2.0-only.
require 'getoptlong'
require 'curses'
require 'stringio'
@imode = :both
@omode = :table
@color = true
@showrps = false
GetoptLong.new(
["--help", "-h", GetoptLong::NO_ARGUMENT],
["--batch", "-b", GetoptLong::NO_ARGUMENT],
["--delay", "-d", GetoptLong::REQUIRED_ARGUMENT],
["--top", "-t", GetoptLong::NO_ARGUMENT],
["--table", "-x", GetoptLong::NO_ARGUMENT],
["--soft", "-s", GetoptLong::NO_ARGUMENT],
["--softirq", GetoptLong::NO_ARGUMENT],
["--softirqs", GetoptLong::NO_ARGUMENT],
["--irq", "-i", GetoptLong::NO_ARGUMENT],
["--irqs", GetoptLong::NO_ARGUMENT],
["--reverse", "-r", GetoptLong::NO_ARGUMENT],
["--nocolor", "-C", GetoptLong::NO_ARGUMENT],
["--eth", "-e", "--pps", GetoptLong::NO_ARGUMENT],
["--rps", "-R", "--xps", GetoptLong::NO_ARGUMENT]
).each do |opt, arg|
case opt
when '--help'
puts " Shows interrupt rates (per second) per cpu."
puts " Also shows irq affinity ('.' for disabled cpus),"
puts " and rps/xps affinity ('+' rx, '-' tx, '*' tx/rx)."
puts " Can show packet rate per eth queue."
puts
puts " Usage: #{$0} [-h] [-d #{@delay}] [-b] [-t|-x] [-i|-s] [-r]"
puts " -d --delay=n refresh interval"
puts " -s --softirq select softirqs only"
puts " -i --irq select hardware irqs only"
puts " -e --eth show extra eth stats (from ethtool)"
puts " -R --rps enable display of rps/xps"
puts " -x --table output in table mode (default)"
puts " -t --top output in flat top mode"
puts " -b --batch output non-interactively"
puts " -r --reverse reverse sort order"
puts " -C --nocolor disable colors"
puts
puts " Rates marked as '.' is forbidden by smp_affinity mask."
exit 0
when '--reverse'
@reverse = !@reverse
when '--batch'
@batch = true
@reverse = !@reverse if @omode == :top
when '--delay'
@delay = arg.to_i
when '--top'
@omode = :top
when '--table'
@omode = :table
when /--irq/
@imode = :irq
when /--soft/
@imode = :soft
when /--pps/
@pps = true
when /--nocolor/
@color = false
when /--rps/
@showrps = !@showrps
end
end
if !@delay && ARGV[0].to_f > 0
@delay = ARGV.shift.to_f
else
@delay = 5
end
@count = ARGV.shift.to_f if ARGV[0].to_i > 0
def read_table(tag, file)
@cpus = []
lines = IO.readlines(file)
@cpus = lines[0].scan(/CPU\d+/)
@icpus = @cpus if tag == 'i'
lines[2..-1].each do |li|
irq, stat, desc = li.match(/^\s*(\S+):((?:\s+\d+)+)(.*)$/).captures
stat = stat.scan(/\d+/)
@irqs << [tag, irq, desc]
stat.each_with_index do |val, i|
# interruptsN, 's|i', irq'N', 'cpuX', 'descr...'
@stats << [val.to_i, tag, irq, @cpus[i], desc.strip]
end
end
end
def read_procstat
@cstat = {}
lines = IO.readlines("/proc/stat").grep(/^cpu\d+ /)
lines.each do |li|
c, *d = li.split(" ")
d = d.map {|e| e.to_i}
@cstat[c] = d
end
end
def read_affinity
@aff = {}
Dir.glob("/proc/irq/*/smp_affinity").each do |af|
irq = af[%r{\d+}].to_i
a = IO.read(af).strip.to_i(16)
@aff[irq] = a
end
end
# list ethernet devices
def net_devices_pci
Dir['/sys/class/net/*'].reject do |f|
f += "/device" unless File.symlink?(f)
if File.symlink?(f)
!(File.readlink(f) =~ %r{devices/pci})
else
false
end
end.map {|f| File.basename(f)}
end
@devlist = net_devices_pci
@devre = Regexp.union(@devlist)
def get_rps(desc)
@rps = @xps = 0
return unless @showrps
return if @devlist.empty?
dev = desc[/\b(#{@devre})\b/, 1]
return unless dev
return unless desc =~ /-(tx|rx)+-\d+/i
qnr = desc[/-(\d+)\s*$/, 1]
return unless qnr
begin
@rps = IO.read("/sys/class/net/#{dev}/queues/rx-#{qnr}/rps_cpus").hex if desc =~ /rx/i
@xps = IO.read("/sys/class/net/#{dev}/queues/tx-#{qnr}/xps_cpus").hex if desc =~ /tx/i
rescue
end
end
def calc_rps(cpu)
m = 0
m |= 1 if @rps & (1 << cpu) != 0
m |= 2 if @xps & (1 << cpu) != 0
" +-*".slice(m, 1)
end
# ethtool -S eth0
def ethtool_grab_stat(dev = nil)
unless dev
@esto = @est if @est
@est = Hash.new { |h,k| h[k] = Hash.new(&h.default_proc) }
@devlist = net_devices_pci
@devre = Regexp.union(@devlist)
# own time counter because this stat could be paused
@ehts = @ets if @ets
@ets = @ts
@edt = @ets - @ehts if @ehts
@devlist.each {|e| ethtool_grab_stat(e)}
return
end
h = Hash.new {|k,v| k[v] = Array.new}
t = `ethtool -S #{dev} 2>/dev/null`
return if t == ''
t.split("\n").map { |e|
e.split(':')
}.reject { |e|
!e[1]
}.each { |k,v|
k.strip!
v = v.strip.to_i
if k =~ /^.x_queue_(\d+)_/
t = k.split('_', 4)
qdir = t[0]
qnr = t[2]
qk = t[3]
@est[dev][qdir][qnr][qk] = v
else
@est[dev][k] = v
end
}
end
def e_queue_stat(dev, qdir, qnr, k)
n = @est[dev][qdir][qnr][k]
o = @esto[dev][qdir][qnr][k]
d = (n - o) / @edt
if d > 0
"%s:%d" % [qdir, d]
else
nil
end
end
def e_dev_stat(dev, k, ks)
n = @est[dev][k]
o = @esto[dev][k]
r = (n - o) / @edt
ks = k unless ks
"%s:%d" % [ks, r]
end
def e_queue_stat_err(dev, qdir, qnr)
r = []
ek = @est[dev][qdir][qnr].keys.reject{|e| e =~ /^(bytes|packets)$/}
ek.each do |k|
n = @est[dev][qdir][qnr][k]
o = @esto[dev][qdir][qnr][k]
d = n - o
r << "%s_%s:%d" % [qdir, k, d] if d.to_i > 0
end
r
end
# this is not rate
def e_dev_stat_sum(dev, rk, ks)
ek = @est[dev].keys.reject{|ek| !(ek =~ rk)}
n = ek.inject(0) {|sum,k| sum += @est[dev][k].to_i}
o = ek.inject(0) {|sum,k| sum += @esto[dev][k].to_i rescue 0}
r = (n - o)
if r > 0
"%s:%d" % [ks, r]
else
nil
end
end
def print_ethstat(desc)
return if @devlist.empty?
dev = desc[/\b(#{@devre})\b/, 1]
return unless dev
unless @esto && @est
print ' []'
return
end
t = []
if desc =~ /-(tx|rx)+-\d+/i
qnr = desc[/-(\d+)\s*$/, 1]
if qnr
if desc =~ /rx/i
t << e_queue_stat(dev, "rx", qnr, "packets")
t += e_queue_stat_err(dev, "rx", qnr)
end
if desc =~ /tx/i
t << e_queue_stat(dev, "tx", qnr, "packets")
t += e_queue_stat_err(dev, "tx", qnr)
end
end
else
t << e_dev_stat(dev, "rx_packets", 'rx')
t << e_dev_stat(dev, "tx_packets", 'tx')
t << e_dev_stat_sum(dev, /_err/, 'err')
t << e_dev_stat_sum(dev, /_drop/, 'drop')
end
t.delete(nil)
print ' [' + t.join(' ') + ']'
end
def grab_stat
# @h[istorical]
@hstats = @stats
@hcstat = @cstat
@hts = @ts
@stats = []
@irqs = []
@ts = Time.now
@dt = @ts - @hts if @hts
read_table 'i', "/proc/interrupts"
read_table 's', "/proc/softirqs"
read_affinity
read_procstat
ethtool_grab_stat if @pps
end
def calc_speed
s = []
# calc speed
h = Hash.new(0)
@hstats.each do |v, t, i, c, d|
h[[t, i, c]] = v
end
# output
@h = {}
@t = Hash.new(0) # rate per cpu
@w = Hash.new(0) # irqs per irqN
@s = @stats.map do |v, t, i, c, d|
rate = (v - h[[t, i, c]]) / @dt
@t[c] += rate if t == 'i'
@w[[t, i]] += (v - h[[t, i, c]])
@h[[t, i, c]] = rate
[rate, v, t, i, c, d]
end
end
def calc_cpu
@cBusy = Hash.new(0)
@cHIrq = Hash.new(0)
@cSIrq = Hash.new(0)
# user, nice, system, [3] idle, [4] iowait, irq, softirq, etc.
@cstat.each do |c, d|
d = d.zip(@hcstat[c]).map {|a, b| a - b}
c = c.upcase
sum = d.reduce(:+)
@cBusy[c] = 100 - (d[3] + d[4]).to_f / sum * 100
@cHIrq[c] = (d[5]).to_f / sum * 100
@cSIrq[c] = (d[6]).to_f / sum * 100
end
end
def show_top
@s.sort!.reverse!
@s.reverse! if @reverse
rej = nil
rej = 's' if @imode == :irq
rej = 'i' if @imode == :soft
@s.each do |s, v, t, i, c, d|
next if t == rej
if s > 0
print "%9.1f %s %s <%s> %s" % [s, c.downcase, t, i, d]
print_ethstat(d) if @pps
puts
end
end
end
@ifilter = {}
def show_interrupts
maxlen = 7
@irqs.reverse! if @reverse
print "%s %*s " % [" ", maxlen, " "]
@icpus.each { |c| print " %6s" % c }
puts
# load
print "%*s: " % [maxlen + 2, "cpuUtil"]
@icpus.each { |c| print " %6.1f" % @cBusy[c] }
puts " total CPU utilization %"
#
print "%*s: " % [maxlen + 2, "%irq"]
@icpus.each { |c| print " %6.1f" % @cHIrq[c] }
puts " hardware IRQ CPU util%"
print "%*s: " % [maxlen + 2, "%sirq"]
@icpus.each { |c| print " %6.1f" % @cSIrq[c] }
puts " software IRQ CPU util%"
# total
print "%*s: " % [maxlen + 2, "irqTotal"]
@icpus.each { |c| print " %6d" % @t[c] }
puts " total hardware IRQs"
rej = nil
rej = 's' if @imode == :irq
rej = 'i' if @imode == :soft
@irqs.each do |t, i, desc|
next if t == rej
# include incrementally and all eth
unless @ifilter[[t, i]] || @showall
next unless @w[[t, i]] > 0 || desc =~ /eth/
@ifilter[[t, i]] = true
end
print "%s %*s: " % [t.to_s, maxlen, i.slice(0, maxlen)]
rps = get_rps(desc)
@icpus.each do |c|
cpu = c[/\d+/].to_i
aff = @aff[i.to_i]
off = ((aff & 1 << cpu) ==0)? true : false if aff
fla = calc_rps(cpu)
begin
v = @h[[t, i, c]]
if v > 0 || !off
print "%6d%c" % [v, fla]
elsif aff
print "%6s%c" % [".", fla]
end
rescue
end
end
print desc
print_ethstat(desc) if @pps
puts
end
end
def select_output
if @omode == :top
show_top
else
show_interrupts
end
end
def curses_choplines(text)
cols = Curses.cols - 1
rows = Curses.lines - 2
lines = text.split("\n").map {|e| e.slice(0, cols)}.slice(0, rows)
text = lines.join("\n")
text << "\n" * (rows - lines.size) if lines.size < rows
text
end
def show_help
puts "irqtop help:"
puts
puts " In table view, cells marked with '.' mean this hw irq is"
puts " disabled via /proc/irq/<irq>/smp_affinity"
puts " Interactive keys:"
puts " i Toggle (hardware) irqs view"
puts " s Toggle software irqs (softirqs) view"
puts " e Show eth stat per queue"
puts " R Show rps/xps affinity"
puts " t Flat top display mode"
puts " x Table display mode"
puts " r Reverse rows order"
puts " c Toggle colors (for eth)"
puts " a Show lines with zero rate (all)"
puts " A Clear lines with zero rates"
puts " . Pause screen updating"
puts " h,? This help screen"
puts " q Quit."
puts " Any other key will update display."
puts
puts "Press any key to continue."
end
hostname = `hostname`.strip
#
grab_stat
sleep 0.5
COLOR_GREEN = "\033[0;32m"
COLOR_YELLOW = "\033[0;33m"
COLOR_CYAN = "\033[0;36m"
COLOR_RED = "\033[0;31m"
COLOR_OFF = "\033[m"
def tty_printline(t)
latr = nil # line color
if t =~ /-rx-/
latr = COLOR_GREEN
elsif t =~ /-tx-/
latr = COLOR_YELLOW
elsif t =~ /\beth/
latr = COLOR_CYAN
end
print latr if latr
if t =~ /cpuUtil:|irq:|sirq:/
# colorize percentage values
t.scan(/\s+\S+/) do |e|
eatr = nil
if e =~ /^\s*[\d.]+$/
if e.to_i >= 90
eatr = COLOR_RED
elsif e.to_i <= 10
eatr = COLOR_GREEN
else
eatr = COLOR_YELLOW
end
end
print eatr if eatr
print e
print (latr)? latr : COLOR_OFF if eatr
end
elsif latr && t =~ / \[[^\]]+\]$/
# colorize eth stats
print $`
print COLOR_OFF if latr
$&.scan(/(.*?)(\w+)(:)(\d+)/) do |e|
eatr = nil
case e[1]
when 'rx'
eatr = COLOR_GREEN
when 'tx'
eatr = COLOR_YELLOW
else
eatr = COLOR_RED
end
eatr = nil if e[3].to_i == 0
print e[0]
print eatr if eatr
print e[1..-1].join
print (latr)? latr : COLOR_OFF if eatr
end
print $'
else
print t
end
print COLOR_OFF if latr
puts
end
def tty_output
if @color
$stdout = StringIO.new
yield
$stdout.rewind
txt = $stdout.read
$stdout = STDOUT
txt.split("\n", -1).each do |li|
tty_printline(li)
end
else
yield
end
end
if @batch
@color = @color && $stdout.tty?
loop do
grab_stat
calc_speed
calc_cpu
puts "#{hostname} - irqtop - #{Time.now}"
tty_output {
select_output
}
$stdout.flush
break if @count && (@count -= 1) == 0
sleep @delay
end
exit 0
end
Curses.init_screen
Curses.start_color
Curses.cbreak
Curses.noecho
Curses.nonl
Curses.init_pair(1, Curses::COLOR_GREEN, Curses::COLOR_BLACK);
Curses.init_pair(2, Curses::COLOR_YELLOW, Curses::COLOR_BLACK);
Curses.init_pair(3, Curses::COLOR_CYAN, Curses::COLOR_BLACK);
Curses.init_pair(4, Curses::COLOR_RED, Curses::COLOR_BLACK);
$stdscr = Curses.stdscr
$stdscr.keypad(true)
def curses_printline(t)
latr = nil # line color
if t =~ /-rx-/
latr = Curses.color_pair(1)
elsif t =~ /-tx-/
latr = Curses.color_pair(2)
elsif t =~ /\beth/
latr = Curses.color_pair(3)
end
$stdscr.attron(latr) if latr
if t =~ /cpuUtil:|irq:|sirq:/
# colorize percentage values
t.scan(/\s+\S+/) do |e|
eatr = nil
if e =~ /^\s*[\d.]+$/
if e.to_i >= 90
eatr = Curses.color_pair(4)
elsif e.to_i <= 10
eatr = Curses.color_pair(1)
else
eatr = Curses.color_pair(2)
end
end
$stdscr.attron(eatr) if eatr
$stdscr.addstr("#{e}")
$stdscr.attroff(eatr) if eatr
end
elsif latr && t =~ / \[[^\]]+\]$/
# colorize eth stats
$stdscr.addstr($`)
$stdscr.attroff(latr) if latr
$&.scan(/(.*?)(\w+)(:)(\d+)/) do |e|
eatr = nil
case e[1]
when 'rx'
eatr = Curses.color_pair(1)
when 'tx'
eatr = Curses.color_pair(2)
else
eatr = Curses.color_pair(4)
end
eatr = nil if e[3].to_i == 0
$stdscr.addstr(e[0])
$stdscr.attron(eatr) if eatr
$stdscr.addstr(e[1..-1].join)
$stdscr.attroff(eatr) if eatr
end
$stdscr.addstr($' + "\n")
else
$stdscr.addstr("#{t}\n")
end
$stdscr.attroff(latr) if latr
end
def curses_output
$stdout = StringIO.new
yield
$stdout.rewind
text = $stdout.read
$stdout = STDOUT
txt = curses_choplines(text)
if @color
txt.split("\n", -1).each_with_index do |li, i|
$stdscr.setpos(i, 0)
curses_printline(li)
end
else
$stdscr.setpos(0, 0)
$stdscr.addstr(txt)
end
$stdscr.setpos(1, 0)
Curses.refresh
end
def curses_enter(text, echo = true)
$stdscr.setpos(1, 0)
$stdscr.addstr(text + "\n")
$stdscr.setpos(1, 0)
Curses.attron(Curses::A_BOLD)
$stdscr.addstr(text)
Curses.attroff(Curses::A_BOLD)
Curses.refresh
Curses.echo if echo
Curses.timeout = -1
line = Curses.getstr
Curses.noecho
line
end
loop do
grab_stat
calc_speed
calc_cpu
curses_output {
puts "#{hostname} - irqtop - #{Time.now}"
select_output
}
Curses.timeout = @delay * 1000
ch = Curses.getch.chr rescue nil
case ch
when "\f"
Curses.clear
when "q", "Z", "z"
break
when 'i'
@imode = (@imode == :both)? :soft : :both
when 's'
@imode = (@imode == :both)? :irq : :both
when 't'
@omode = (@omode == :top)? :table : :top
when 'x'
@omode = (@omode == :table)? :top : :table
when 'e', 'p'
@pps = !@pps
when 'r'
@reverse = !@reverse
when 'c'
@color = !@color
when 'A'
@ifilter = {}
when 'a'
@ifilter = {}
@showall = !@showall
when 'R'
@showrps = !@showrps
when '.'
curses_enter("Pause, press enter to to continue: ", false)
when 'd'
d = curses_enter("Enter display interval: ")
@delay = d.to_f if d.to_f > 0
when 'h', '?'
curses_output { show_help }
Curses.timeout = -1
ch = Curses.getch.chr rescue nil
break if ch == 'q'
end
end