Stuff media files into their own table - warvox - VoIP based wardialing tool, forked from rapid7/warvox.
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
---
(DIR) commit 62d6590a9a59c044bf16fc5a750600a9ffd3ea5e
(DIR) parent 65579c3b33e9c77d614fae06c76e09381980c311
(HTM) Author: HD Moore <hd_moore@rapid7.com>
Date: Fri, 28 Dec 2012 13:22:53 -0600
Stuff media files into their own table
Diffstat:
M app/controllers/analyze_controller… | 2 +-
M app/controllers/dial_results_contr… | 36 ++++++++++++-------------------
M app/models/dial_job.rb | 2 +-
M app/models/dial_result.rb | 12 +++++++++++-
A app/models/dial_result_medium.rb | 3 +++
M bin/analyze_result.rb | 6 ++++--
M bin/export_audio.rb | 11 ++++++-----
A db/migrate/20121228171549_initial_… | 71 +++++++++++++++++++++++++++++++
M db/schema.rb | 38 ++++++++++++++++---------------
M lib/warvox/jobs/analysis.rb | 42 ++++++++++++++++----------------
M lib/warvox/jobs/dialer.rb | 7 +++++--
11 files changed, 157 insertions(+), 73 deletions(-)
---
(DIR) diff --git a/app/controllers/analyze_controller.rb b/app/controllers/analyze_controller.rb
@@ -98,7 +98,7 @@ class AnalyzeController < ApplicationController
cpath = nil
cdata = "File not found"
- res = DialResult.find(params[:result_id])
+ res = DialResultMedium.where(:dial_result_id => params[:result_id].to_i).first
if res
case params[:type]
(DIR) diff --git a/app/controllers/dial_results_controller.rb b/app/controllers/dial_results_controller.rb
@@ -79,15 +79,17 @@ class DialResultsController < ApplicationController
:per_page => 30
)
- if(@dial_results)
- @call_results = {
- :Timeout => DialResult.count(:conditions =>['dial_job_id = ? and completed = ?', params[:id], false]),
- :Busy => DialResult.count(:conditions =>['dial_job_id = ? and busy = ?', params[:id], true]),
- :Answered => DialResult.count(:conditions =>['dial_job_id = ? and completed = ?', params[:id], true]),
- }
+ unless @dial_results and @dial_results.length > 0
+ redirect_to :action => :index
+ return
end
+ @call_results = {
+ :Timeout => DialResult.count(:conditions =>['dial_job_id = ? and completed = ?', params[:id], false]),
+ :Busy => DialResult.count(:conditions =>['dial_job_id = ? and busy = ?', params[:id], true]),
+ :Answered => DialResult.count(:conditions =>['dial_job_id = ? and completed = ?', params[:id], true]),
+ }
- respond_to do |format|
+ respond_to do |format|
format.html # index.html.erb
format.xml { render :xml => @dial_results }
end
@@ -98,6 +100,11 @@ class DialResultsController < ApplicationController
def show
@dial_result = DialResult.find(params[:id])
+ unless @dial_result
+ redirect_to :action => :index
+ return
+ end
+
respond_to do |format|
format.html # show.html.erb
format.xml { render :xml => @dial_result }
@@ -159,23 +166,8 @@ class DialResultsController < ApplicationController
def purge
@job = DialJob.find(params[:id])
- @job.dial_results.each do |r|
- r.destroy
- end
@job.destroy
- dir = nil
- jid = @job.id
- dfd = Dir.new(WarVOX::Config.data_path)
- dfd.entries.each do |ent|
- j,m = ent.split('-', 2)
- if (m and j == jid)
- dir = File.join(WarVOX::Config.data_path, ent)
- end
- end
-
- FileUtils.rm_rf(dir) if dir
-
respond_to do |format|
format.html { redirect_to :action => 'index' }
format.xml { head :ok }
(DIR) diff --git a/app/models/dial_job.rb b/app/models/dial_job.rb
@@ -1,7 +1,7 @@
class DialJob < ActiveRecord::Base
attr_accessor :range_file
- has_many :dial_results
+ has_many :dial_results, :dependent => :destroy
validates_presence_of :range, :lines, :seconds
validates_numericality_of :lines, :less_than => 256, :greater_than => 0
(DIR) diff --git a/app/models/dial_result.rb b/app/models/dial_result.rb
@@ -1,7 +1,7 @@
class DialResult < ActiveRecord::Base
belongs_to :provider
belongs_to :dial_job
-
+ has_one :dial_result_medium, :dependent => :delete
has_many :matches, :class_name => 'DialResult', :finder_sql => proc {
'SELECT dial_results.*, ' +
@@ -21,4 +21,14 @@ class DialResult < ActiveRecord::Base
"dial_results.id != \'#{id}\' " +
'ORDER BY matchscore DESC'
}
+
+
+ def media
+ DialResultMedium.find_or_create_by_dial_result_id(self[:id])
+ end
+
+ def media_fields
+ DialResultMedium.columns_hash.keys.reject{|x| x =~ /^id|_id$/}
+ end
+
end
(DIR) diff --git a/app/models/dial_result_medium.rb b/app/models/dial_result_medium.rb
@@ -0,0 +1,3 @@
+class DialResultMedium < ActiveRecord::Base
+ belongs_to :dial_result
+end
(DIR) diff --git a/bin/analyze_result.rb b/bin/analyze_result.rb
@@ -16,14 +16,16 @@ require 'warvox'
#
inp = ARGV.shift || exit(0)
-$0 = "warvox(analyzer): #{inp}"
+num = ARGV.shift || exit(0)
+
+$0 = "warvox(analyzer): #{inp} #{num}"
$stdout.write(
Marshal.dump(
WarVOX::Jobs::CallAnalysis.new(
0
).analyze_call(
- inp
+ inp, num
)
)
)
(DIR) diff --git a/bin/export_audio.rb b/bin/export_audio.rb
@@ -40,7 +40,7 @@ end
if(not job)
$stderr.puts "Listing all available jobs"
$stderr.puts "=========================="
- DialJob.find(:all).each do |j|
+ DialJob.all.each do |j|
puts "#{j.id}\t#{j.started_at} --> #{j.completed_at}"
end
exit
@@ -51,13 +51,14 @@ end
begin
cnt = 0
- job = DialJob.find(job.to_i)
- job.dial_results.each do |r|
+ DialResult.where(:dial_job_id => job.to_i).find_each do |r|
next if not r.number
- next if r.audio.to_s.length == 0
+ m = r.media
+ next if not m
+ next if m.audio.to_s.length == 0
out = ::File.join(dir, "#{r.number}.raw")
::File.open(out, "wb") do |fd|
- fd.write( r.audio )
+ fd.write( m.audio )
end
cnt += 1
end
(DIR) diff --git a/db/migrate/20121228171549_initial_schema.rb b/db/migrate/20121228171549_initial_schema.rb
@@ -0,0 +1,71 @@
+class InitialSchema < ActiveRecord::Migration
+ def up
+
+ # Require the intarray extension
+ execute("CREATE EXTENSION IF NOT EXISTS intarray")
+
+ create_table "dial_jobs" do |t|
+ t.timestamps
+ t.text "range"
+ t.integer "seconds"
+ t.integer "lines"
+ t.text "status"
+ t.integer "progress"
+ t.datetime "started_at"
+ t.datetime "completed_at"
+ t.boolean "processed"
+ t.text "cid_mask"
+ end
+
+ create_table "dial_results" do |t|
+ t.timestamps
+ t.text "number"
+ t.integer "dial_job_id"
+ t.integer "provider_id"
+ t.boolean "completed"
+ t.boolean "busy"
+ t.integer "seconds"
+ t.integer "ringtime"
+ t.boolean "processed"
+ t.datetime "processed_at"
+ t.text "cid"
+ t.float "peak_freq"
+ t.text "peak_freq_data"
+ t.text "sig_data"
+ t.text "line_type"
+ t.text "notes"
+ t.text "signatures"
+ t.integer "fprint", :array => true
+ end
+
+ create_table "dial_result_media" do |t|
+ t.integer "dial_result_id"
+ t.binary "audio"
+ t.binary "mp3"
+ t.binary "png_big"
+ t.binary "png_big_dots"
+ t.binary "png_big_freq"
+ t.binary "png_sig"
+ t.binary "png_sig_freq"
+ end
+
+ create_table "providers" do |t|
+ t.timestamps
+ t.text "name"
+ t.text "host"
+ t.integer "port"
+ t.text "user"
+ t.text "pass"
+ t.integer "lines"
+ t.boolean "enabled"
+ end
+
+ end
+
+ def down
+ remove_table "providers"
+ remove_table "dial_result_media"
+ remove_table "dial_results"
+ remove_table "dial_jobs"
+ end
+end
(DIR) diff --git a/db/schema.rb b/db/schema.rb
@@ -11,12 +11,13 @@
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20110801000003) do
+ActiveRecord::Schema.define(:version => 20121228171549) do
- # Require the intarray extension
- execute "CREATE EXTENSION IF NOT EXISTS intarray"
+ add_extension "intarray"
create_table "dial_jobs", :force => true do |t|
+ t.datetime "created_at", :null => false
+ t.datetime "updated_at", :null => false
t.text "range"
t.integer "seconds"
t.integer "lines"
@@ -25,12 +26,23 @@ ActiveRecord::Schema.define(:version => 20110801000003) do
t.datetime "started_at"
t.datetime "completed_at"
t.boolean "processed"
- t.datetime "created_at", :null => false
- t.datetime "updated_at", :null => false
t.text "cid_mask"
end
+ create_table "dial_result_media", :force => true do |t|
+ t.integer "dial_result_id"
+ t.binary "audio"
+ t.binary "mp3"
+ t.binary "png_big"
+ t.binary "png_big_dots"
+ t.binary "png_big_freq"
+ t.binary "png_sig"
+ t.binary "png_sig_freq"
+ end
+
create_table "dial_results", :force => true do |t|
+ t.datetime "created_at", :null => false
+ t.datetime "updated_at", :null => false
t.text "number"
t.integer "dial_job_id"
t.integer "provider_id"
@@ -38,10 +50,7 @@ ActiveRecord::Schema.define(:version => 20110801000003) do
t.boolean "busy"
t.integer "seconds"
t.integer "ringtime"
- t.text "rawfile"
t.boolean "processed"
- t.datetime "created_at", :null => false
- t.datetime "updated_at", :null => false
t.datetime "processed_at"
t.text "cid"
t.float "peak_freq"
@@ -50,25 +59,18 @@ ActiveRecord::Schema.define(:version => 20110801000003) do
t.text "line_type"
t.text "notes"
t.text "signatures"
- t.integer "fprint", :array => true
- t.binary "audio"
- t.binary "mp3"
- t.binary "png_big"
- t.binary "png_big_dots"
- t.binary "png_big_freq"
- t.binary "png_sig"
- t.binary "png_sig_freq"
+ t.integer "fprint", :array => true
end
create_table "providers", :force => true do |t|
+ t.datetime "created_at", :null => false
+ t.datetime "updated_at", :null => false
t.text "name"
t.text "host"
t.integer "port"
t.text "user"
t.text "pass"
t.integer "lines"
- t.datetime "created_at", :null => false
- t.datetime "updated_at", :null => false
t.boolean "enabled"
end
(DIR) diff --git a/lib/warvox/jobs/analysis.rb b/lib/warvox/jobs/analysis.rb
@@ -74,15 +74,7 @@ class Analysis < Base
end
def start_processing
- todo = ::DialResult.find_all_by_dial_job_id(@name)
- jobs = []
- todo.each do |r|
- next if r.processed
- next if not r.completed
- next if r.busy
- jobs << r
- end
-
+ jobs = ::DialResult.where(:dial_job_id => @name, :processed => false, :completed => true, :busy => false).all
max_threads = WarVOX::Config.analysis_threads
while(not jobs.empty?)
@@ -102,29 +94,35 @@ class Analysis < Base
end
end
- def run_analyze_call(r)
- $stderr.puts "DEBUG: Processing audio for #{r.number}..."
+ def run_analyze_call(dr)
+ $stderr.puts "DEBUG: Processing audio for #{dr.number}..."
bin = File.join(WarVOX::Base, 'bin', 'analyze_result.rb')
tmp = Tempfile.new("Analysis")
begin
+ mr = dr.media
::File.open(tmp.path, "wb") do |fd|
- fd.write(r.audio)
+ fd.write(mr.audio)
end
- pfd = IO.popen("#{bin} '#{tmp.path}'")
+ pfd = IO.popen("#{bin} '#{tmp.path}' '#{ dr.number.gsub("'", '') }'")
out = Marshal.load(pfd.read) rescue nil
pfd.close
return if not out
+ mf = dr.media_fields
out.each_key do |k|
- r[k] = out[k]
+ if mf.include?(k.to_s)
+ mr[k] = out[k]
+ else
+ dr[k] = out[k]
+ end
end
- r.processed_at = Time.now
- r.processed = true
+ dr.processed_at = Time.now
+ dr.processed = true
rescue ::Interrupt
ensure
@@ -132,24 +130,26 @@ class Analysis < Base
tmp.unlink
end
+ mr.save
+
true
end
# Takes the raw file path as an argument, returns a hash
- def analyze_call(input)
+ def analyze_call(input, num=nil)
return if not input
return if not File.exist?(input)
- bname = File.expand_path(File.dirname(input))
- num = File.basename(input)
- res = {}
+ bname = File.expand_path(File.dirname(input))
+ num ||= File.basename(input)
+ res = {}
#
# Create the signature database
#
raw = WarVOX::Audio::Raw.from_file(input)
- fft = KissFFT.fftr(8192, 8000, 1, raw.samples)
+ fft = KissFFT.fftr(8192, 8000, 1, raw.samples) || []
freq = raw.to_freq_sig_arr()
(DIR) diff --git a/lib/warvox/jobs/dialer.rb b/lib/warvox/jobs/dialer.rb
@@ -44,7 +44,7 @@ class Dialer < Base
def get_providers
res = []
- ::Provider.find_all_by_enabled(true).each do |prov|
+ ::Provider.where(:enabled => true).all.each do |prov|
info = {
:name => prov.name,
:id => prov.id,
@@ -175,10 +175,13 @@ class Dialer < Base
res.seconds = (byte / 16000) # 8khz @ 16-bit
res.ringtime = ring
res.processed = false
+ res.save
if(File.exists?(out))
File.open(out, "rb") do |fd|
- res.audio = fd.read(fd.stat.size)
+ med = res.media
+ med.audio = fd.read(fd.stat.size)
+ med.save
end
end