Steve Linabery
2009-Mar-13 18:50 UTC
[Ovirt-devel] [PATCH server] Add method to Stats.rb to return regularly-spaced, padded aggregate stats
Also an instance variable name change for StatsDataList for clarity's sake.
---
src/app/controllers/graph_controller.rb | 4 +-
src/app/util/stats/Stats.rb | 89 ++++++++++++++++++++++++++++++-
src/app/util/stats/StatsDataList.rb | 12 ++--
src/app/util/stats/StatsRequest.rb | 6 ++-
4 files changed, 101 insertions(+), 10 deletions(-)
diff --git a/src/app/controllers/graph_controller.rb
b/src/app/controllers/graph_controller.rb
index 06f1c06..558787c 100644
--- a/src/app/controllers/graph_controller.rb
+++ b/src/app/controllers/graph_controller.rb
@@ -55,7 +55,7 @@ class GraphController < ApplicationController
dataFunction)
}
}
- statsList = getAggregateStatsData?(requestList)
+ statsList = getPaddedAggregateStatsData?(requestList,startTime,endTime)
stat = statsList[0]
vectors = [ ]
@@ -68,7 +68,7 @@ class GraphController < ApplicationController
graph = { :vectors => vectors,
:max_value => stat.get_max_value?,
:description => target,
- :resolution => stat.get_resolution?
+ :resolution => stat.get_interval?
}
render :json => graph
end
diff --git a/src/app/util/stats/Stats.rb b/src/app/util/stats/Stats.rb
index f7e9dbe..fee3b28 100644
--- a/src/app/util/stats/Stats.rb
+++ b/src/app/util/stats/Stats.rb
@@ -24,7 +24,6 @@ require 'util/stats/StatsData'
require 'util/stats/StatsDataList'
require 'util/stats/StatsRequest'
-
# This fetches a rolling average, basically average points before and after.
@@ -42,6 +41,7 @@ def fetchRollingAve?(rrdPath, start, endTime, interval,
myFunction, lIndex, retu
(fstart, fend, names, data, interval) = RRD.fetch(rrdPath,
"--start", start.to_s, \
"--end", endTime.to_s, myFunction,
"-r", interval.to_s)
+ returnList.set_interval interval
i = 0
# For some reason, we get an extra datapoint at the end. Just chop it off
now...
data.delete_at(-1)
@@ -110,6 +110,7 @@ def fetchRollingCalcUsedData?(rrdPath, start, endTime,
interval, myFunction, lIn
(fstart, fend, names, data, interval) = RRD.fetch(rrdPath,
"--start", start.to_s, \
"--end", endTime.to_s, lFunc,
"-r", interval.to_s)
+ returnList.set_interval interval
i = 0
# For some reason, we get an extra datapoint at the end. Just chop it off
now...
data.delete_at(-1)
@@ -177,6 +178,7 @@ def fetchCalcUsedData?(rrdPath, start, endTime, interval,
myFunction, lIndex, re
(fstart, fend, names, data, interval) = RRD.fetch(rrdPath,
"--start", start.to_s, \
"--end", endTime.to_s, lFunc,
"-r", interval.to_s)
+ returnList.set_interval interval
i = 0
# For some reason, we get an extra datapoint at the end. Just chop it off
now...
data.delete_at(-1)
@@ -214,6 +216,7 @@ def fetchRegData?(rrdPath, start, endTime, interval,
myFunction, lIndex, returnL
(fstart, fend, names, data, interval) = RRD.fetch(rrdPath,
"--start", start.to_s, "--end", \
endTime.to_s, myFunction,
"-r", interval.to_s)
+ returnList.set_interval interval
i = 0
# For some reason, we get an extra datapoint at the end. Just chop it off
now...
data.delete_at(-1)
@@ -475,3 +478,87 @@ def getAggregateStatsData?(statRequestList)
return myList
end
+
+# This function also aggregates all of the values returned into one list before
+# returning. It is up to the caller to ensure that the request list has
+# "like" items. For instance if you request CPU Utilization and
Network bytes,
+# this function will be happy to aggregate them for you...
+# This function, however, also takes a start and end time, and will pad with
+# zero data points to fill any gaps in the returned data. It also returns
+# the data points with regular temporal spacing based on the oldest/coarsest
+# resolution available in the data.
+def getPaddedAggregateStatsData?(statRequestList, startTime, endTime)
+
+ fetchResults = []
+ myList = []
+ my_min = 0
+ my_max = 0
+ interval = 0
+
+ node = "Aggregate"
+ returnList = StatsDataList.new("Aggregate", 0, 0, 0, 0, 0, 0)
+ statRequestList.each do |request|
+ node = request.get_node?
+ counter = request.get_counter?
+
+ #Later time-ranged requests might have a finer resolution than earlier
+ #time-ranged requests. We assume that the ActiveRecord sql will order
+ #these by ascending startTime, and that the oldest rrd data will always
+ #have the coarsest resolution.
+ if request.get_precision? < interval
+ request.set_precision interval
+ end
+ tmpResult = fetchData?(request.get_node?, request.get_devClass?,
+ request.get_instance?, request.get_counter?,
+ request.get_starttime?, request.get_duration?,
+ request.get_precision?, request.get_function?)
+ fetchResults.push tmpResult
+
+ if interval == 0
+ interval = tmpResult.get_interval?
+ end
+ end
+
+ if interval != 0
+
+ sTime = (startTime.to_i / interval).to_i * interval
+ eTime = (endTime.to_i / interval).to_i * interval
+ pointCount = ((eTime - sTime) / interval) + 1
+
+ #ensure the results are sorted for the following loop
+ fetchResults.sort! {|x,y|
+ xTime = x.get_data?.empty? ? 0 : x.get_data?[0].get_timestamp?
+ yTime = y.get_data?.empty? ? 0 : y.get_data?[0].get_timestamp?
+ xTime <=> yTime
+ }
+
+ myCount = 0
+ while myCount < pointCount do
+ myTime = sTime + interval * (myCount + 1)
+ newDatum = StatsData.new(myTime,0)
+
+ fetchResults.each do |result|
+ if (! result.get_data?.empty?) &&
+ result.get_data?[0].get_timestamp? == myTime
+
+ datum = result.get_data?.shift
+ myValue = datum.get_value?.is_a?(Float) &&
datum.get_value?.nan? ? 0 : datum.get_value?
+ newDatum.set_value(newDatum.get_value? + myValue)
+ end
+ end
+ returnList.append_data(newDatum)
+
+ my_min = [my_min, newDatum.get_value?].min
+ my_max = [my_max, newDatum.get_value?].max
+ myCount += 1
+ end
+
+ returnList.set_min_value(my_min)
+ returnList.set_max_value(my_max)
+ returnList.set_interval(interval)
+ end
+
+ myList << returnList
+ return myList
+
+end
diff --git a/src/app/util/stats/StatsDataList.rb
b/src/app/util/stats/StatsDataList.rb
index 6c71400..4f7928f 100644
--- a/src/app/util/stats/StatsDataList.rb
+++ b/src/app/util/stats/StatsDataList.rb
@@ -20,7 +20,7 @@
#define class StatsData List
class StatsDataList
- def initialize(node, devClass, instance, counter, status, function,
resolution)
+ def initialize(node, devClass, instance, counter, status, function, interval)
# Instance variables
@node = node
@devClass = devClass
@@ -31,7 +31,7 @@ class StatsDataList
@function = function
@min_value = 0
@max_value = 0
- @resolution = 0
+ @interval = 0
end
def get_node?()
@@ -90,11 +90,11 @@ class StatsDataList
return @max_value
end
- def set_resolution(value)
- @resolution = value
+ def set_interval(value)
+ @interval = value
end
- def get_resolution?()
- return @resolution
+ def get_interval?()
+ return @interval
end
end
diff --git a/src/app/util/stats/StatsRequest.rb
b/src/app/util/stats/StatsRequest.rb
index dcd19ac..bd37628 100644
--- a/src/app/util/stats/StatsRequest.rb
+++ b/src/app/util/stats/StatsRequest.rb
@@ -36,7 +36,11 @@ class StatsRequest
@precision = precision
@function = function
end
-
+
+ def set_precision (newPrecision)
+ @precision = newPrecision
+ end
+
def get_node?()
return @node
end
--
1.6.0.6
Steve Linabery
2009-Mar-13 19:00 UTC
[Ovirt-devel] Re: [PATCH server] Add method to Stats.rb to return regularly-spaced, padded aggregate stats
On Fri, Mar 13, 2009 at 01:50:37PM -0500, Steve Linabery wrote:> Also an instance variable name change for StatsDataList for clarity's sake. > --- > src/app/controllers/graph_controller.rb | 4 +- > src/app/util/stats/Stats.rb | 89 ++++++++++++++++++++++++++++++- > src/app/util/stats/StatsDataList.rb | 12 ++-- > src/app/util/stats/StatsRequest.rb | 6 ++- > 4 files changed, 101 insertions(+), 10 deletions(-)Just wanted to mention that I am still testing this on a new server installation. No data points showing up yet. Thx Steve
Scott Seago
2009-Mar-13 20:41 UTC
[Ovirt-devel] [PATCH server] Add method to Stats.rb to return regularly-spaced, padded aggregate stats
Steve Linabery wrote:> Also an instance variable name change for StatsDataList for clarity's sake. > --- > src/app/controllers/graph_controller.rb | 4 +- > src/app/util/stats/Stats.rb | 89 ++++++++++++++++++++++++++++++- > src/app/util/stats/StatsDataList.rb | 12 ++-- > src/app/util/stats/StatsRequest.rb | 6 ++- > 4 files changed, 101 insertions(+), 10 deletions(-) > > diff --git a/src/app/controllers/graph_controller.rb b/src/app/controllers/graph_controller.rb > index 06f1c06..558787c 100644 > --- a/src/app/controllers/graph_controller.rb > +++ b/src/app/controllers/graph_controller.rb > @@ -55,7 +55,7 @@ class GraphController < ApplicationController > dataFunction) > } > } > - statsList = getAggregateStatsData?(requestList) > + statsList = getPaddedAggregateStatsData?(requestList,startTime,endTime) > > stat = statsList[0] > vectors = [ ] > @@ -68,7 +68,7 @@ class GraphController < ApplicationController > graph = { :vectors => vectors, > :max_value => stat.get_max_value?, > :description => target, > - :resolution => stat.get_resolution? > + :resolution => stat.get_interval? > } > render :json => graph > end > diff --git a/src/app/util/stats/Stats.rb b/src/app/util/stats/Stats.rb > index f7e9dbe..fee3b28 100644 > --- a/src/app/util/stats/Stats.rb > +++ b/src/app/util/stats/Stats.rb > @@ -24,7 +24,6 @@ require 'util/stats/StatsData' > require 'util/stats/StatsDataList' > require 'util/stats/StatsRequest' > > - > # This fetches a rolling average, basically average points before and after. > > > @@ -42,6 +41,7 @@ def fetchRollingAve?(rrdPath, start, endTime, interval, myFunction, lIndex, retu > > (fstart, fend, names, data, interval) = RRD.fetch(rrdPath, "--start", start.to_s, \ > "--end", endTime.to_s, myFunction, "-r", interval.to_s) > + returnList.set_interval interval > i = 0 > # For some reason, we get an extra datapoint at the end. Just chop it off now... > data.delete_at(-1) > @@ -110,6 +110,7 @@ def fetchRollingCalcUsedData?(rrdPath, start, endTime, interval, myFunction, lIn > > (fstart, fend, names, data, interval) = RRD.fetch(rrdPath, "--start", start.to_s, \ > "--end", endTime.to_s, lFunc, "-r", interval.to_s) > + returnList.set_interval interval > i = 0 > # For some reason, we get an extra datapoint at the end. Just chop it off now... > data.delete_at(-1) > @@ -177,6 +178,7 @@ def fetchCalcUsedData?(rrdPath, start, endTime, interval, myFunction, lIndex, re > > (fstart, fend, names, data, interval) = RRD.fetch(rrdPath, "--start", start.to_s, \ > "--end", endTime.to_s, lFunc, "-r", interval.to_s) > + returnList.set_interval interval > i = 0 > # For some reason, we get an extra datapoint at the end. Just chop it off now... > data.delete_at(-1) > @@ -214,6 +216,7 @@ def fetchRegData?(rrdPath, start, endTime, interval, myFunction, lIndex, returnL > > (fstart, fend, names, data, interval) = RRD.fetch(rrdPath, "--start", start.to_s, "--end", \ > endTime.to_s, myFunction, "-r", interval.to_s) > + returnList.set_interval interval > i = 0 > # For some reason, we get an extra datapoint at the end. Just chop it off now... > data.delete_at(-1) > @@ -475,3 +478,87 @@ def getAggregateStatsData?(statRequestList) > return myList > > end > + > +# This function also aggregates all of the values returned into one list before > +# returning. It is up to the caller to ensure that the request list has > +# "like" items. For instance if you request CPU Utilization and Network bytes, > +# this function will be happy to aggregate them for you... > +# This function, however, also takes a start and end time, and will pad with > +# zero data points to fill any gaps in the returned data. It also returns > +# the data points with regular temporal spacing based on the oldest/coarsest > +# resolution available in the data. > +def getPaddedAggregateStatsData?(statRequestList, startTime, endTime) > + > + fetchResults = [] > + myList = [] > + my_min = 0 > + my_max = 0 > + interval = 0 > + > + node = "Aggregate" > + returnList = StatsDataList.new("Aggregate", 0, 0, 0, 0, 0, 0) > + statRequestList.each do |request| > + node = request.get_node? > + counter = request.get_counter? > + > + #Later time-ranged requests might have a finer resolution than earlier > + #time-ranged requests. We assume that the ActiveRecord sql will order > + #these by ascending startTime, and that the oldest rrd data will always > + #have the coarsest resolution. > + if request.get_precision? < interval > + request.set_precision interval > + end > + tmpResult = fetchData?(request.get_node?, request.get_devClass?, > + request.get_instance?, request.get_counter?, > + request.get_starttime?, request.get_duration?, > + request.get_precision?, request.get_function?) > + fetchResults.push tmpResult > + > + if interval == 0 > + interval = tmpResult.get_interval? > + end > + end > + > + if interval != 0 > + > + sTime = (startTime.to_i / interval).to_i * interval > + eTime = (endTime.to_i / interval).to_i * interval > + pointCount = ((eTime - sTime) / interval) + 1 > + > + #ensure the results are sorted for the following loop > + fetchResults.sort! {|x,y| > + xTime = x.get_data?.empty? ? 0 : x.get_data?[0].get_timestamp? > + yTime = y.get_data?.empty? ? 0 : y.get_data?[0].get_timestamp? > + xTime <=> yTime > + } > + > + myCount = 0 > + while myCount < pointCount do > + myTime = sTime + interval * (myCount + 1) > + newDatum = StatsData.new(myTime,0) > + > + fetchResults.each do |result| > + if (! result.get_data?.empty?) && > + result.get_data?[0].get_timestamp? == myTime > + > + datum = result.get_data?.shift > + myValue = datum.get_value?.is_a?(Float) && datum.get_value?.nan? ? 0 : datum.get_value? > + newDatum.set_value(newDatum.get_value? + myValue) > + end > + end > + returnList.append_data(newDatum) > + > + my_min = [my_min, newDatum.get_value?].min > + my_max = [my_max, newDatum.get_value?].max > + myCount += 1 > + end > + > + returnList.set_min_value(my_min) > + returnList.set_max_value(my_max) > + returnList.set_interval(interval) > + end > + > + myList << returnList > + return myList > + > +end > diff --git a/src/app/util/stats/StatsDataList.rb b/src/app/util/stats/StatsDataList.rb > index 6c71400..4f7928f 100644 > --- a/src/app/util/stats/StatsDataList.rb > +++ b/src/app/util/stats/StatsDataList.rb > @@ -20,7 +20,7 @@ > > #define class StatsData List > class StatsDataList > - def initialize(node, devClass, instance, counter, status, function, resolution) > + def initialize(node, devClass, instance, counter, status, function, interval) > # Instance variables > @node = node > @devClass = devClass > @@ -31,7 +31,7 @@ class StatsDataList > @function = function > @min_value = 0 > @max_value = 0 > - @resolution = 0 > + @interval = 0 > end > > def get_node?() > @@ -90,11 +90,11 @@ class StatsDataList > return @max_value > end > > - def set_resolution(value) > - @resolution = value > + def set_interval(value) > + @interval = value > end > > - def get_resolution?() > - return @resolution > + def get_interval?() > + return @interval > end > end > diff --git a/src/app/util/stats/StatsRequest.rb b/src/app/util/stats/StatsRequest.rb > index dcd19ac..bd37628 100644 > --- a/src/app/util/stats/StatsRequest.rb > +++ b/src/app/util/stats/StatsRequest.rb > @@ -36,7 +36,11 @@ class StatsRequest > @precision = precision > @function = function > end > - > + > + def set_precision (newPrecision) > + @precision = newPrecision > + end > + > def get_node?() > return @node > end >OK this works for me. ACK Scott