-
Notifications
You must be signed in to change notification settings - Fork 1.9k
add root cause localization transformer #4925
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
d5ee205
f727a79
92de1dc
798289c
f2e128d
51569e3
59c6e89
29216e0
69da330
e1c5432
3a1d1c5
8f97602
48123f4
c47302f
b07ad28
c729877
ebbdb0d
0d43b0d
5778eed
c9ed044
e440f25
feba6f4
686831c
ddc8a36
475ee8a
8d874ca
0674ab3
4c5b8fb
08d607c
c688233
98637db
4ff2ed1
c22ad50
ea7ddbe
547aef2
8d17c3c
66b614a
12e7e18
e213615
30915cd
fda4ec7
620ef58
ae5722f
7f89fea
42dcbc2
9893fad
c831e43
16f5b33
9cd8739
f80c200
7c1c348
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5,6 +5,7 @@ | |
using System; | ||
using System.Collections.Generic; | ||
using System.Linq; | ||
using System.Runtime.CompilerServices; | ||
using Microsoft.ML.Internal.Utilities; | ||
|
||
namespace Microsoft.ML.TimeSeries | ||
|
@@ -29,7 +30,7 @@ public RootCause Analyze() | |
return AnalyzeOneLayer(_src); | ||
} | ||
|
||
//This is a function for analyze one layer for root cause, we select one dimension with values who contributes the most to the anomaly. For full result, call this function recursively | ||
//This is a function for analyze one layer for root cause, we select one dimension with values who contributes the most to the anomaly. | ||
private RootCause AnalyzeOneLayer(RootCauseLocalizationInput src) | ||
{ | ||
suxi-ms marked this conversation as resolved.
Show resolved
Hide resolved
|
||
RootCause dst = new RootCause(); | ||
|
@@ -42,11 +43,7 @@ private RootCause AnalyzeOneLayer(RootCauseLocalizationInput src) | |
Dictionary<string, Point> dimPointMapping = pointInfo.Item3; | ||
|
||
//which means there is no anomaly point with the anomaly dimension or no point under anomaly dimension | ||
if (anomalyTree.ParentNode == null) | ||
{ | ||
return dst; | ||
} | ||
if (dimPointMapping.Count == 0) | ||
if (anomalyTree.ParentNode == null || dimPointMapping.Count == 0) | ||
{ | ||
return dst; | ||
} | ||
|
@@ -59,16 +56,8 @@ private RootCause AnalyzeOneLayer(RootCauseLocalizationInput src) | |
|
||
protected List<Point> GetTotalPointsForAnomalyTimestamp(RootCauseLocalizationInput src) | ||
{ | ||
List<Point> points = new List<Point>(); | ||
foreach (MetricSlice slice in src.Slices) | ||
{ | ||
if (slice.TimeStamp.Equals(src.AnomalyTimestamp)) | ||
{ | ||
points = slice.Points; | ||
} | ||
} | ||
|
||
return points; | ||
MetricSlice slice = src.Slices.Single(slice => slice.TimeStamp.Equals(src.AnomalyTimestamp)); | ||
return slice.Points; | ||
} | ||
|
||
protected DimensionInfo SeperateDimension(Dictionary<string, Object> dimensions, Object aggSymbol) | ||
|
@@ -92,8 +81,8 @@ protected DimensionInfo SeperateDimension(Dictionary<string, Object> dimensions, | |
|
||
protected Tuple<PointTree, PointTree, Dictionary<string, Point>> GetPointsInfo(RootCauseLocalizationInput src, DimensionInfo dimensionInfo) | ||
{ | ||
PointTree pointTree = PointTree.CreateDefaultInstance(); | ||
PointTree anomalyTree = PointTree.CreateDefaultInstance(); | ||
PointTree pointTree = new PointTree(); | ||
PointTree anomalyTree = new PointTree(); | ||
Dictionary<string, Point> dimPointMapping = new Dictionary<string, Point>(); | ||
|
||
List<Point> totalPoints = GetTotalPointsForAnomalyTimestamp(src); | ||
|
@@ -125,12 +114,8 @@ protected Tuple<PointTree, PointTree, Dictionary<string, Point>> GetPointsInfo(R | |
|
||
protected Dictionary<string, Object> GetSubDim(Dictionary<string, Object> dimension, List<string> keyList) | ||
{ | ||
Dictionary<string, Object> subDim = new Dictionary<string, Object>(); | ||
foreach (string dim in keyList) | ||
{ | ||
subDim.Add(dim, dimension[dim]); | ||
} | ||
return subDim; | ||
return new Dictionary<string, object>(keyList.Select(dim => new KeyValuePair<string, object>(dim, dimension[dim])).ToDictionary(kvp => kvp.Key, kvp => kvp.Value)); | ||
|
||
suxi-ms marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
|
||
protected List<RootCauseItem> LocalizeRootCauseByDimension(PointTree anomalyTree, PointTree pointTree, Dictionary<string, Object> anomalyDimension, List<string> aggDims) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. suggestion: Please consider something along the lines of the following for the GetSubDim function.
#Resolved There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
updated #Resolved |
||
|
@@ -203,7 +188,8 @@ protected List<Point> GetTopAnomaly(List<Point> anomalyPoints, Point root, List< | |
{ | ||
anomalyPoints.Reverse(); | ||
} | ||
else { | ||
else | ||
{ | ||
anomalyPoints = anomalyPoints.FindAll(x => x.Delta < 0); | ||
} | ||
if (anomalyPoints.Count == 1) | ||
|
@@ -245,21 +231,23 @@ protected BestDimension SelectBestDimension(List<Point> totalPoints, List<Point> | |
|
||
foreach (string dimKey in aggDim) | ||
{ | ||
BestDimension dimension = BestDimension.CreateDefaultInstance(); | ||
BestDimension dimension = new BestDimension(); | ||
dimension.DimensionKey = dimKey; | ||
|
||
UpdateDistribution(dimension.PointDis, totalPoints, dimKey); | ||
UpdateDistribution(dimension.AnomalyDis, anomalyPoints, dimKey); | ||
|
||
double relativeEntropy = GetDimensionEntropy(dimension.PointDis, dimension.AnomalyDis); | ||
double gain = totalEntropy - relativeEntropy; | ||
if (Double.IsNaN(gain)) { | ||
if (Double.IsNaN(gain)) | ||
{ | ||
gain = 0; | ||
} | ||
entroyGainMap.Add(dimension, gain); | ||
|
||
double gainRatio = gain / GetDimensionInstrinsicValue(dimension.PointDis); | ||
if (Double.IsInfinity(gainRatio)) { | ||
if (Double.IsInfinity(gainRatio)) | ||
{ | ||
gainRatio = 0; | ||
} | ||
entroyGainRatioMap.Add(dimension, gainRatio); | ||
|
@@ -282,7 +270,7 @@ private BestDimension SelectBestDimension(Dictionary<string, List<Point>> pointC | |
|
||
foreach (string dimKey in aggDim) | ||
{ | ||
BestDimension dimension = BestDimension.CreateDefaultInstance(); | ||
BestDimension dimension = new BestDimension(); | ||
dimension.DimensionKey = dimKey; | ||
|
||
if (pointChildren.ContainsKey(dimKey)) | ||
|
@@ -295,14 +283,16 @@ private BestDimension SelectBestDimension(Dictionary<string, List<Point>> pointC | |
} | ||
|
||
double entropy = GetEntropy(dimension.PointDis.Count, dimension.AnomalyDis.Count); | ||
if (Double.IsNaN(entropy)) { | ||
if (Double.IsNaN(entropy)) | ||
{ | ||
entropy = Double.MaxValue; | ||
} | ||
entropyMap.Add(dimension, entropy); | ||
|
||
double gainRatio = entropy / GetDimensionInstrinsicValue(dimension.PointDis); | ||
|
||
if (Double.IsInfinity(gainRatio)) { | ||
if (Double.IsInfinity(gainRatio)) | ||
{ | ||
gainRatio = 0; | ||
} | ||
entropyRatioMap.Add(dimension, gainRatio); | ||
|
@@ -326,7 +316,8 @@ private AnomalyDirection GetRootCauseDirection(Point rootCausePoint) | |
{ | ||
return AnomalyDirection.Down; | ||
} | ||
else { | ||
else | ||
{ | ||
return AnomalyDirection.Same; | ||
} | ||
} | ||
|
@@ -357,7 +348,8 @@ private void GetRootCauseDirectionAndScore(Dictionary<string, Point> dimPointMap | |
{ | ||
dst.Items[i].Score = 1; | ||
} | ||
else { | ||
else | ||
{ | ||
dst.Items[i].Score = GetFinalScore(scoreList[i].Surprise, Math.Abs(scoreList[i].ExplainaryScore), beta); | ||
suxi-ms marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
} | ||
|
@@ -372,7 +364,8 @@ private void GetRootCauseDirectionAndScore(Dictionary<string, Point> dimPointMap | |
{ | ||
dst.Items[0].Score = 1; | ||
} | ||
else { | ||
else | ||
{ | ||
dst.Items[0].Score = GetFinalScore(scores.Item1, scores.Item2, beta); | ||
} | ||
dst.Items[0].Direction = GetRootCauseDirection(rootCausePoint); | ||
|
@@ -469,25 +462,33 @@ private BestDimension FindBestDimension(SortedDictionary<BestDimension, double> | |
{ | ||
if (dimension.Key.AnomalyDis.Count == 1 || (isLeavesLevel ? dimension.Value >= meanGain : dimension.Value <= meanGain)) | ||
{ | ||
if (dimension.Key.AnomalyDis.Count > 1) | ||
if (best == null) | ||
{ | ||
if (best == null || (!Double.IsNaN(valueRatioMap[best]) && (best.AnomalyDis.Count != 1 && (isLeavesLevel ? valueRatioMap[best].CompareTo(dimension.Value) <= 0 : valueRatioMap[best].CompareTo(dimension.Value) >= 0)))) | ||
{ | ||
best = dimension.Key; | ||
} | ||
best = dimension.Key; | ||
} | ||
else | ||
{ | ||
if (best == null || best.AnomalyDis.Count > 1) | ||
bool isRatioNan = Double.IsNaN(valueRatioMap[best]); | ||
if (dimension.Key.AnomalyDis.Count > 1) | ||
{ | ||
best = dimension.Key; | ||
if (!isRatioNan && (best.AnomalyDis.Count != 1 && (isLeavesLevel ? valueRatioMap[best].CompareTo(dimension.Value) <= 0 : valueRatioMap[best].CompareTo(dimension.Value) >= 0))) | ||
{ | ||
best = dimension.Key; | ||
} | ||
} | ||
else | ||
{ | ||
if (!Double.IsNaN(valueRatioMap[best]) && (isLeavesLevel ? valueRatioMap[best].CompareTo(dimension.Value) <= 0 : valueRatioMap[best].CompareTo(dimension.Value) >= 0)) | ||
if (best.AnomalyDis.Count > 1) | ||
{ | ||
best = dimension.Key; | ||
} | ||
else | ||
{ | ||
if (!isRatioNan && (isLeavesLevel ? valueRatioMap[best].CompareTo(dimension.Value) <= 0 : valueRatioMap[best].CompareTo(dimension.Value) >= 0)) | ||
{ | ||
best = dimension.Key; | ||
} | ||
} | ||
} | ||
} | ||
} | ||
|
@@ -660,26 +661,19 @@ private void UpdateDistribution(Dictionary<string, int> distribution, List<Point | |
} | ||
} | ||
|
||
private double Log2(double val) | ||
{ | ||
if (Double.IsNaN(val)) | ||
{ | ||
return 0; | ||
} | ||
[MethodImplAttribute(MethodImplOptions.AggressiveInlining)] | ||
private double Log2(double val) => Double.IsNaN(val) ? 0 : Math.Log(val) / Math.Log(2); | ||
|
||
return Math.Log(val) / Math.Log(2); | ||
} | ||
|
||
private static bool ContainsAll(Dictionary<string, Object> bigDic, Dictionary<string, Object> smallDic) | ||
private static bool ContainsAll(Dictionary<string, Object> bigDictionary, Dictionary<string, Object> smallDictionary) | ||
{ | ||
suxi-ms marked this conversation as resolved.
Show resolved
Hide resolved
|
||
foreach (var item in smallDic) | ||
foreach (var item in smallDictionary) | ||
{ | ||
if (!bigDic.ContainsKey(item.Key)) | ||
if (!bigDictionary.ContainsKey(item.Key)) | ||
{ | ||
return false; | ||
} | ||
|
||
if (bigDic.ContainsKey(item.Key) && !bigDic[item.Key].Equals(smallDic[item.Key])) | ||
if (bigDictionary.ContainsKey(item.Key) && !bigDictionary[item.Key].Equals(smallDictionary[item.Key])) | ||
{ | ||
suxi-ms marked this conversation as resolved.
Show resolved
Hide resolved
|
||
return false; | ||
} | ||
|
@@ -714,12 +708,10 @@ public class PointTree | |
public Dictionary<string, List<Point>> ChildrenNodes; | ||
public List<Point> Leaves; | ||
|
||
public static PointTree CreateDefaultInstance() | ||
public PointTree() | ||
{ | ||
suxi-ms marked this conversation as resolved.
Show resolved
Hide resolved
|
||
PointTree instance = new PointTree(); | ||
instance.Leaves = new List<Point>(); | ||
instance.ChildrenNodes = new Dictionary<string, List<Point>>(); | ||
return instance; | ||
Leaves = new List<Point>(); | ||
ChildrenNodes = new Dictionary<string, List<Point>>(); | ||
} | ||
} | ||
|
||
|
@@ -729,13 +721,10 @@ public sealed class BestDimension : IComparable | |
public Dictionary<string, int> AnomalyDis; | ||
public Dictionary<string, int> PointDis; | ||
|
||
public BestDimension() { } | ||
public static BestDimension CreateDefaultInstance() | ||
public BestDimension() | ||
{ | ||
BestDimension instance = new BestDimension(); | ||
instance.AnomalyDis = new Dictionary<string, int>(); | ||
instance.PointDis = new Dictionary<string, int>(); | ||
return instance; | ||
AnomalyDis = new Dictionary<string, int>(); | ||
PointDis = new Dictionary<string, int>(); | ||
} | ||
|
||
public int CompareTo(object obj) | ||
|
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You don't need this if check. The CheckUserArg is performing the check. #Resolved
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
will update #Resolved