Skip to content

Update release for 0.9 #2067

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 26 commits into from
Jan 8, 2019
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
25f3287
Move Static API extensions to separate assembly (#1930)
abgoswam Jan 3, 2019
1ebb023
Bump version to 0.10 (#2005)
shauheen Jan 3, 2019
21d64aa
Make CompositeSchema not an ISchema (#2001)
wschin Jan 3, 2019
85db282
making GetCoefficientStatistics public (#1979)
sfilipi Jan 3, 2019
1d864c4
Update tests (using Iris dataset) to use new API (#2008)
abgoswam Jan 3, 2019
0d903ab
iteration setup removed (#2011)
Anipik Jan 4, 2019
9a6711b
Onnxtransform - api changes for GPU support (#1922)
jignparm Jan 4, 2019
5c00980
Fixing typo in the FeatureContributionCalculation sample. (#2012)
zeahmed Jan 4, 2019
16bfd54
Updating tests in Microsoft.ML.Tests/Scenarios to new API (#2024)
abgoswam Jan 4, 2019
0cfba68
Convert CSharpAPI based test to EntryPoint based test. (#2020)
codemzs Jan 4, 2019
1319e2c
Remove Legacy dependency in ML.FSharp.Tests (#2021)
artidoro Jan 4, 2019
312f9e4
Remove ColumnType "Is" properties (except IsKey and IsVector) (#2023)
eerhardt Jan 4, 2019
7647e52
fix build break (#2030)
abgoswam Jan 5, 2019
76aab70
Added more documentation and extra CPU path test (#2029)
jignparm Jan 5, 2019
b7f5ba7
Remove collectionsdatasource tests. (#2032)
codemzs Jan 5, 2019
2c91cf3
Remove Legacy dependency from Microsoft.ML.Benchmarks (#2027)
najeeb-kazmi Jan 5, 2019
7caac81
Remove legacy API test that already have coverage and refactor code. …
codemzs Jan 5, 2019
496e185
Remove legacy dependency from TextLoaderTests.cs (#2033)
artidoro Jan 5, 2019
faffd17
New ONNX converter interface and some tests (#2013)
wschin Jan 6, 2019
e2f7075
Remove legacy project and related stuff. (#2043)
codemzs Jan 7, 2019
d9d4b22
Expose internal APIs to components that use them in an internal repo …
yaeldMS Jan 7, 2019
cdd92c2
Merge branch 'master' into release/v09rc2
Jan 8, 2019
54bceac
Fix version for RC2 v 0.9
Jan 8, 2019
4cafb8f
Improve documentation for Feature Contribution Calculation (#2007)
artidoro Jan 7, 2019
7e28d21
Add release notes for ML.NET 0.9 (#2061)
shauheen Jan 8, 2019
30dd0c2
New ML.NET statically typed pipeline Nuget. (#2076)
TomFinley Jan 8, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove ColumnType "Is" properties (except IsKey and IsVector) (#2023)
* Remove ColumnType.IsBool

* Remove ColumnType.IsPrimitive

* Remove ColumnType.IsNumber

* Remove ColumnType.IsText

* Move ColumnType.IsStandardScalar to an extension method.
  • Loading branch information
eerhardt authored Jan 4, 2019
commit 312f9e4c71953bee701ea2a63be3cb4d2d276d20
61 changes: 1 addition & 60 deletions src/Microsoft.ML.Core/Data/ColumnType.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@ public abstract class ColumnType : IEquatable<ColumnType>
// This private constructor sets all the IsXxx flags. It is invoked by other ctors.
private ColumnType()
{
IsPrimitive = this is PrimitiveType;
IsVector = this is VectorType;
IsNumber = this is NumberType;
IsKey = this is KeyType;
}

Expand Down Expand Up @@ -73,58 +71,6 @@ private protected ColumnType(Type rawType, DataKind rawKind)
[BestFriend]
internal DataKind RawKind { get; }

/// <summary>
/// Whether this is a primitive type. External code should use <c>is <see cref="PrimitiveType"/></c>.
/// </summary>
[BestFriend]
internal bool IsPrimitive { get; }

/// <summary>
/// Whether this type is a standard numeric type. External code should use <c>is <see cref="NumberType"/></c>.
/// </summary>
[BestFriend]
internal bool IsNumber { get; }

/// <summary>
/// Whether this type is the standard text type. External code should use <c>is <see cref="TextType"/></c>.
/// </summary>
[BestFriend]
internal bool IsText
{
get
{
if (!(this is TextType))
return false;
// TextType is a singleton.
Contracts.Assert(this == TextType.Instance);
return true;
}
}

/// <summary>
/// Whether this type is the standard boolean type. External code should use <c>is <see cref="BoolType"/></c>.
/// </summary>
[BestFriend]
internal bool IsBool
{
get
{
if (!(this is BoolType))
return false;
// BoolType is a singleton.
Contracts.Assert(this == BoolType.Instance);
return true;
}
}

/// <summary>
/// Whether this type is a standard scalar type completely determined by its <see cref="RawType"/>
/// (not a <see cref="KeyType"/> or <see cref="StructuredType"/>, etc).
/// </summary>
[BestFriend]
internal bool IsStandardScalar => IsNumber || IsText || IsBool ||
(this is TimeSpanType) || (this is DateTimeType) || (this is DateTimeOffsetType);

/// <summary>
/// Whether this type is a key type, which implies that the order of values is not significant,
/// and arithmetic is non-sensical. A key type can define a cardinality.
Expand Down Expand Up @@ -230,13 +176,11 @@ public abstract class StructuredType : ColumnType
protected StructuredType(Type rawType)
: base(rawType)
{
Contracts.Assert(!IsPrimitive);
}

private protected StructuredType(Type rawType, DataKind rawKind)
: base(rawType, rawKind)
{
Contracts.Assert(!IsPrimitive);
}
}

Expand All @@ -249,15 +193,13 @@ public abstract class PrimitiveType : ColumnType
protected PrimitiveType(Type rawType)
: base(rawType)
{
Contracts.Assert(IsPrimitive);
Contracts.CheckParam(!typeof(IDisposable).IsAssignableFrom(RawType), nameof(rawType),
"A " + nameof(PrimitiveType) + " cannot have a disposable " + nameof(RawType));
}

private protected PrimitiveType(Type rawType, DataKind rawKind)
: base(rawType, rawKind)
{
Contracts.Assert(IsPrimitive);
Contracts.Assert(!typeof(IDisposable).IsAssignableFrom(RawType));
}

Expand Down Expand Up @@ -322,7 +264,6 @@ private NumberType(DataKind kind, string name)
{
Contracts.AssertNonEmpty(name);
_name = name;
Contracts.Assert(IsNumber);
}

private static volatile NumberType _instI1;
Expand Down Expand Up @@ -496,7 +437,7 @@ public override bool Equals(ColumnType other)
{
if (other == this)
return true;
Contracts.Assert(other == null || !other.IsNumber || other.RawKind != RawKind);
Contracts.Assert(other == null || !(other is NumberType) || other.RawKind != RawKind);
return false;
}

Expand Down
21 changes: 21 additions & 0 deletions src/Microsoft.ML.Core/Data/ColumnTypeExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

namespace Microsoft.ML.Data
{
/// <summary>
/// Extension methods related to the ColumnType class.
/// </summary>
[BestFriend]
internal static class ColumnTypeExtensions
{
/// <summary>
/// Whether this type is a standard scalar type completely determined by its <see cref="ColumnType.RawType"/>
/// (not a <see cref="KeyType"/> or <see cref="StructuredType"/>, etc).
/// </summary>
public static bool IsStandardScalar(this ColumnType columnType) =>
(columnType is NumberType) || (columnType is TextType) || (columnType is BoolType) ||
(columnType is TimeSpanType) || (columnType is DateTimeType) || (columnType is DateTimeOffsetType);
}
}
10 changes: 5 additions & 5 deletions src/Microsoft.ML.Core/Data/MetadataUtils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ internal static IEnumerable<int> GetColumnSet(this Schema schema, string metadat
for (int col = 0; col < schema.Count; col++)
{
var columnType = schema[col].Metadata.Schema.GetColumnOrNull(metadataKind)?.Type;
if (columnType != null && columnType.IsText)
if (columnType != null && columnType is TextType)
{
ReadOnlyMemory<char> val = default;
schema[col].Metadata.GetValue(metadataKind, ref val);
Expand Down Expand Up @@ -318,7 +318,7 @@ internal static bool HasSlotNames(this Schema.Column column, int vectorSize)
metaColumn != null
&& metaColumn.Value.Type.IsVector
&& metaColumn.Value.Type.VectorSize == vectorSize
&& metaColumn.Value.Type.ItemType.IsText;
&& metaColumn.Value.Type.ItemType is TextType;
}

public static void GetSlotNames(this Schema.Column column, ref VBuffer<ReadOnlyMemory<char>> slotNames)
Expand Down Expand Up @@ -348,15 +348,15 @@ internal static bool HasKeyValues(this Schema.Column column, int keyCount)
metaColumn != null
&& metaColumn.Value.Type.IsVector
&& metaColumn.Value.Type.VectorSize == keyCount
&& metaColumn.Value.Type.ItemType.IsText;
&& metaColumn.Value.Type.ItemType is TextType;
}

[BestFriend]
internal static bool HasKeyValues(this SchemaShape.Column col)
{
return col.Metadata.TryFindColumn(Kinds.KeyValues, out var metaCol)
&& metaCol.Kind == SchemaShape.Column.VectorKind.Vector
&& metaCol.ItemType.IsText;
&& metaCol.ItemType is TextType;
}

/// <summary>
Expand All @@ -365,7 +365,7 @@ internal static bool HasKeyValues(this SchemaShape.Column col)
public static bool IsNormalized(this Schema.Column column)
{
var metaColumn = column.Metadata.Schema.GetColumnOrNull((Kinds.IsNormalized));
if (metaColumn == null || !metaColumn.Value.Type.IsBool)
if (metaColumn == null || !(metaColumn.Value.Type is BoolType))
return false;

bool value = default;
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.ML.Data/Commands/DataCommand.cs
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ protected void SendTelemetryMetric(Dictionary<string, IDataView>[] metricValues)
{
var nameOfMetric = "TLC_" + cursor.Schema[currentIndex].Name;
var type = cursor.Schema[currentIndex].Type;
if (type.IsNumber)
if (type is NumberType)
{
var getter = RowCursorUtils.GetGetterAs<double>(NumberType.R8, cursor, currentIndex);
double metricValue = 0;
Expand Down
6 changes: 3 additions & 3 deletions src/Microsoft.ML.Data/Commands/ShowSchemaCommand.cs
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ private static void PrintSchema(TextWriter writer, Arguments args, Schema schema
ColumnType typeNames;
if ((typeNames = schema[col].Metadata.Schema.GetColumnOrNull(MetadataUtils.Kinds.SlotNames)?.Type) == null)
continue;
if (typeNames.VectorSize != type.VectorSize || !typeNames.ItemType.IsText)
if (typeNames.VectorSize != type.VectorSize || !(typeNames.ItemType is TextType))
{
Contracts.Assert(false, "Unexpected slot names type");
continue;
Expand Down Expand Up @@ -212,7 +212,7 @@ private static void ShowMetadataValue(IndentedTextWriter itw, Schema schema, int
Contracts.AssertValue(type);
Contracts.Assert(!type.IsVector);

if (!type.IsStandardScalar && !type.IsKey)
if (!type.IsStandardScalar() && !type.IsKey)
{
itw.Write(": Can't display value of this type");
return;
Expand Down Expand Up @@ -252,7 +252,7 @@ private static void ShowMetadataValueVec(IndentedTextWriter itw, Schema schema,
Contracts.AssertValue(type);
Contracts.Assert(type.IsVector);

if (!type.ItemType.IsStandardScalar && !type.ItemType.IsKey)
if (!type.ItemType.IsStandardScalar() && !type.ItemType.IsKey)
{
itw.Write(": Can't display value of this type");
return;
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.ML.Data/Commands/TypeInfoCommand.cs
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ private TypeNaInfo KindReport<T>(IChannel ch, PrimitiveType type)
{
Contracts.AssertValue(ch);
ch.AssertValue(type);
ch.Assert(type.IsStandardScalar);
ch.Assert(type.IsStandardScalar());

var conv = Conversions.Instance;
InPredicate<T> isNaDel;
Expand Down
12 changes: 6 additions & 6 deletions src/Microsoft.ML.Data/Data/Conversion.cs
Original file line number Diff line number Diff line change
Expand Up @@ -452,12 +452,12 @@ public bool TryGetStandardConversion(ColumnType typeSrc, ColumnType typeDst,
}
else if (typeDst is KeyType keyDst)
{
if (!typeSrc.IsText)
if (!(typeSrc is TextType))
return false;
conv = GetKeyParse(keyDst);
return true;
}
else if (!typeDst.IsStandardScalar)
else if (!typeDst.IsStandardScalar())
return false;

Contracts.Assert(typeSrc.RawKind != 0);
Expand Down Expand Up @@ -567,7 +567,7 @@ public ValueMapper<TSrc, SB> GetKeyStringConversion<TSrc>(KeyType key)
public TryParseMapper<TDst> GetTryParseConversion<TDst>(ColumnType typeDst)
{
Contracts.CheckValue(typeDst, nameof(typeDst));
Contracts.CheckParam(typeDst.IsStandardScalar || typeDst.IsKey, nameof(typeDst),
Contracts.CheckParam(typeDst.IsStandardScalar() || typeDst.IsKey, nameof(typeDst),
"Parse conversion only supported for standard types");
Contracts.Check(typeDst.RawType == typeof(TDst), "Wrong TDst type parameter");

Expand Down Expand Up @@ -676,7 +676,7 @@ public InPredicate<T> GetIsDefaultPredicate<T>(ColumnType type)

var t = type;
Delegate del;
if (!t.IsStandardScalar && !t.IsKey || !_isDefaultDelegates.TryGetValue(t.RawKind, out del))
if (!t.IsStandardScalar() && !t.IsKey || !_isDefaultDelegates.TryGetValue(t.RawKind, out del))
throw Contracts.Except("No IsDefault predicate for '{0}'", type);

return (InPredicate<T>)del;
Expand Down Expand Up @@ -719,7 +719,7 @@ public bool TryGetIsNAPredicate(ColumnType type, out Delegate del)
Contracts.Assert(_isDefaultDelegates.ContainsKey(t.RawKind));
del = _isDefaultDelegates[t.RawKind];
}
else if (!t.IsStandardScalar || !_isNADelegates.TryGetValue(t.RawKind, out del))
else if (!t.IsStandardScalar() || !_isNADelegates.TryGetValue(t.RawKind, out del))
{
del = null;
return false;
Expand All @@ -742,7 +742,7 @@ public InPredicate<VBuffer<T>> GetHasMissingPredicate<T>(VectorType type)
Contracts.Assert(_hasZeroDelegates.ContainsKey(t.RawKind));
del = _hasZeroDelegates[t.RawKind];
}
else if (!t.IsStandardScalar || !_hasNADelegates.TryGetValue(t.RawKind, out del))
else if (!t.IsStandardScalar() || !_hasNADelegates.TryGetValue(t.RawKind, out del))
throw Contracts.Except("No HasMissing predicate for '{0}'", type);

return (InPredicate<VBuffer<T>>)del;
Expand Down
4 changes: 2 additions & 2 deletions src/Microsoft.ML.Data/Data/DataViewUtils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ public static bool AllCachable(Schema schema, Func<int, bool> predicate)
/// </summary>
public static bool IsCachable(this ColumnType type)
{
return type != null && (type.IsPrimitive || type.IsVector);
return type != null && (type is PrimitiveType || type.IsVector);
}

/// <summary>
Expand Down Expand Up @@ -859,7 +859,7 @@ public static OutPipe Create(ColumnType type, object pool)
pipeType = typeof(ImplVec<>).MakeGenericType(type.ItemType.RawType);
else
{
Contracts.Assert(type.IsPrimitive);
Contracts.Assert(type is PrimitiveType);
pipeType = typeof(ImplOne<>).MakeGenericType(type.RawType);
}
var constructor = pipeType.GetConstructor(new Type[] { typeof(object) });
Expand Down
16 changes: 8 additions & 8 deletions src/Microsoft.ML.Data/Data/RowCursorUtils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,13 @@ private static Delegate GetGetterAsDelegateCore<TValue>(Row row, int col)
public static Delegate GetGetterAs(ColumnType typeDst, Row row, int col)
{
Contracts.CheckValue(typeDst, nameof(typeDst));
Contracts.CheckParam(typeDst.IsPrimitive, nameof(typeDst));
Contracts.CheckParam(typeDst is PrimitiveType, nameof(typeDst));
Contracts.CheckValue(row, nameof(row));
Contracts.CheckParam(0 <= col && col < row.Schema.Count, nameof(col));
Contracts.CheckParam(row.IsColumnActive(col), nameof(col), "column was not active");

var typeSrc = row.Schema[col].Type;
Contracts.Check(typeSrc.IsPrimitive, "Source column type must be primitive");
Contracts.Check(typeSrc is PrimitiveType, "Source column type must be primitive");

Func<ColumnType, ColumnType, Row, int, ValueGetter<int>> del = GetGetterAsCore<int, int>;
var methodInfo = del.GetMethodInfo().GetGenericMethodDefinition().MakeGenericMethod(typeSrc.RawType, typeDst.RawType);
Expand All @@ -66,14 +66,14 @@ public static Delegate GetGetterAs(ColumnType typeDst, Row row, int col)
public static ValueGetter<TDst> GetGetterAs<TDst>(ColumnType typeDst, Row row, int col)
{
Contracts.CheckValue(typeDst, nameof(typeDst));
Contracts.CheckParam(typeDst.IsPrimitive, nameof(typeDst));
Contracts.CheckParam(typeDst is PrimitiveType, nameof(typeDst));
Contracts.CheckParam(typeDst.RawType == typeof(TDst), nameof(typeDst));
Contracts.CheckValue(row, nameof(row));
Contracts.CheckParam(0 <= col && col < row.Schema.Count, nameof(col));
Contracts.CheckParam(row.IsColumnActive(col), nameof(col), "column was not active");

var typeSrc = row.Schema[col].Type;
Contracts.Check(typeSrc.IsPrimitive, "Source column type must be primitive");
Contracts.Check(typeSrc is PrimitiveType, "Source column type must be primitive");

Func<ColumnType, ColumnType, Row, int, ValueGetter<TDst>> del = GetGetterAsCore<int, TDst>;
var methodInfo = del.GetMethodInfo().GetGenericMethodDefinition().MakeGenericMethod(typeSrc.RawType, typeof(TDst));
Expand Down Expand Up @@ -118,7 +118,7 @@ public static ValueGetter<StringBuilder> GetGetterAsStringBuilder(Row row, int c
Contracts.CheckParam(row.IsColumnActive(col), nameof(col), "column was not active");

var typeSrc = row.Schema[col].Type;
Contracts.Check(typeSrc.IsPrimitive, "Source column type must be primitive");
Contracts.Check(typeSrc is PrimitiveType, "Source column type must be primitive");
return Utils.MarshalInvoke(GetGetterAsStringBuilderCore<int>, typeSrc.RawType, typeSrc, row, col);
}

Expand Down Expand Up @@ -356,7 +356,7 @@ public static string TestGetLabelGetter(ColumnType type)

public static string TestGetLabelGetter(ColumnType type, bool allowKeys)
{
if (type == NumberType.R4 || type == NumberType.R8 || type.IsBool)
if (type == NumberType.R4 || type == NumberType.R8 || type is BoolType)
return null;

if (allowKeys && type.IsKey)
Expand Down Expand Up @@ -394,7 +394,7 @@ private static ValueGetter<Single> GetLabelGetterNotFloat(Row cursor, int labelI
Contracts.Assert(type != NumberType.R4 && type != NumberType.R8);

// boolean type label mapping: True -> 1, False -> 0.
if (type.IsBool)
if (type is BoolType)
{
var getBoolSrc = cursor.GetGetter<bool>(labelIndex);
return
Expand Down Expand Up @@ -429,7 +429,7 @@ public static ValueGetter<VBuffer<Single>> GetLabelGetter(SlotCursor cursor)
var type = cursor.GetSlotType().ItemType;
if (type == NumberType.R4)
return cursor.GetGetter<Single>();
if (type == NumberType.R8 || type.IsBool)
if (type == NumberType.R8 || type is BoolType)
return GetVecGetterAs<Single>(NumberType.R4, cursor);
Contracts.Check(type.IsKey, "Only floating point number, boolean, and key type values can be used as label.");
Contracts.Assert(TestGetLabelGetter(type) == null);
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.ML.Data/DataLoadSave/DataOperations.cs
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public IDataView FilterByColumn(IDataView input, string columnName, double lower
Environment.CheckParam(lowerBound <= upperBound, nameof(upperBound), "Must be no less than lowerBound");

var type = input.Schema[columnName].Type;
if (!type.IsNumber)
if (!(type is NumberType))
throw Environment.ExceptSchemaMismatch(nameof(columnName), "filter", columnName, "number", type.ToString());
return new RangeFilter(Environment, input, columnName, lowerBound, upperBound, false);
}
Expand Down
4 changes: 2 additions & 2 deletions src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ private Func<RowSet, ColumnPipe> GetCreatorOneCore(PrimitiveType type)

private Func<RowSet, ColumnPipe> GetCreatorOneCore<T>(PrimitiveType type)
{
Contracts.Assert(type.IsStandardScalar || type.IsKey);
Contracts.Assert(type.IsStandardScalar() || type.IsKey);
Contracts.Assert(typeof(T) == type.RawType);
var fn = _conv.GetTryParseConversion<T>(type);
return rows => new PrimitivePipe<T>(rows, type, fn);
Expand All @@ -84,7 +84,7 @@ private Func<RowSet, ColumnPipe> GetCreatorVecCore(PrimitiveType type)

private Func<RowSet, ColumnPipe> GetCreatorVecCore<T>(PrimitiveType type)
{
Contracts.Assert(type.IsStandardScalar || type.IsKey);
Contracts.Assert(type.IsStandardScalar() || type.IsKey);
Contracts.Assert(typeof(T) == type.RawType);
var fn = _conv.GetTryParseConversion<T>(type);
return rows => new VectorPipe<T>(rows, type, fn);
Expand Down
Loading