Skip to content

Hiding of ColumnOptions #2959

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Mar 19, 2019
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
review comments
  • Loading branch information
artidoro committed Mar 19, 2019
commit 50b99e478460571a2a23b0ec0d0e2c455083d781
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@ public static void Example()
};

var model = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedWords", "Sentiment_Text")
.Append(mlContext.Transforms.Conversion.MapValue("VariableLenghtFeatures", lookupMap, "Words", "Ids", "TokenizedWords"))
.Append(mlContext.Transforms.Conversion.MapValue("VariableLenghtFeatures", lookupMap,
lookupMap.Schema["Words"], lookupMap.Schema["Ids"], "TokenizedWords"))
.Append(mlContext.Transforms.CustomMapping(ResizeFeaturesAction, "Resize"))
.Append(tensorFlowModel.ScoreTensorFlowModel(new[] { "Prediction/Softmax" }, new[] { "Features" }))
.Append(mlContext.Transforms.CopyColumns("Prediction", "Prediction/Softmax"))
Expand Down
12 changes: 5 additions & 7 deletions docs/samples/Microsoft.ML.Samples/Dynamic/ValueMapping.cs
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,13 @@ public static void Example()

// If the list of keys and values are known, they can be passed to the API. The ValueMappingEstimator can also get the mapping through an IDataView
// Creating a list of key-value pairs based on the Education values from the dataset.
var educationKeyValuePairs = new List<KeyValuePair<string, string>>()
{
new KeyValuePair<string, string>("0-5yrs", "Undergraduate"),
new KeyValuePair<string, string>("6-11yrs", "Postgraduate"),
new KeyValuePair<string, string>("12+yrs", "Postgraduate")
};
var educationMap = new Dictionary<string, string> ();
educationMap["0-5yrs"] = "Undergraduate";
educationMap["6-11yrs"] = "Postgraduate";
educationMap["12+yrs"] = "Postgraduate";

// Constructs the ValueMappingEstimator making the ML.net pipeline
var pipeline = mlContext.Transforms.Conversion.MapValue("EducationCategory", educationKeyValuePairs, "Education");
var pipeline = mlContext.Transforms.Conversion.MapValue("EducationCategory", educationMap, "Education");

// Fits the ValueMappingEstimator and transforms the data converting the Education to EducationCategory.
IDataView transformedData = pipeline.Fit(trainData).Transform(trainData);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,13 @@ public static void Example()

// If the list of keys and values are known, they can be passed to the API. The ValueMappingEstimator can also get the mapping through an IDataView
// Creating a list of key-value pairs based on the induced value from the dataset
var temperatureKeyValuePairs = new List<KeyValuePair<float, string>>()
{
new KeyValuePair<float, string>(36.0f, "T1"),
new KeyValuePair<float, string>(35.0f, "T2"),
new KeyValuePair<float, string>(34.0f, "T3")
};

var temperatureMap = new Dictionary<float, string>();
temperatureMap[36.0f] = "T1";
temperatureMap[35.0f] = "T2";
temperatureMap[34.0f] = "T3";

// Constructs the ValueMappingEstimator making the ML.net pipeline
var pipeline = mlContext.Transforms.Conversion.MapValue("TemperatureCategory", temperatureKeyValuePairs, "Temperature");
var pipeline = mlContext.Transforms.Conversion.MapValue("TemperatureCategory", temperatureMap, "Temperature");

// Fits the ValueMappingEstimator and transforms the data adding the TemperatureCategory column.
IDataView transformedData = pipeline.Fit(trainData).Transform(trainData);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,13 @@ public static void Example()

// If the list of keys and values are known, they can be passed to the API. The ValueMappingEstimator can also get the mapping through an IDataView
// Creating a list of key-value pairs based on the Education values from the dataset
var educationKeyValuePairs = new List<KeyValuePair<string, int[]>>()
{
new KeyValuePair<string, int[]>("0-5yrs", new int[] { 1,2,3 }),
new KeyValuePair<string, int[]>("6-11yrs", new int[] { 1,2,3 }),
new KeyValuePair<string, int[]>("12+yrs", new int[] { 1,2,3 })
};
var educationMap = new Dictionary<string, int[]>();
educationMap["0-5yrs"] = new int[] { 1, 2, 3 };
educationMap["6-11yrs"] = new int[] { 5, 6, 7 };
educationMap["12+yrs"] = new int[] { 42, 32, 64 };

// Constructs the ValueMappingEstimator making the ML.net pipeline
var pipeline = mlContext.Transforms.Conversion.MapValue<string, int>("EducationFeature", educationKeyValuePairs, "Education");
var pipeline = mlContext.Transforms.Conversion.MapValue<string, int>("EducationFeature", educationMap, "Education");

// Fits the ValueMappingEstimator and transforms the data adding the EducationFeature column.
IDataView transformedData = pipeline.Fit(trainData).Transform(trainData);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,15 @@ public static void Example()

// Creating a list of key-value pairs based on the Education values from the dataset
// These lists are created by hand for the demonstration, but the ValueMappingEstimator does take an IEnumerable.
var educationKeyValuePairs = new List<KeyValuePair<string,string>>()
{
new KeyValuePair<string,string>("0-5yrs", "Undergraduate"),
new KeyValuePair<string,string>("6-11yrs", "Postgraduate"),
new KeyValuePair<string,string>("12+yrs", "Postgraduate")
};
var educationMap = new Dictionary<string, string>();
educationMap["0-5yrs"] = "Undergraduate";
educationMap["6-11yrs"] = "Postgraduate";
educationMap["12+yrs"] = "Postgraduate";

// Generate the ValueMappingEstimator that will output KeyTypes even though our values are strings.
// The KeyToValueMappingEstimator is added to provide a reverse lookup of the KeyType, converting the KeyType value back
// to the original value.
var pipeline = mlContext.Transforms.Conversion.MapValue("EducationKeyType", educationKeyValuePairs, "Education", true)
var pipeline = mlContext.Transforms.Conversion.MapValue("EducationKeyType", educationMap, "Education", true)
.Append(mlContext.Transforms.Conversion.MapKeyToValue("EducationCategory", "EducationKeyType"));

// Fits the ValueMappingEstimator and transforms the data adding the EducationKeyType column.
Expand Down
16 changes: 8 additions & 8 deletions src/Microsoft.ML.Data/Transforms/ConversionsExtensionsCatalog.cs
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,7 @@ internal static ValueMappingEstimator<TInputType, TOutputType> MapValue<TInputTy
/// </summary>
/// <param name="catalog">The conversion transform's catalog</param>
/// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.</param>
/// <param name="lookupMap">An instance of <see cref="IDataView"/> that contains the key and value columns.</param>
/// <param name="lookupMap">An instance of <see cref="IDataView"/> that contains the <paramref name="keyColumn"/> and <paramref name="valueColumn"/> columns.</param>
/// <param name="keyColumn">The key column in <paramref name="lookupMap"/>.</param>
/// <param name="valueColumn">The value column in <paramref name="lookupMap"/>.</param>
/// <param name="inputColumnName">Name of the column to transform. If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.</param>
Expand All @@ -343,19 +343,19 @@ internal static ValueMappingEstimator<TInputType, TOutputType> MapValue<TInputTy
/// </example>
public static ValueMappingEstimator MapValue(
this TransformsCatalog.ConversionTransforms catalog,
string outputColumnName, IDataView lookupMap, string keyColumn, string valueColumn, string inputColumnName = null)
string outputColumnName, IDataView lookupMap, DataViewSchema.Column keyColumn, DataViewSchema.Column valueColumn, string inputColumnName = null)
{
return new ValueMappingEstimator(CatalogUtils.GetEnvironment(catalog), lookupMap, keyColumn, valueColumn,
return new ValueMappingEstimator(CatalogUtils.GetEnvironment(catalog), lookupMap, keyColumn.Name, valueColumn.Name,
new[] { (outputColumnName, inputColumnName ?? outputColumnName) });
}

/// <summary>
/// <see cref="ValueMappingEstimator"/>
/// </summary>
/// <param name="catalog">The conversion transform's catalog</param>
/// <param name="lookupMap">An instance of <see cref="IDataView"/> that contains the key and value columns.</param>
/// <param name="keyColumnName">Name of the key column in <paramref name="lookupMap"/>.</param>
/// <param name="valueColumnName">Name of the value column in <paramref name="lookupMap"/>.</param>
/// <param name="lookupMap">An instance of <see cref="IDataView"/> that contains the <paramref name="keyColumn"/> and <paramref name="valueColumn"/> columns.</param>
/// <param name="keyColumn">Name of the key column in <paramref name="lookupMap"/>.</param>
/// <param name="valueColumn">Name of the value column in <paramref name="lookupMap"/>.</param>
/// <param name="columns">The columns to apply this transform on.</param>
/// <returns>A instance of the ValueMappingEstimator</returns>
/// <example>
Expand All @@ -370,8 +370,8 @@ public static ValueMappingEstimator MapValue(
[BestFriend]
internal static ValueMappingEstimator MapValue(
this TransformsCatalog.ConversionTransforms catalog,
IDataView lookupMap, string keyColumnName, string valueColumnName, params ColumnOptions[] columns)
=> new ValueMappingEstimator(CatalogUtils.GetEnvironment(catalog), lookupMap, keyColumnName, valueColumnName,
IDataView lookupMap, DataViewSchema.Column keyColumn, DataViewSchema.Column valueColumn, params ColumnOptions[] columns)
=> new ValueMappingEstimator(CatalogUtils.GetEnvironment(catalog), lookupMap, keyColumn.Name, valueColumn.Name,
ColumnOptions.ConvertToValueTuples(columns));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -999,7 +999,8 @@ public void TensorFlowSentimentClassificationTest()
// Then this integer vector is retrieved from the pipeline and resized to fixed length.
// The second pipeline 'tfEnginePipe' takes the resized integer vector and passes it to TensoFlow and gets the classification scores.
var estimator = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedWords", "Sentiment_Text")
.Append(mlContext.Transforms.Conversion.MapValue(lookupMap, "Words", "Ids", new ColumnOptions[] { ("Features", "TokenizedWords") }));
.Append(mlContext.Transforms.Conversion.MapValue(lookupMap, lookupMap.Schema["Words"], lookupMap.Schema["Ids"],
new ColumnOptions[] { ("Features", "TokenizedWords") }));
var model = estimator.Fit(dataView);
var dataPipe = mlContext.Model.CreatePredictionEngine<TensorFlowSentiment, TensorFlowSentiment>(model);

Expand Down