Skip to content

Commit 4b43d60

Browse files
DamiaoCastroDamiaoCastro
committed
support for most types,
cope with sub-entities, unit and component tests passing
1 parent 8c37af1 commit 4b43d60

File tree

7 files changed

+202
-95
lines changed

7 files changed

+202
-95
lines changed

‎AzureFunctions.Extensions.GoogleBigQuery.ComponentTests/BigQueryServiceTests.cs‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public void Initialize() {
1818
}
1919

2020
[TestMethod]
21-
public async Task InsertRowsAsync_EmptyTestBigQueryRow() {
21+
public async Task InsertRowsAsync_EmptyRow() {
2222

2323
//Arrange
2424
TestBigQueryRow testBigQueryRow = new TestBigQueryRow(DateTime.UtcNow, "insertId_1") {
@@ -30,6 +30,7 @@ public async Task InsertRowsAsync_EmptyTestBigQueryRow() {
3030
BoolEnumerable = new bool[] { },
3131
Byte1Enumerable = new byte[] { },
3232
Byte2Enumerable = new byte[] { },
33+
Byte1Array = new byte[] { },
3334
CharEnumerable = new char[] { },
3435
DateTimeEnumerable = new DateTime[] { },
3536
DateTimeOffsetEnumerable = new DateTimeOffset[] { },
@@ -45,6 +46,7 @@ public async Task InsertRowsAsync_EmptyTestBigQueryRow() {
4546
UInt32Enumerable = new UInt32[] { },
4647
UInt64Enumerable = new UInt64[] { },
4748

49+
4850
Record2 = new List<SimpleEntity1> { },
4951
Record3 = new SimpleEntity1[] { }
5052
};
@@ -56,5 +58,53 @@ public async Task InsertRowsAsync_EmptyTestBigQueryRow() {
5658

5759
}
5860

61+
[TestMethod]
62+
public async Task InsertRowsAsync_WithSomeRecords() {
63+
64+
//Arrange
65+
TestBigQueryRow testBigQueryRow = new TestBigQueryRow(DateTime.UtcNow, "insertId_2") {
66+
Double1Enumerable = new double[] { },
67+
Double2Enumerable = new double[] { },
68+
String1Enumerable = new string[] { },
69+
String2Enumerable = new string[] { },
70+
BooleanEnumerable = new bool[] { },
71+
BoolEnumerable = new bool[] { },
72+
Byte1Enumerable = new byte[] { },
73+
Byte2Enumerable = new byte[] { },
74+
Byte1Array = new byte[] { },
75+
CharEnumerable = new char[] { },
76+
DateTimeEnumerable = new DateTime[] { },
77+
DateTimeOffsetEnumerable = new DateTimeOffset[] { },
78+
DecimalEnumerable = new decimal[] { },
79+
FloatEnumerable = new float[] { },
80+
GuidEnumerable = new Guid[] { },
81+
Int16Enumerable = new Int16[] { },
82+
Int32Enumerable = new Int32[] { },
83+
Int64Enumerable = new Int64[] { },
84+
IntEnumerable = new int[] { },
85+
SingleEnumerable = new Single[] { },
86+
UInt16Enumerable = new UInt16[] { },
87+
UInt32Enumerable = new UInt32[] { },
88+
UInt64Enumerable = new UInt64[] { },
89+
90+
Record1 = new SimpleEntity1() { MySubProperty1 = 1, MySubProperty2 = "Record1.MySubProperty2" },
91+
Record2 = new List<SimpleEntity1> {
92+
new SimpleEntity1() { MySubProperty1 = 1, MySubProperty2 = "Record2.MySubProperty2-1" } ,
93+
new SimpleEntity1() { MySubProperty1 = 2, MySubProperty2 = "Record2.MySubProperty2-2" },
94+
new SimpleEntity1() { MySubProperty1 = 3, MySubProperty2 = "Record2.MySubProperty2-3" }
95+
},
96+
Record3 = new SimpleEntity1[] {
97+
new SimpleEntity1() { MySubProperty1 = 1, MySubProperty2 = "Record3.MySubProperty3-1" } ,
98+
new SimpleEntity1() { MySubProperty1 = 2, MySubProperty2 = "Record3.MySubProperty3-2" }
99+
}
100+
};
101+
102+
//Act
103+
await bigQueryService.InsertRowsAsync(testBigQueryRow.Date, new TestBigQueryRow[] { testBigQueryRow }, CancellationToken.None);
104+
105+
//Assert
106+
107+
}
108+
59109
}
60110
}

‎AzureFunctions.Extensions.GoogleBigQuery.UnitTests/TableSchemaBuilderServiceTests.cs‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,31 @@ public void GetTableSchema_TestBigQueryRow() {
2828
Assert.AreEqual("REPEATED", tabelSchema.Fields[4].Mode);
2929
Assert.AreEqual("BOOLEAN", tabelSchema.Fields[4].Type);
3030
Assert.IsNull(tabelSchema.Fields[4].Fields);
31-
31+
32+
//...
33+
34+
Assert.IsNull(tabelSchema.Fields[8].Description);
35+
Assert.AreEqual("Byte1Nullable", tabelSchema.Fields[8].Name);
36+
Assert.AreEqual("NULLABLE", tabelSchema.Fields[8].Mode);
37+
Assert.AreEqual("INTEGER", tabelSchema.Fields[8].Type);
38+
Assert.IsNull(tabelSchema.Fields[8].Fields);
39+
40+
//...
41+
42+
Assert.IsNull(tabelSchema.Fields[10].Description);
43+
Assert.AreEqual("Byte1Enumerable", tabelSchema.Fields[10].Name);
44+
Assert.AreEqual("NULLABLE", tabelSchema.Fields[10].Mode);
45+
Assert.AreEqual("BYTES", tabelSchema.Fields[10].Type);
46+
Assert.IsNull(tabelSchema.Fields[10].Fields);
47+
48+
//...
49+
50+
Assert.IsNull(tabelSchema.Fields[15].Description);
51+
Assert.AreEqual("CharEnumerable", tabelSchema.Fields[15].Name);
52+
Assert.AreEqual("NULLABLE", tabelSchema.Fields[15].Mode);
53+
Assert.AreEqual("STRING", tabelSchema.Fields[15].Type);
54+
Assert.IsNull(tabelSchema.Fields[15].Fields);
55+
3256
//...
3357

3458
Assert.IsNull(tabelSchema.Fields[65].Description);

‎AzureFunctions.Extensions.GoogleBigQuery.sln‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureFunctions.Extensions.G
99
EndProject
1010
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureFunctions.Extensions.GoogleBigQuery.DemoProject1", "AzureFunctions.Extensions.GoogleBigQuery.DemoProject1\AzureFunctions.Extensions.GoogleBigQuery.DemoProject1.csproj", "{B117DF3E-C655-420C-B1F1-A30854B76334}"
1111
EndProject
12-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureFunctions.Extensions.GoogleBigQuery.TestsCommon", "AzureFunctions.Extensions.GoogleBigQuery.TestsCommon\AzureFunctions.Extensions.GoogleBigQuery.TestsCommon.csproj", "{DC9B4723-4D19-4DEE-B0E2-49025FEC252E}"
12+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureFunctions.Extensions.GoogleBigQuery.TestsCommon", "AzureFunctions.Extensions.GoogleBigQuery.TestsCommon\AzureFunctions.Extensions.GoogleBigQuery.TestsCommon.csproj", "{DC9B4723-4D19-4DEE-B0E2-49025FEC252E}"
1313
EndProject
14-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureFunctions.Extensions.GoogleBigQuery.UnitTests", "AzureFunctions.Extensions.GoogleBigQuery.UnitTests\AzureFunctions.Extensions.GoogleBigQuery.UnitTests.csproj", "{452CBDD6-B6DF-4C80-A76E-8501D98456C8}"
14+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureFunctions.Extensions.GoogleBigQuery.UnitTests", "AzureFunctions.Extensions.GoogleBigQuery.UnitTests\AzureFunctions.Extensions.GoogleBigQuery.UnitTests.csproj", "{452CBDD6-B6DF-4C80-A76E-8501D98456C8}"
1515
EndProject
1616
Global
1717
GlobalSection(SolutionConfigurationPlatforms) = preSolution

‎AzureFunctions.Extensions.GoogleBigQuery/AzureFunctions.Extensions.GoogleBigQuery.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ If this package was done by Microsoft itself would be under the namespace "Micro
1818
</ItemGroup>
1919

2020
<ItemGroup>
21+
<PackageReference Include="Google.Cloud.BigQuery.V2" Version="1.0.1" />
2122
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions" Version="3.0.0-beta3" />
22-
<PackageReference Include="Google.Cloud.BigQuery.V2" Version="1.0.1" />
2323
</ItemGroup>
2424

2525
</Project>

‎AzureFunctions.Extensions.GoogleBigQuery/BigQueryInsertRowService.cs‎

Lines changed: 100 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -30,86 +30,118 @@ private static IDictionary<string, object> GetDictionaryOfValues(IDictionary<str
3030

3131
private static object GetBigQueryValue(IDictionary<string, IEnumerable<PropertyInfo>> dictionaryOfProperties, PropertyInfo property, object obj) {
3232
switch (property.PropertyType.Name.ToUpper()) {
33+
case "IENUMERABLE`1":
34+
return GetArrayFromEnumreable(dictionaryOfProperties, property, obj);
35+
case "NULLABLE`1":
36+
var value = property.GetValue(obj);
37+
if (value == null) {
38+
return null;
39+
} else {
40+
var propertyTypeName = property.PropertyType.GenericTypeArguments[0].Name;
41+
return GetNonEnumerableBigQueryValue(propertyTypeName, value);
42+
}
43+
}
44+
45+
if (property.PropertyType.IsClass && property.PropertyType.Namespace != "System") {//crappy but works for now
46+
if (property.PropertyType.IsArray) {
47+
var array = (IEnumerable<object>)property.GetValue(obj);
48+
return GetSubEntitiesBigQueryInsertRows(dictionaryOfProperties, array);
49+
} else {
50+
var value = property.GetValue(obj);
51+
if (value == null) {
52+
return null;
53+
} else {
54+
return GetSubEntitiesBigQueryInsertRows(dictionaryOfProperties, new List<object> { value }).First();
55+
}
56+
}
57+
}
58+
59+
return GetNonEnumerableBigQueryValue(property.PropertyType.Name, property.GetValue(obj));
60+
}
61+
62+
private static object GetNonEnumerableBigQueryValue(string propertyTypeName, object value) {
63+
switch (propertyTypeName.ToUpper()) {
3364
case "BYTE":
34-
return (int)(byte)property.GetValue(obj);
65+
return (int)(byte)value;
3566
case "CHAR":
36-
return ((char)property.GetValue(obj)).ToString();
67+
return ((char)value).ToString();
3768
case "CHAR[]":
38-
return ((char[])property.GetValue(obj)).ToString();
69+
return ((char[])value).ToString();
3970
case "DATETIME":
40-
var datetimeValue = (DateTime)property.GetValue(obj);
41-
return datetimeValue.ToString(BigQueryDateTimeFormat, cultureUS);
71+
return ((DateTime)value).ToString(BigQueryDateTimeFormat, cultureUS);
72+
case "DATETIMEOFFSET":
73+
return ((DateTimeOffset)value).ToString(BigQueryDateTimeFormat, cultureUS);
4274
case "DECIMAL":
43-
return (float)(decimal)property.GetValue(obj);
75+
return (float)(decimal)value;
4476
case "GUID":
45-
return ((Guid)property.GetValue(obj)).ToString();
77+
return ((Guid)value).ToString();
4678
case "UINT64":
47-
return (int)(UInt64)property.GetValue(obj);
48-
case "IENUMERABLE`1":
49-
var enumerableValue = property.GetValue(obj);
50-
51-
Type innerPropertyType = property.PropertyType.GenericTypeArguments[0];
52-
switch (innerPropertyType.Name.ToUpper()) {
53-
case "BYTE":
54-
return enumerableValue;
55-
case "BOOLEAN":
56-
return ((bool[])enumerableValue);
57-
case "CHAR":
58-
return ((char[])enumerableValue).ToString();
59-
case "DATETIME":
60-
return (DateTime[])enumerableValue;
61-
case "DATETIMEOFFSET":
62-
return (DateTimeOffset[])enumerableValue;
63-
case "DOUBLE":
64-
return ((double[])enumerableValue).Select(c => (float)c).ToArray();
65-
case "DECIMAL":
66-
return ((decimal[])enumerableValue).Select(c => (float)c).ToArray();
67-
case "SINGLE":
68-
return ((float[])enumerableValue);
69-
case "GUID":
70-
return ((Guid[])enumerableValue).Select(c => c.ToString()).ToArray();
71-
case "UINT16":
72-
return ((UInt16[])enumerableValue);
73-
case "INT16":
74-
return ((Int16[])enumerableValue);
75-
case "INT":
76-
case "INT32":
77-
return ((int[])enumerableValue);
78-
case "UINT32":
79-
return ((UInt32[])enumerableValue);
80-
case "INT64":
81-
return ((Int64[])enumerableValue);
82-
case "UINT64":
83-
return ((UInt64[])enumerableValue);
84-
default:
85-
if (property.PropertyType.IsArray) {
86-
return enumerableValue;
87-
}
88-
89-
IEnumerable<object> ie = (IEnumerable<object>)enumerableValue;
90-
if (innerPropertyType.IsClass && innerPropertyType.Namespace != "System") {
91-
return (IEnumerable<Dictionary<string, object>>) ie.Select(c => GetDictionaryOfValues(dictionaryOfProperties, c)).ToArray();
92-
} else {
93-
var length = ie.Count();
94-
var i = Array.CreateInstance(innerPropertyType, length);
95-
Array.Copy(ie.ToArray(), i, length);
96-
return i;
97-
}
98-
}
79+
return (int)(UInt64)value;
80+
default:
81+
return value;
82+
}
83+
}
9984

85+
private static object GetArrayFromEnumreable(IDictionary<string, IEnumerable<PropertyInfo>> dictionaryOfProperties, PropertyInfo property, object obj) {
86+
var enumerableValue = property.GetValue(obj);
87+
88+
Type innerPropertyType = property.PropertyType.GenericTypeArguments[0];
89+
switch (innerPropertyType.Name.ToUpper()) {
90+
case "BYTE":
91+
return (byte[])enumerableValue;
92+
case "BOOLEAN":
93+
return ((bool[])enumerableValue);
94+
case "CHAR":
95+
return ((char[])enumerableValue).ToString();
96+
case "DATETIME":
97+
return (DateTime[])enumerableValue;
98+
case "DATETIMEOFFSET":
99+
return (DateTimeOffset[])enumerableValue;
100+
case "DOUBLE":
101+
return ((double[])enumerableValue).Select(c => (float)c).ToArray();
102+
case "DECIMAL":
103+
return ((decimal[])enumerableValue).Select(c => (float)c).ToArray();
104+
case "SINGLE":
105+
return ((float[])enumerableValue);
106+
case "GUID":
107+
return ((Guid[])enumerableValue).Select(c => c.ToString()).ToArray();
108+
case "UINT16":
109+
return ((UInt16[])enumerableValue);
110+
case "INT16":
111+
return ((Int16[])enumerableValue);
112+
case "INT":
113+
case "INT32":
114+
return ((int[])enumerableValue);
115+
case "UINT32":
116+
return ((UInt32[])enumerableValue);
117+
case "INT64":
118+
return ((Int64[])enumerableValue);
119+
case "UINT64":
120+
return ((UInt64[])enumerableValue);
100121
default:
101-
if (property.PropertyType.IsClass && property.PropertyType.Namespace != "System") {//crappy but works for now
102-
if (property.PropertyType.IsArray) {
103-
var array = (IEnumerable<object>)property.GetValue(obj);
104-
//return array.Select(c => GetDictionaryOfValues(dictionaryOfProperties, c)).ToArray();
105-
return null;
106-
} else {
107-
return GetDictionaryOfValues(dictionaryOfProperties, property.GetValue(obj));
108-
}
122+
if (property.PropertyType.IsArray) {
123+
return enumerableValue;
109124
}
110125

111-
return property.GetValue(obj);
126+
IEnumerable<object> ie = (IEnumerable<object>)enumerableValue;
127+
if (innerPropertyType.IsClass && innerPropertyType.Namespace != "System") {
128+
return GetSubEntitiesBigQueryInsertRows(dictionaryOfProperties, ie);
129+
} else {
130+
var length = ie.Count();
131+
var i = Array.CreateInstance(innerPropertyType, length);
132+
Array.Copy(ie.ToArray(), i, length);
133+
return i;
134+
}
135+
}
136+
}
137+
138+
private static BigQueryInsertRow[] GetSubEntitiesBigQueryInsertRows(IDictionary<string, IEnumerable<PropertyInfo>> dictionaryOfProperties, IEnumerable<object> objs) {
139+
140+
if (objs.Count() > 0) {
141+
return objs.Select(c => new BigQueryInsertRow() { GetDictionaryOfValues(dictionaryOfProperties, c) }).ToArray();
112142
}
143+
144+
return new BigQueryInsertRow[] { };
113145
}
114146

115147
}

‎AzureFunctions.Extensions.GoogleBigQuery/BigQueryService.cs‎

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
using Google.Apis.Auth.OAuth2;
66
using Google.Apis.Bigquery.v2.Data;
77
using System.Threading.Tasks;
8-
using System.Reflection;
98
using System.Threading;
109

1110
namespace AzureFunctions.Extensions.GoogleBigQuery {
@@ -33,19 +32,14 @@ private Task<BigQueryTable> GetTable(DateTime date, CancellationToken cancellati
3332
if (credentials != null) {
3433
googleCredential = GoogleCredential.FromStream(new System.IO.MemoryStream(credentials));
3534
}
36-
var client = Google.Cloud.BigQuery.V2.BigQueryClient.Create(projectId, googleCredential);
37-
38-
return client.GetOrCreateTableAsync(
39-
datasetId,
40-
$"{tableId}${date.ToString("yyyyMMdd")}",
41-
//tableId,
42-
tableSchema,
43-
new GetTableOptions(),
44-
new CreateTableOptions() {
45-
FriendlyName = $"{tableId}${date.ToString("yyyyMMdd")}",
46-
TimePartitioning = new TimePartitioning() { Type = "DAY"/*, Field = "_PARTITIONTIME"*/ }
47-
},
48-
cancellationToken);
35+
var client = BigQueryClient.Create(projectId, googleCredential);
36+
37+
//return client.GetOrCreateTableAsync(datasetId, tableId, tableSchema, null, new CreateTableOptions() { TimePartitioning = new TimePartitioning() { Type = "DAY" } }, cancellationToken)
38+
// .ContinueWith((createTableTask) => {
39+
// return client.GetTableAsync(datasetId, $"{tableId}${date:yyyyMMdd}", null, cancellationToken);
40+
// }, cancellationToken).Unwrap();
41+
return client.GetTableAsync(datasetId, $"{tableId}${date:yyyyMMdd}", null, cancellationToken);
42+
4943
}
5044

5145
public Task InsertRowsAsync(DateTime date, IEnumerable<GoogleBigQueryRow> rows, CancellationToken cancellationToken) {
@@ -56,15 +50,15 @@ public Task InsertRowsAsync(DateTime date, IEnumerable<GoogleBigQueryRow> rows,
5650
if (dateDiff >= -31 && dateDiff <= 16) {
5751

5852
var bigQueryRows = rows.Select(c => BigQueryInsertRowService.GetBigQueryInsertRow(c, dictionaryOfProperties));
59-
53+
6054
return GetTable(date, cancellationToken)
6155
.ContinueWith((tableTask) => {
6256
BigQueryTable table = tableTask.Result;
63-
57+
6458
return table.InsertRowsAsync(bigQueryRows, new InsertOptions() { AllowUnknownFields = true }, cancellationToken)
6559
.ContinueWith((insertRowsTask) => {
6660
if (insertRowsTask.IsFaulted) {
67-
//... log trace
61+
throw insertRowsTask.Exception.InnerExceptions.First();
6862
}
6963
});
7064
}, cancellationToken).Unwrap();
@@ -74,6 +68,6 @@ public Task InsertRowsAsync(DateTime date, IEnumerable<GoogleBigQueryRow> rows,
7468

7569
return Task.WhenAll();
7670
}
77-
71+
7872
}
7973
}

0 commit comments

Comments
 (0)