Skip to content

Commit 8c37af1

Browse files
DamiaoCastroDamiaoCastro
committed
problem with entities still to resolve
1 parent c369784 commit 8c37af1

File tree

5 files changed

+165
-114
lines changed

5 files changed

+165
-114
lines changed

‎AzureFunctions.Extensions.GoogleBigQuery.ComponentTests/BigQueryServiceTests.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using Microsoft.Azure.WebJobs.Host.Config;
33
using Microsoft.VisualStudio.TestTools.UnitTesting;
44
using System;
5+
using System.Collections.Generic;
56
using System.Threading;
67
using System.Threading.Tasks;
78

@@ -31,7 +32,7 @@ public async Task InsertRowsAsync_EmptyTestBigQueryRow() {
3132
Byte2Enumerable = new byte[] { },
3233
CharEnumerable = new char[] { },
3334
DateTimeEnumerable = new DateTime[] { },
34-
DateTimeOffsetEnumerable =new DateTimeOffset[] { },
35+
DateTimeOffsetEnumerable = new DateTimeOffset[] { },
3536
DecimalEnumerable = new decimal[] { },
3637
FloatEnumerable = new float[] { },
3738
GuidEnumerable = new Guid[] { },
@@ -43,6 +44,9 @@ public async Task InsertRowsAsync_EmptyTestBigQueryRow() {
4344
UInt16Enumerable = new UInt16[] { },
4445
UInt32Enumerable = new UInt32[] { },
4546
UInt64Enumerable = new UInt64[] { },
47+
48+
Record2 = new List<SimpleEntity1> { },
49+
Record3 = new SimpleEntity1[] { }
4650
};
4751

4852
//Act

‎AzureFunctions.Extensions.GoogleBigQuery/BigQueryInsertRowService.cs‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
using Google.Cloud.BigQuery.V2;
2+
using System;
3+
using System.Collections.Generic;
4+
using System.Linq;
5+
using System.Reflection;
6+
7+
namespace AzureFunctions.Extensions.GoogleBigQuery {
8+
public class BigQueryInsertRowService {
9+
10+
private const string BigQueryDateTimeFormat = "yyyy-MM-dd HH:mm:ss";
11+
private static System.Globalization.CultureInfo cultureUS = new System.Globalization.CultureInfo("en-US");
12+
13+
public static BigQueryInsertRow GetBigQueryInsertRow(GoogleBigQueryRow row, IDictionary<string, IEnumerable<PropertyInfo>> dictionaryOfProperties) {
14+
if (row == null) { throw new System.ArgumentNullException(nameof(row)); }
15+
if (dictionaryOfProperties == null) { throw new System.ArgumentNullException(nameof(dictionaryOfProperties)); }
16+
17+
IDictionary<string, object> dic = GetDictionaryOfValues(dictionaryOfProperties, row);
18+
19+
return new BigQueryInsertRow(row.InsertId) { dic };
20+
}
21+
22+
private static IDictionary<string, object> GetDictionaryOfValues(IDictionary<string, IEnumerable<PropertyInfo>> dictionaryOfProperties, object obj) {
23+
if (obj == null) { return null; }
24+
25+
var properties = dictionaryOfProperties[obj.GetType().FullName];
26+
IDictionary<string, object> dictionaryOfValues = properties.ToDictionary(c => c.Name, c => GetBigQueryValue(dictionaryOfProperties, c, obj));
27+
28+
return dictionaryOfValues;
29+
}
30+
31+
private static object GetBigQueryValue(IDictionary<string, IEnumerable<PropertyInfo>> dictionaryOfProperties, PropertyInfo property, object obj) {
32+
switch (property.PropertyType.Name.ToUpper()) {
33+
case "BYTE":
34+
return (int)(byte)property.GetValue(obj);
35+
case "CHAR":
36+
return ((char)property.GetValue(obj)).ToString();
37+
case "CHAR[]":
38+
return ((char[])property.GetValue(obj)).ToString();
39+
case "DATETIME":
40+
var datetimeValue = (DateTime)property.GetValue(obj);
41+
return datetimeValue.ToString(BigQueryDateTimeFormat, cultureUS);
42+
case "DECIMAL":
43+
return (float)(decimal)property.GetValue(obj);
44+
case "GUID":
45+
return ((Guid)property.GetValue(obj)).ToString();
46+
case "UINT64":
47+
return (int)(UInt64)property.GetValue(obj);
48+
case "IENUMERABLE`1":
49+
var enumerableValue = property.GetValue(obj);
50+
51+
Type innerPropertyType = property.PropertyType.GenericTypeArguments[0];
52+
switch (innerPropertyType.Name.ToUpper()) {
53+
case "BYTE":
54+
return enumerableValue;
55+
case "BOOLEAN":
56+
return ((bool[])enumerableValue);
57+
case "CHAR":
58+
return ((char[])enumerableValue).ToString();
59+
case "DATETIME":
60+
return (DateTime[])enumerableValue;
61+
case "DATETIMEOFFSET":
62+
return (DateTimeOffset[])enumerableValue;
63+
case "DOUBLE":
64+
return ((double[])enumerableValue).Select(c => (float)c).ToArray();
65+
case "DECIMAL":
66+
return ((decimal[])enumerableValue).Select(c => (float)c).ToArray();
67+
case "SINGLE":
68+
return ((float[])enumerableValue);
69+
case "GUID":
70+
return ((Guid[])enumerableValue).Select(c => c.ToString()).ToArray();
71+
case "UINT16":
72+
return ((UInt16[])enumerableValue);
73+
case "INT16":
74+
return ((Int16[])enumerableValue);
75+
case "INT":
76+
case "INT32":
77+
return ((int[])enumerableValue);
78+
case "UINT32":
79+
return ((UInt32[])enumerableValue);
80+
case "INT64":
81+
return ((Int64[])enumerableValue);
82+
case "UINT64":
83+
return ((UInt64[])enumerableValue);
84+
default:
85+
if (property.PropertyType.IsArray) {
86+
return enumerableValue;
87+
}
88+
89+
IEnumerable<object> ie = (IEnumerable<object>)enumerableValue;
90+
if (innerPropertyType.IsClass && innerPropertyType.Namespace != "System") {
91+
return (IEnumerable<Dictionary<string, object>>) ie.Select(c => GetDictionaryOfValues(dictionaryOfProperties, c)).ToArray();
92+
} else {
93+
var length = ie.Count();
94+
var i = Array.CreateInstance(innerPropertyType, length);
95+
Array.Copy(ie.ToArray(), i, length);
96+
return i;
97+
}
98+
}
99+
100+
default:
101+
if (property.PropertyType.IsClass && property.PropertyType.Namespace != "System") {//crappy but works for now
102+
if (property.PropertyType.IsArray) {
103+
var array = (IEnumerable<object>)property.GetValue(obj);
104+
//return array.Select(c => GetDictionaryOfValues(dictionaryOfProperties, c)).ToArray();
105+
return null;
106+
} else {
107+
return GetDictionaryOfValues(dictionaryOfProperties, property.GetValue(obj));
108+
}
109+
}
110+
111+
return property.GetValue(obj);
112+
}
113+
}
114+
115+
}
116+
}

‎AzureFunctions.Extensions.GoogleBigQuery/BigQueryService.cs‎

Lines changed: 14 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -12,80 +12,29 @@ namespace AzureFunctions.Extensions.GoogleBigQuery {
1212

1313
public class BigQueryService {
1414

15-
private const string BigQueryDateTimeFormat = "yyyy-MM-dd HH:mm:ss";
16-
private System.Globalization.CultureInfo cultureUS = new System.Globalization.CultureInfo("en-US");
17-
1815
private readonly byte[] credentials;
1916
private readonly string projectId;
2017
private readonly string datasetId;
2118
private readonly string tableId;
2219
private readonly TableSchema tableSchema;
23-
private readonly IEnumerable<PropertyInfo> properties;
20+
private readonly IDictionary<string, IEnumerable<System.Reflection.PropertyInfo>> dictionaryOfProperties;
2421

2522
public BigQueryService(byte[] credentials, string projectId, string datasetId, string tableId, Type itemType) {
2623
this.credentials = credentials;
2724
this.projectId = projectId;
2825
this.datasetId = datasetId;
2926
this.tableId = tableId;
30-
(this.tableSchema, this.properties) = TableSchemaBuilderService.GetTableSchema(itemType);
27+
(this.tableSchema, this.dictionaryOfProperties) = TableSchemaBuilderService.GetTableSchema(itemType);
3128
}
3229

33-
//private (TableSchema, IEnumerable<System.Reflection.PropertyInfo>) GetTableSchema(Type tableType) {
34-
35-
// var properties = tableType.GetProperties()
36-
// .Where(c => c.PropertyType.IsPublic && c.CustomAttributes.Any(a => a.AttributeType == typeof(ColumnAttribute)));
37-
38-
// var fields = from property in properties
39-
// let type = GetBigQueryType(property.PropertyType)
40-
// let mode = property.CustomAttributes.Any(a => a.AttributeType == typeof(RequiredAttribute)) ? "REQUIRED" : "NULLABLE"
41-
// select new TableFieldSchema() { Name = property.Name, Type = type, Mode = mode }
42-
// ;
43-
44-
// var schema = new Google.Apis.Bigquery.v2.Data.TableSchema() { Fields = fields.ToList() };
45-
46-
// return (schema, properties);
47-
//}
48-
49-
//private string GetBigQueryType(Type propertyType) {
50-
51-
// //STRING
52-
// //BYTES
53-
// //INTEGER
54-
// //FLOAT
55-
// //BOOLEAN
56-
// //TIMESTAMP
57-
// //DATE
58-
// //TIME
59-
// //DATETIME
60-
// //RECORD
61-
62-
// if (propertyType.Name.Equals("Nullable`1")) {
63-
// propertyType = propertyType.GenericTypeArguments[0];
64-
// }
65-
66-
// switch (propertyType.Name.ToUpper()) {
67-
// case "INT":
68-
// case "INT32":
69-
// case "INT64":
70-
// return "INTEGER";
71-
// default:
72-
// return propertyType.Name.ToUpper();
73-
// }
74-
//}
75-
7630
private Task<BigQueryTable> GetTable(DateTime date, CancellationToken cancellationToken) {
7731

7832
GoogleCredential googleCredential = null;
7933
if (credentials != null) {
8034
googleCredential = GoogleCredential.FromStream(new System.IO.MemoryStream(credentials));
8135
}
8236
var client = Google.Cloud.BigQuery.V2.BigQueryClient.Create(projectId, googleCredential);
83-
84-
85-
//client.GetTable
86-
87-
//tableSchema.Fields.Insert(0, new TableFieldSchema() { Name = "_PARTITIONTIME", Mode = "REQUIRED", Type = "DATE" });
88-
37+
8938
return client.GetOrCreateTableAsync(
9039
datasetId,
9140
$"{tableId}${date.ToString("yyyyMMdd")}",
@@ -106,59 +55,25 @@ public Task InsertRowsAsync(DateTime date, IEnumerable<GoogleBigQueryRow> rows,
10655

10756
if (dateDiff >= -31 && dateDiff <= 16) {
10857

109-
var bigQueryRows = from r in rows
110-
let dic = properties.ToDictionary(c => c.Name, c => GetBigQueryValue(c, r))
111-
select new BigQueryInsertRow(r.InsertId) { dic };
112-
58+
var bigQueryRows = rows.Select(c => BigQueryInsertRowService.GetBigQueryInsertRow(c, dictionaryOfProperties));
59+
11360
return GetTable(date, cancellationToken)
11461
.ContinueWith((tableTask) => {
11562
BigQueryTable table = tableTask.Result;
116-
return table.InsertRowsAsync(bigQueryRows, new InsertOptions() { AllowUnknownFields = true }, cancellationToken);
63+
64+
return table.InsertRowsAsync(bigQueryRows, new InsertOptions() { AllowUnknownFields = true }, cancellationToken)
65+
.ContinueWith((insertRowsTask) => {
66+
if (insertRowsTask.IsFaulted) {
67+
//... log trace
68+
}
69+
});
11770
}, cancellationToken).Unwrap();
11871

11972
}
12073
}
12174

12275
return Task.WhenAll();
12376
}
124-
125-
private object GetBigQueryValue(PropertyInfo property, GoogleBigQueryRow row) {
126-
switch (property.PropertyType.Name.ToUpper()) {
127-
case "BYTE":
128-
return (int)(byte)property.GetValue(row);
129-
case "CHAR":
130-
return ((char)property.GetValue(row)).ToString();
131-
case "CHAR[]":
132-
return ((char[])property.GetValue(row)).ToString();
133-
case "DATETIME":
134-
var datetimeValue = (DateTime)property.GetValue(row);
135-
return datetimeValue.ToString(BigQueryDateTimeFormat, cultureUS);
136-
case "DECIMAL":
137-
return (float)(decimal)property.GetValue(row);
138-
case "GUID":
139-
return ((Guid)property.GetValue(row)).ToString();
140-
case "UINT64":
141-
return (int)(UInt64)property.GetValue(row);
142-
case "IENUMERABLE`1":
143-
var enumerableValue = property.GetValue(row);
144-
Type innerPropertyType = property.PropertyType.GenericTypeArguments[0];
145-
switch (innerPropertyType.Name.ToUpper()) {
146-
case "BOOLEAN":
147-
return new bool[] { };
148-
case "CHAR":
149-
return ((char[])enumerableValue).ToString();
150-
case "DECIMAL":
151-
return ((decimal[])enumerableValue).Select(c => (float)c);
152-
case "SINGLE":
153-
//return ((Single[])enumerableValue).Select(c => (float)c);
154-
return new float[] { };
155-
}
156-
157-
return enumerableValue;
158-
default:
159-
return property.GetValue(row);
160-
}
161-
}
162-
77+
16378
}
164-
}
79+
}

‎AzureFunctions.Extensions.GoogleBigQuery/GoogleBigQueryExtensionConfig.cs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,13 @@ Task IAsyncCollector<GoogleBigQueryRow>.FlushAsync(CancellationToken cancellatio
4848
var tasks = new List<Task>();
4949

5050
if (items.Count > 0) {
51-
var path = System.IO.Path.GetDirectoryName(typeof(GoogleBigQueryAttribute).Assembly.Location);
52-
var fullPath = System.IO.Path.Combine(path, "..", googleBigQueryAttribute.CredentialsFileName);
53-
var credentials = System.IO.File.ReadAllBytes(fullPath);
51+
52+
byte[] credentials = null;
53+
if (!string.IsNullOrWhiteSpace(googleBigQueryAttribute.CredentialsFileName)) {
54+
var path = System.IO.Path.GetDirectoryName(typeof(GoogleBigQueryAttribute).Assembly.Location);
55+
var fullPath = System.IO.Path.Combine(path, "..", googleBigQueryAttribute.CredentialsFileName);
56+
credentials = System.IO.File.ReadAllBytes(fullPath);
57+
}
5458

5559
Type itemType = items.First().GetType();
5660

‎AzureFunctions.Extensions.GoogleBigQuery/TableSchemaBuilderService.cs‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,21 @@
1111
namespace AzureFunctions.Extensions.GoogleBigQuery {
1212
public class TableSchemaBuilderService {
1313

14-
public static (TableSchema, IEnumerable<System.Reflection.PropertyInfo>) GetTableSchema(Type tableType) {
14+
public static (TableSchema, IDictionary<string, IEnumerable<System.Reflection.PropertyInfo>>) GetTableSchema(Type tableType) {
1515

16-
var properties = tableType.GetProperties()
17-
.Where(c => c.PropertyType.IsPublic && c.CustomAttributes.Any(a => a.AttributeType == typeof(ColumnAttribute)));
16+
var properties = GetPropertyInfo(tableType);
17+
18+
var dictionaryOfProperties = new Dictionary<string, IEnumerable<System.Reflection.PropertyInfo>>();
19+
dictionaryOfProperties.Add(tableType.FullName, properties);
1820

19-
var fields = properties.Select(p => GetTableFieldSchema(p));
21+
var fields = properties.Select(p => GetTableFieldSchema(p, dictionaryOfProperties));
2022

2123
var schema = new Google.Apis.Bigquery.v2.Data.TableSchema() { Fields = fields.ToList() };
2224

23-
return (schema, properties);
25+
return (schema, dictionaryOfProperties);
2426
}
2527

26-
private static TableFieldSchema GetTableFieldSchema(PropertyInfo propertyInfo) {
28+
private static TableFieldSchema GetTableFieldSchema(PropertyInfo propertyInfo, Dictionary<string, IEnumerable<System.Reflection.PropertyInfo>> dictionaryOfProperties) {
2729

2830
//var fields = from property in properties
2931
// let mode = BigQueryFieldMode.Nullable
@@ -59,10 +61,14 @@ private static TableFieldSchema GetTableFieldSchema(PropertyInfo propertyInfo) {
5961
mode = BigQueryFieldMode.Repeated;
6062
}
6163

62-
fields = innerPropertyType
63-
.GetProperties()
64-
.Where(c => c.PropertyType.IsPublic && c.CustomAttributes.Any(a => a.AttributeType == typeof(ColumnAttribute)))
65-
.Select(p => GetTableFieldSchema(p)).ToList();
64+
IEnumerable<PropertyInfo> properties = null;
65+
if (dictionaryOfProperties.ContainsKey(innerPropertyType.FullName)) {
66+
properties = dictionaryOfProperties[innerPropertyType.FullName];
67+
} else {
68+
properties = GetPropertyInfo(innerPropertyType);
69+
dictionaryOfProperties.Add(innerPropertyType.FullName, properties);
70+
}
71+
fields = properties.Select(p => GetTableFieldSchema(p, dictionaryOfProperties)).ToList();
6672

6773
type = "RECORD";
6874

@@ -159,5 +165,11 @@ private static (string, BigQueryFieldMode) GetBigQueryTypeAndMode(PropertyInfo p
159165
return (type, fieldMode);
160166
}
161167

168+
private static IEnumerable<PropertyInfo> GetPropertyInfo(Type type) {
169+
return type
170+
.GetProperties()
171+
.Where(c => c.PropertyType.IsPublic && c.CustomAttributes.Any(a => a.AttributeType == typeof(ColumnAttribute)));
172+
}
173+
162174
}
163-
}
175+
}

0 commit comments

Comments
 (0)