Skip to content

Commit 9bb98a1

Browse files
DamiaoCastroDamiaoCastro
committed
full support from configuration settings
1 parent 5f1882b commit 9bb98a1

File tree

9 files changed

+180
-28
lines changed

9 files changed

+180
-28
lines changed

‎AzureFunctions.Extensions.GoogleBigQuery.ComponentTests/BigQueryServiceTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ public class BigQueryServiceTests {
1414

1515
[TestInitialize]
1616
public void Initialize() {
17-
bigQueryService = new BigQueryService(null, "damiao-1982", "extensiontest", "table1", typeof(TestBigQueryRow));
17+
bigQueryService = new BigQueryService(new GoogleBigQueryAttribute(null, "damiao-1982", "extensiontest", "table1"), typeof(TestBigQueryRow));
1818
}
1919

2020
[TestMethod]

‎AzureFunctions.Extensions.GoogleBigQuery.DemoProject1/AzureFunctions.Extensions.GoogleBigQuery.DemoProject1.csproj‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010
<ProjectReference Include="..\AzureFunctions.Extensions.GoogleBigQuery\AzureFunctions.Extensions.GoogleBigQuery.csproj" />
1111
</ItemGroup>
1212
<ItemGroup>
13+
<None Update="credencials.json">
14+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
15+
</None>
1316
<None Update="host.json">
1417
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
1518
</None>

‎AzureFunctions.Extensions.GoogleBigQuery.DemoProject1/Function1.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,14 @@ public MyBigQueryRow(DateTime date, string insertId) : base(date, insertId) {
1515

1616
}
1717

18+
[Disable]
1819
[FunctionName("Function1")]
1920
public static void Run(
2021
[TimerTrigger("0 */5 * * * *")]TimerInfo myTimer,
2122
[GoogleBigQuery("credencials.json", "projectId", "datasetId","tableId")]
2223
ICollector<MyBigQueryRow> rows) {
2324

24-
rows.Add(new MyBigQueryRow(DateTime.UtcNow, "insertId") { SomeIntegerValue = 1 });
25+
rows.Add(new MyBigQueryRow(DateTime.UtcNow, "insertId1") { SomeIntegerValue = 1 });
2526

2627
}
2728

‎AzureFunctions.Extensions.GoogleBigQuery.DemoProject1/Function2.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
using System;
2+
using System.ComponentModel.DataAnnotations.Schema;
3+
using Microsoft.Azure.WebJobs;
4+
5+
namespace AzureFunctions.Extensions.GoogleBigQuery.DemoProject1 {
6+
public static class Function2 {
7+
8+
public class MyBigQueryRow : GoogleBigQueryRow {
9+
public MyBigQueryRow(DateTime date, string insertId) : base(date, insertId) {
10+
}
11+
12+
[Column]
13+
public string FunctionName { get; set; }
14+
15+
[Column]
16+
public Int64 SomeIntegerValue { get; set; }
17+
18+
}
19+
20+
[FunctionName("Function2")]
21+
public static void Run([TimerTrigger("0 */5 * * * *", RunOnStartup = true)]TimerInfo myTimer,
22+
[GoogleBigQuery("MyGoogleBigQueryConfig2")]
23+
ICollector<MyBigQueryRow> rows) {
24+
25+
rows.Add(new MyBigQueryRow(DateTime.UtcNow, "insertId2") { FunctionName = "Function2", SomeIntegerValue = Int64.Parse(DateTime.UtcNow.ToString("yyyyMMddHHmm")) });
26+
27+
}
28+
}
29+
}

‎AzureFunctions.Extensions.GoogleBigQuery.DemoProject1/Function3.cs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
using System;
2+
using System.ComponentModel.DataAnnotations.Schema;
3+
using Microsoft.Azure.WebJobs;
4+
5+
namespace AzureFunctions.Extensions.GoogleBigQuery.DemoProject1 {
6+
public static class Function3 {
7+
8+
public class MyBigQueryRow : GoogleBigQueryRow {
9+
public MyBigQueryRow(DateTime date, string insertId) : base(date, insertId) {
10+
}
11+
12+
[Column]
13+
public string FunctionName { get; set; }
14+
15+
[Column]
16+
public Int64 SomeIntegerValue { get; set; }
17+
18+
}
19+
20+
[Disable]
21+
[FunctionName("Function3")]
22+
public static void Run([TimerTrigger("0 */5 * * * *", RunOnStartup = true)]TimerInfo myTimer,
23+
[GoogleBigQuery("MyGoogleBigQueryConfig3")]
24+
ICollector<MyBigQueryRow> rows) {
25+
26+
rows.Add(new MyBigQueryRow(DateTime.UtcNow, "insertId3") { FunctionName = "Function3", SomeIntegerValue = Int64.Parse(DateTime.UtcNow.ToString("yyyyMMddHHmm")) });
27+
28+
}
29+
}
30+
}

‎AzureFunctions.Extensions.GoogleBigQuery.DemoProject1/Function4.cs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
using System;
2+
using System.ComponentModel.DataAnnotations.Schema;
3+
using Microsoft.Azure.WebJobs;
4+
5+
namespace AzureFunctions.Extensions.GoogleBigQuery.DemoProject1 {
6+
public static class Function4 {
7+
8+
public class MyBigQueryRow : GoogleBigQueryRow {
9+
public MyBigQueryRow(DateTime date, string insertId) : base(date, insertId) {
10+
}
11+
12+
[Column]
13+
public string FunctionName { get; set; }
14+
15+
[Column]
16+
public Int64 SomeIntegerValue { get; set; }
17+
18+
}
19+
20+
[Disable]
21+
[FunctionName("Function4")]
22+
public static void Run([TimerTrigger("0 */1 * * * *", RunOnStartup = true)]TimerInfo myTimer,
23+
[GoogleBigQuery("MyGoogleBigQueryConfig4")]
24+
ICollector<MyBigQueryRow> rows) {
25+
26+
rows.Add(new MyBigQueryRow(DateTime.UtcNow, "insertId4") { FunctionName = "Function4", SomeIntegerValue = Int64.Parse(DateTime.UtcNow.ToString("yyyyMMddHHmm")) });
27+
28+
}
29+
}
30+
}

‎AzureFunctions.Extensions.GoogleBigQuery/BigQueryService.cs‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,35 +11,35 @@ namespace AzureFunctions.Extensions.GoogleBigQuery {
1111

1212
public class BigQueryService {
1313

14-
private readonly byte[] credentials;
15-
private readonly string projectId;
16-
private readonly string datasetId;
17-
private readonly string tableId;
14+
private readonly GoogleBigQueryAttribute googleBigQueryAttribute;
1815
private readonly TableSchema tableSchema;
1916
private readonly IDictionary<string, IEnumerable<System.Reflection.PropertyInfo>> dictionaryOfProperties;
2017

21-
public BigQueryService(byte[] credentials, string projectId, string datasetId, string tableId, Type itemType) {
22-
this.credentials = credentials;
23-
this.projectId = projectId;
24-
this.datasetId = datasetId;
25-
this.tableId = tableId;
18+
public BigQueryService(GoogleBigQueryAttribute googleBigQueryAttribute, Type itemType) {
19+
this.googleBigQueryAttribute = GoogleBigQueryAttribute.GetAttributeByConfiguration(googleBigQueryAttribute);
2620
(this.tableSchema, this.dictionaryOfProperties) = TableSchemaBuilderService.GetTableSchema(itemType);
2721
}
2822

2923
private Task<BigQueryTable> GetTable(DateTime date, CancellationToken cancellationToken) {
3024

3125
GoogleCredential googleCredential = null;
32-
if (credentials != null) {
33-
googleCredential = GoogleCredential.FromStream(new System.IO.MemoryStream(credentials));
26+
if (googleBigQueryAttribute.Credentials != null) {
27+
googleCredential = GoogleCredential.FromStream(new System.IO.MemoryStream(googleBigQueryAttribute.Credentials));
28+
} else {
29+
if (!string.IsNullOrWhiteSpace(googleBigQueryAttribute.CredentialsFileName)) {
30+
var path = System.IO.Path.GetDirectoryName(typeof(GoogleBigQueryAttribute).Assembly.Location);
31+
var fullPath = System.IO.Path.Combine(path, "..", googleBigQueryAttribute.CredentialsFileName);
32+
var credentials = System.IO.File.ReadAllBytes(fullPath);
33+
googleCredential = GoogleCredential.FromStream(new System.IO.MemoryStream(credentials));
34+
}
3435
}
35-
var client = BigQueryClient.Create(projectId, googleCredential);
36+
var client = BigQueryClient.Create(googleBigQueryAttribute.ProjectId, googleCredential);
3637

3738
//return client.GetOrCreateTableAsync(datasetId, tableId, tableSchema, null, new CreateTableOptions() { TimePartitioning = new TimePartitioning() { Type = "DAY" } }, cancellationToken)
3839
// .ContinueWith((createTableTask) => {
3940
// return client.GetTableAsync(datasetId, $"{tableId}${date:yyyyMMdd}", null, cancellationToken);
4041
// }, cancellationToken).Unwrap();
41-
return client.GetTableAsync(datasetId, $"{tableId}${date:yyyyMMdd}", null, cancellationToken);
42-
42+
return client.GetTableAsync(googleBigQueryAttribute.DatasetId, $"{googleBigQueryAttribute.TableId}${date:yyyyMMdd}", null, cancellationToken);
4343
}
4444

4545
public Task InsertRowsAsync(DateTime date, IEnumerable<GoogleBigQueryRow> rows, CancellationToken cancellationToken) {

‎AzureFunctions.Extensions.GoogleBigQuery/GoogleBigQueryAttribute.cs‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,74 @@ public GoogleBigQueryAttribute(string credentialsFileName, string projectId, str
2828
TableId = tableId;
2929
}
3030

31+
internal GoogleBigQueryAttribute(byte[] credentials, string projectId, string datasetId, string tableId) {
32+
if (string.IsNullOrWhiteSpace(projectId)) { throw new ArgumentNullException(nameof(projectId)); }
33+
if (string.IsNullOrWhiteSpace(datasetId)) { throw new ArgumentNullException(nameof(datasetId)); }
34+
if (string.IsNullOrWhiteSpace(tableId)) { throw new ArgumentNullException(nameof(tableId)); }
35+
36+
Credentials = credentials ?? throw new ArgumentNullException(nameof(credentials));
37+
ProjectId = projectId;
38+
DatasetId = datasetId;
39+
TableId = tableId;
40+
}
41+
42+
internal GoogleBigQueryAttribute(string projectId, string datasetId, string tableId) {
43+
if (string.IsNullOrWhiteSpace(projectId)) { throw new ArgumentNullException(nameof(projectId)); }
44+
if (string.IsNullOrWhiteSpace(datasetId)) { throw new ArgumentNullException(nameof(datasetId)); }
45+
if (string.IsNullOrWhiteSpace(tableId)) { throw new ArgumentNullException(nameof(tableId)); }
46+
47+
ProjectId = projectId;
48+
DatasetId = datasetId;
49+
TableId = tableId;
50+
}
51+
52+
/// <summary>
53+
/// using this contructor, the settings will come from the configuration file.
54+
/// you should configure:
55+
/// 'your configuration node name'.Credentials -> string representation of the JSON credential files given in the google cloud "service account" bit
56+
/// 'your configuration node name'.ProjectId -> projectId where the refered google pubsub is contained in
57+
/// 'your configuration node name'.DatasetId -> datasetId of the refered google bigquery
58+
/// 'your configuration node name'.TableId -> tableId of the refered google bigquery
59+
/// </summary>
60+
/// <param name="configurationNodeName">prefix name that you gave to your configuration.</param>
61+
public GoogleBigQueryAttribute(string configurationNodeName) {
62+
if (string.IsNullOrWhiteSpace(configurationNodeName)) { throw new ArgumentNullException(nameof(configurationNodeName)); }
63+
64+
ConfigurationNodeName = configurationNodeName;
65+
}
66+
3167
public string CredentialsFileName { get; }
68+
internal byte[] Credentials { get; }
69+
3270
public string ProjectId { get; }
3371
public string DatasetId { get; }
3472
public string TableId { get; }
3573

74+
public string ConfigurationNodeName { get; }
75+
76+
internal static GoogleBigQueryAttribute GetAttributeByConfiguration(GoogleBigQueryAttribute googleBigQueryAttribute) {
77+
if (string.IsNullOrWhiteSpace(googleBigQueryAttribute.ConfigurationNodeName)) { return googleBigQueryAttribute; }
78+
79+
var credentialsString = Environment.GetEnvironmentVariable($"{googleBigQueryAttribute.ConfigurationNodeName}.{nameof(Credentials)}", EnvironmentVariableTarget.Process);
80+
var credentialsFileName = Environment.GetEnvironmentVariable($"{googleBigQueryAttribute.ConfigurationNodeName}.{nameof(CredentialsFileName)}", EnvironmentVariableTarget.Process);
81+
var projectId = Environment.GetEnvironmentVariable($"{googleBigQueryAttribute.ConfigurationNodeName}.{nameof(ProjectId)}", EnvironmentVariableTarget.Process);
82+
var datasetId = Environment.GetEnvironmentVariable($"{googleBigQueryAttribute.ConfigurationNodeName}.{nameof(DatasetId)}", EnvironmentVariableTarget.Process);
83+
var tableId = Environment.GetEnvironmentVariable($"{googleBigQueryAttribute.ConfigurationNodeName}.{nameof(TableId)}", EnvironmentVariableTarget.Process);
84+
85+
GoogleBigQueryAttribute newGoogleBigQueryAttribute = null;
86+
if (string.IsNullOrWhiteSpace(credentialsFileName) && string.IsNullOrWhiteSpace(credentialsString)) {
87+
newGoogleBigQueryAttribute = new GoogleBigQueryAttribute(projectId, datasetId, tableId);
88+
} else {
89+
if (string.IsNullOrWhiteSpace(credentialsString)) {
90+
newGoogleBigQueryAttribute = new GoogleBigQueryAttribute(credentialsFileName, projectId, datasetId, tableId);
91+
} else {
92+
var credentials = System.Text.Encoding.UTF8.GetBytes(credentialsString);
93+
newGoogleBigQueryAttribute = new GoogleBigQueryAttribute(credentials, projectId, datasetId, tableId);
94+
}
95+
}
96+
97+
return newGoogleBigQueryAttribute;
98+
}
99+
36100
}
37101
}

‎AzureFunctions.Extensions.GoogleBigQuery/GoogleBigQueryExtensionConfig.cs‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -49,21 +49,16 @@ Task IAsyncCollector<GoogleBigQueryRow>.FlushAsync(CancellationToken cancellatio
4949

5050
if (items.Count > 0) {
5151

52-
byte[] credentials = null;
53-
if (!string.IsNullOrWhiteSpace(googleBigQueryAttribute.CredentialsFileName)) {
54-
var path = System.IO.Path.GetDirectoryName(typeof(GoogleBigQueryAttribute).Assembly.Location);
55-
var fullPath = System.IO.Path.Combine(path, "..", googleBigQueryAttribute.CredentialsFileName);
56-
credentials = System.IO.File.ReadAllBytes(fullPath);
57-
}
52+
//byte[] credentials = null;
53+
//if (!string.IsNullOrWhiteSpace(googleBigQueryAttribute.CredentialsFileName)) {
54+
// var path = System.IO.Path.GetDirectoryName(typeof(GoogleBigQueryAttribute).Assembly.Location);
55+
// var fullPath = System.IO.Path.Combine(path, "..", googleBigQueryAttribute.CredentialsFileName);
56+
// credentials = System.IO.File.ReadAllBytes(fullPath);
57+
//}
5858

5959
Type itemType = items.First().GetType();
6060

61-
var bqService =
62-
new BigQueryService(credentials,
63-
googleBigQueryAttribute.ProjectId,
64-
googleBigQueryAttribute.DatasetId,
65-
googleBigQueryAttribute.TableId,
66-
itemType);
61+
var bqService = new BigQueryService(googleBigQueryAttribute, itemType);
6762

6863
var groups = items.GroupBy(c => c.Date.Date);
6964
foreach (var group in groups) {

0 commit comments

Comments
 (0)