feat: add support for Linux node config (terraform-google-modules#782)

m10ev · Tencho Tenev · bharathkkb · web-flow · commit 98826e6029d5 · 2021-01-14T11:16:42.000-06:00
* Add linux node config

* Add variable node_pools_linux_node_configs

* Generate modules

* Fix dynamic block

* Generate modules

* Deep merge for sysctls maps

* Generate modules

* Add sysctls to node_pool example and test

* Flatten sysctls map

* Generate modules

* Fix typo in test

* Fix trailing whitespace

* Fix parens

* Update autogen/main/cluster.tf.tmpl

Co-authored-by: Tencho Tenev &lt;tencho@improbable.io&gt;

* Generate modules

Co-authored-by: Tencho Tenev &lt;tencho@improbable.io&gt;
Co-authored-by: Bharath KKB &lt;bharathkrishnakb@gmail.com&gt;
diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
@@ -98,7 +98,7 @@ resource "google_container_cluster" "primary" {
       }
     }
   }
-  
+
   vertical_pod_autoscaling {
     enabled = var.enable_vertical_pod_autoscaling
   }
@@ -565,6 +565,20 @@ resource "google_container_node_pool" "pools" {
         cpu_manager_policy = lookup(each.value, "cpu_manager_policy")
       }
     }
+
+    dynamic "linux_node_config" {
+      for_each = merge(
+        local.node_pools_linux_node_configs_sysctls["all"],
+        local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+      ) != {} ? [1] : []
+
+      content {
+        sysctls = merge(
+          local.node_pools_linux_node_configs_sysctls["all"],
+          local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+        )
+      }
+    }
     {% endif %}
 
     shielded_instance_config {
diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl
@@ -194,6 +194,19 @@ variable "node_pools_metadata" {
     default-node-pool = {}
   }
 }
+{% if beta_cluster %}
+
+variable "node_pools_linux_node_configs_sysctls" {
+  type        = map(map(string))
+  description = "Map of maps containing linux node config sysctls by node-pool name"
+
+  # Default is being set in variables_defaults.tf
+  default = {
+    all               = {}
+    default-node-pool = {}
+  }
+}
+{% endif %}
 
 variable "resource_usage_export_dataset_id" {
   type        = string
diff --git a/autogen/main/variables_defaults.tf b/autogen/main/variables_defaults.tf
@@ -69,4 +69,16 @@ locals {
     ),
     var.node_pools_oauth_scopes
   )
+  {% if beta_cluster %}
+
+  node_pools_linux_node_configs_sysctls = merge(
+    { all = {} },
+    { default-node-pool = {} },
+    zipmap(
+      [for node_pool in var.node_pools : node_pool["name"]],
+      [for node_pool in var.node_pools : {}]
+    ),
+    var.node_pools_linux_node_configs_sysctls
+  )
+  {% endif %}
 }
diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf
@@ -121,4 +121,16 @@ module "gke" {
       "pool-01-example",
     ]
   }
+
+  node_pools_linux_node_configs_sysctls = {
+    all = {
+      "net.core.netdev_max_backlog" = "10000"
+    }
+    pool-01 = {
+      "net.core.rmem_max" = "10000"
+    }
+    pool-03 = {
+      "net.core.netdev_max_backlog" = "20000"
+    }
+  }
 }
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
@@ -220,6 +220,7 @@ Then perform the following commands on the root folder:
 | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA_SERVER"` | no |
 | node\_pools | List of maps containing node pools | `list(map(string))` | <pre>[<br>  {<br>    "name": "default-node-pool"<br>  }<br>]</pre> | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
+| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | <pre>{<br>  "all": [<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ],<br>  "default-node-pool": []<br>}</pre> | no |
 | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | <pre>{<br>  "all": [],<br>  "default-node-pool": []<br>}</pre> | no |
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -522,6 +522,20 @@ resource "google_container_node_pool" "pools" {
       }
     }
 
+    dynamic "linux_node_config" {
+      for_each = merge(
+        local.node_pools_linux_node_configs_sysctls["all"],
+        local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+      ) != {} ? [1] : []
+
+      content {
+        sysctls = merge(
+          local.node_pools_linux_node_configs_sysctls["all"],
+          local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+        )
+      }
+    }
+
     shielded_instance_config {
       enable_secure_boot          = lookup(each.value, "enable_secure_boot", false)
       enable_integrity_monitoring = lookup(each.value, "enable_integrity_monitoring", true)
diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
@@ -193,6 +193,17 @@ variable "node_pools_metadata" {
   }
 }
 
+variable "node_pools_linux_node_configs_sysctls" {
+  type        = map(map(string))
+  description = "Map of maps containing linux node config sysctls by node-pool name"
+
+  # Default is being set in variables_defaults.tf
+  default = {
+    all               = {}
+    default-node-pool = {}
+  }
+}
+
 variable "resource_usage_export_dataset_id" {
   type        = string
   description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export."
diff --git a/modules/beta-private-cluster-update-variant/variables_defaults.tf b/modules/beta-private-cluster-update-variant/variables_defaults.tf
@@ -69,4 +69,14 @@ locals {
     ),
     var.node_pools_oauth_scopes
   )
+
+  node_pools_linux_node_configs_sysctls = merge(
+    { all = {} },
+    { default-node-pool = {} },
+    zipmap(
+      [for node_pool in var.node_pools : node_pool["name"]],
+      [for node_pool in var.node_pools : {}]
+    ),
+    var.node_pools_linux_node_configs_sysctls
+  )
 }
diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
@@ -198,6 +198,7 @@ Then perform the following commands on the root folder:
 | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA_SERVER"` | no |
 | node\_pools | List of maps containing node pools | `list(map(string))` | <pre>[<br>  {<br>    "name": "default-node-pool"<br>  }<br>]</pre> | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
+| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | <pre>{<br>  "all": [<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ],<br>  "default-node-pool": []<br>}</pre> | no |
 | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | <pre>{<br>  "all": [],<br>  "default-node-pool": []<br>}</pre> | no |
diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
@@ -438,6 +438,20 @@ resource "google_container_node_pool" "pools" {
       }
     }
 
+    dynamic "linux_node_config" {
+      for_each = merge(
+        local.node_pools_linux_node_configs_sysctls["all"],
+        local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+      ) != {} ? [1] : []
+
+      content {
+        sysctls = merge(
+          local.node_pools_linux_node_configs_sysctls["all"],
+          local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+        )
+      }
+    }
+
     shielded_instance_config {
       enable_secure_boot          = lookup(each.value, "enable_secure_boot", false)
       enable_integrity_monitoring = lookup(each.value, "enable_integrity_monitoring", true)
diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf
@@ -193,6 +193,17 @@ variable "node_pools_metadata" {
   }
 }
 
+variable "node_pools_linux_node_configs_sysctls" {
+  type        = map(map(string))
+  description = "Map of maps containing linux node config sysctls by node-pool name"
+
+  # Default is being set in variables_defaults.tf
+  default = {
+    all               = {}
+    default-node-pool = {}
+  }
+}
+
 variable "resource_usage_export_dataset_id" {
   type        = string
   description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export."
diff --git a/modules/beta-private-cluster/variables_defaults.tf b/modules/beta-private-cluster/variables_defaults.tf
@@ -69,4 +69,14 @@ locals {
     ),
     var.node_pools_oauth_scopes
   )
+
+  node_pools_linux_node_configs_sysctls = merge(
+    { all = {} },
+    { default-node-pool = {} },
+    zipmap(
+      [for node_pool in var.node_pools : node_pool["name"]],
+      [for node_pool in var.node_pools : {}]
+    ),
+    var.node_pools_linux_node_configs_sysctls
+  )
 }
diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md
@@ -209,6 +209,7 @@ Then perform the following commands on the root folder:
 | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA_SERVER"` | no |
 | node\_pools | List of maps containing node pools | `list(map(string))` | <pre>[<br>  {<br>    "name": "default-node-pool"<br>  }<br>]</pre> | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
+| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | <pre>{<br>  "all": [<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ],<br>  "default-node-pool": []<br>}</pre> | no |
 | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | <pre>{<br>  "all": [],<br>  "default-node-pool": []<br>}</pre> | no |
diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf
@@ -503,6 +503,20 @@ resource "google_container_node_pool" "pools" {
       }
     }
 
+    dynamic "linux_node_config" {
+      for_each = merge(
+        local.node_pools_linux_node_configs_sysctls["all"],
+        local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+      ) != {} ? [1] : []
+
+      content {
+        sysctls = merge(
+          local.node_pools_linux_node_configs_sysctls["all"],
+          local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+        )
+      }
+    }
+
     shielded_instance_config {
       enable_secure_boot          = lookup(each.value, "enable_secure_boot", false)
       enable_integrity_monitoring = lookup(each.value, "enable_integrity_monitoring", true)
diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf
@@ -193,6 +193,17 @@ variable "node_pools_metadata" {
   }
 }
 
+variable "node_pools_linux_node_configs_sysctls" {
+  type        = map(map(string))
+  description = "Map of maps containing linux node config sysctls by node-pool name"
+
+  # Default is being set in variables_defaults.tf
+  default = {
+    all               = {}
+    default-node-pool = {}
+  }
+}
+
 variable "resource_usage_export_dataset_id" {
   type        = string
   description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export."
diff --git a/modules/beta-public-cluster-update-variant/variables_defaults.tf b/modules/beta-public-cluster-update-variant/variables_defaults.tf
@@ -69,4 +69,14 @@ locals {
     ),
     var.node_pools_oauth_scopes
   )
+
+  node_pools_linux_node_configs_sysctls = merge(
+    { all = {} },
+    { default-node-pool = {} },
+    zipmap(
+      [for node_pool in var.node_pools : node_pool["name"]],
+      [for node_pool in var.node_pools : {}]
+    ),
+    var.node_pools_linux_node_configs_sysctls
+  )
 }
diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md
@@ -187,6 +187,7 @@ Then perform the following commands on the root folder:
 | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA_SERVER"` | no |
 | node\_pools | List of maps containing node pools | `list(map(string))` | <pre>[<br>  {<br>    "name": "default-node-pool"<br>  }<br>]</pre> | no |
 | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
+| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | <pre>{<br>  "all": {},<br>  "default-node-pool": {}<br>}</pre> | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | <pre>{<br>  "all": [<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ],<br>  "default-node-pool": []<br>}</pre> | no |
 | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | <pre>{<br>  "all": [],<br>  "default-node-pool": []<br>}</pre> | no |
diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
@@ -419,6 +419,20 @@ resource "google_container_node_pool" "pools" {
       }
     }
 
+    dynamic "linux_node_config" {
+      for_each = merge(
+        local.node_pools_linux_node_configs_sysctls["all"],
+        local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+      ) != {} ? [1] : []
+
+      content {
+        sysctls = merge(
+          local.node_pools_linux_node_configs_sysctls["all"],
+          local.node_pools_linux_node_configs_sysctls[each.value["name"]]
+        )
+      }
+    }
+
     shielded_instance_config {
       enable_secure_boot          = lookup(each.value, "enable_secure_boot", false)
       enable_integrity_monitoring = lookup(each.value, "enable_integrity_monitoring", true)
diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf
@@ -193,6 +193,17 @@ variable "node_pools_metadata" {
   }
 }
 
+variable "node_pools_linux_node_configs_sysctls" {
+  type        = map(map(string))
+  description = "Map of maps containing linux node config sysctls by node-pool name"
+
+  # Default is being set in variables_defaults.tf
+  default = {
+    all               = {}
+    default-node-pool = {}
+  }
+}
+
 variable "resource_usage_export_dataset_id" {
   type        = string
   description = "The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export."
diff --git a/modules/beta-public-cluster/variables_defaults.tf b/modules/beta-public-cluster/variables_defaults.tf
@@ -69,4 +69,14 @@ locals {
     ),
     var.node_pools_oauth_scopes
   )
+
+  node_pools_linux_node_configs_sysctls = merge(
+    { all = {} },
+    { default-node-pool = {} },
+    zipmap(
+      [for node_pool in var.node_pools : node_pool["name"]],
+      [for node_pool in var.node_pools : {}]
+    ),
+    var.node_pools_linux_node_configs_sysctls
+  )
 }
diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb
@@ -174,6 +174,22 @@
             )
           )
         end
+
+        it "has the expected linux node config sysctls" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-01",
+              "config" => including(
+                "linuxNodeConfig" => including(
+                  "sysctls" => including(
+                    "net.core.netdev_max_backlog" => "10000",
+                    "net.core.rmem_max" => "10000"
+                  )
+                )
+              )
+            )
+          )
+        end
       end
 
       describe "pool-02" do
@@ -303,6 +319,21 @@
             )
           )
         end
+
+        it "has the expected linux node config sysctls" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-02",
+              "config" => including(
+                "linuxNodeConfig" => including(
+                  "sysctls" => including(
+                    "net.core.netdev_max_backlog" => "10000"
+                  )
+                )
+              )
+            )
+          )
+        end
       end
 
       describe "pool-03" do
@@ -396,6 +427,21 @@
             )
           )
         end
+
+        it "has the expected linux node config sysctls" do
+          expect(data['nodePools']).to include(
+            including(
+              "name" => "pool-03",
+              "config" => including(
+                "linuxNodeConfig" => including(
+                  "sysctls" => including(
+                    "net.core.netdev_max_backlog" => "20000"
+                  )
+                )
+              )
+            )
+          )
+        end
       end
     end
   end

Original file line number	Diff line number	Diff line change
`@@ -98,7 +98,7 @@ resource "google_container_cluster" "primary" {`
`98`	`98`	`}`
`99`	`99`	`}`
`100`	`100`	`}`
`101`		`-`
	`101`	`+`
`102`	`102`	`vertical_pod_autoscaling {`
`103`	`103`	`enabled = var.enable_vertical_pod_autoscaling`
`104`	`104`	`}`
`@@ -565,6 +565,20 @@ resource "google_container_node_pool" "pools" {`
`565`	`565`	`cpu_manager_policy = lookup(each.value, "cpu_manager_policy")`
`566`	`566`	`}`
`567`	`567`	`}`
	`568`	`+`
	`569`	`+ dynamic "linux_node_config" {`
	`570`	`+ for_each = merge(`
	`571`	`+ local.node_pools_linux_node_configs_sysctls["all"],`
	`572`	`+ local.node_pools_linux_node_configs_sysctls[each.value["name"]]`
	`573`	`+ ) != {} ? [1] : []`
	`574`	`+`
	`575`	`+ content {`
	`576`	`+ sysctls = merge(`
	`577`	`+ local.node_pools_linux_node_configs_sysctls["all"],`
	`578`	`+ local.node_pools_linux_node_configs_sysctls[each.value["name"]]`
	`579`	`+ )`
	`580`	`+ }`
	`581`	`+ }`
`568`	`582`	`{% endif %}`
`569`	`583`
`570`	`584`	`shielded_instance_config {`
Original file line number	Diff line number	Diff line change
`@@ -121,4 +121,16 @@ module "gke" {`
`121`	`121`	`"pool-01-example",`
`122`	`122`	`]`
`123`	`123`	`}`
	`124`	`+`
	`125`	`+ node_pools_linux_node_configs_sysctls = {`
	`126`	`+ all = {`
	`127`	`+ "net.core.netdev_max_backlog" = "10000"`
	`128`	`+ }`
	`129`	`+ pool-01 = {`
	`130`	`+ "net.core.rmem_max" = "10000"`
	`131`	`+ }`
	`132`	`+ pool-03 = {`
	`133`	`+ "net.core.netdev_max_backlog" = "20000"`
	`134`	`+ }`
	`135`	`+ }`
`124`	`136`	`}`