Skip to content

Commit 369484f

Browse files
AndrewSirenkoAndrewSirenko
authored
feat: Add volumeInitializationRate option to EBS blockDeviceMappings (#8048)
1 parent 81f2ea8 commit 369484f

File tree

10 files changed

+124
-8
lines changed

10 files changed

+124
-8
lines changed

‎charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,17 @@ spec:
198198
Valid Range: Minimum value of 125. Maximum value of 1000.
199199
format: int64
200200
type: integer
201+
volumeInitializationRate:
202+
description: |-
203+
VolumeInitializationRate specifies the Amazon EBS Provisioned Rate for Volume Initialization,
204+
in MiB/s, at which to download the snapshot blocks from Amazon S3 to the volume. This is also known as volume
205+
initialization. Specifying a volume initialization rate ensures that the volume is initialized at a
206+
predictable and consistent rate after creation. Only allowed if SnapshotID is set.
207+
Valid Range: Minimum value of 100. Maximum value of 300.
208+
format: int32
209+
maximum: 300
210+
minimum: 100
211+
type: integer
201212
volumeSize:
202213
description: |-
203214
VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or
@@ -231,6 +242,8 @@ spec:
231242
x-kubernetes-validations:
232243
- message: snapshotID or volumeSize must be defined
233244
rule: has(self.snapshotID) || has(self.volumeSize)
245+
- message: snapshotID must be set when volumeInitializationRate is set
246+
rule: '!has(self.volumeInitializationRate) || (has(self.snapshotID) && self.snapshotID != '''')'
234247
rootVolume:
235248
description: |-
236249
RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ require (
1010
github.com/aws/aws-sdk-go-v2 v1.36.3
1111
github.com/aws/aws-sdk-go-v2/config v1.29.14
1212
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30
13-
github.com/aws/aws-sdk-go-v2/service/ec2 v1.212.0
13+
github.com/aws/aws-sdk-go-v2/service/ec2 v1.215.0
1414
github.com/aws/aws-sdk-go-v2/service/eks v1.64.0
1515
github.com/aws/aws-sdk-go-v2/service/fis v1.33.2
1616
github.com/aws/aws-sdk-go-v2/service/iam v1.41.1

‎go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0io
2626
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
2727
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
2828
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
29-
github.com/aws/aws-sdk-go-v2/service/ec2 v1.212.0 h1:z5thR/zKUlw7gd1OT59xBHm4AKBf2kPXKHFvVzLMfBk=
30-
github.com/aws/aws-sdk-go-v2/service/ec2 v1.212.0/go.mod h1:ouvGEfHbLaIlWwpDpOVWPWR+YwO0HDv3vm5tYLq8ImY=
29+
github.com/aws/aws-sdk-go-v2/service/ec2 v1.215.0 h1:6a5U/gnVIPWWtS2CCdOkrxos3Se8IL9jNMJLyD4BEU8=
30+
github.com/aws/aws-sdk-go-v2/service/ec2 v1.215.0/go.mod h1:ouvGEfHbLaIlWwpDpOVWPWR+YwO0HDv3vm5tYLq8ImY=
3131
github.com/aws/aws-sdk-go-v2/service/eks v1.64.0 h1:EYeOThTRysemFtC6J6h6b7dNg3jN03QuO5cg92ojIQE=
3232
github.com/aws/aws-sdk-go-v2/service/eks v1.64.0/go.mod h1:v1xXy6ea0PHtWkjFUvAUh6B/5wv7UF909Nru0dOIJDk=
3333
github.com/aws/aws-sdk-go-v2/service/fis v1.33.2 h1:XGjI4EWC1sR1voaYJU2gGK96WjKIYV9K0YrSDk1P8n0=

‎pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,17 @@ spec:
195195
Valid Range: Minimum value of 125. Maximum value of 1000.
196196
format: int64
197197
type: integer
198+
volumeInitializationRate:
199+
description: |-
200+
VolumeInitializationRate specifies the Amazon EBS Provisioned Rate for Volume Initialization,
201+
in MiB/s, at which to download the snapshot blocks from Amazon S3 to the volume. This is also known as volume
202+
initialization. Specifying a volume initialization rate ensures that the volume is initialized at a
203+
predictable and consistent rate after creation. Only allowed if SnapshotID is set.
204+
Valid Range: Minimum value of 100. Maximum value of 300.
205+
format: int32
206+
maximum: 300
207+
minimum: 100
208+
type: integer
198209
volumeSize:
199210
description: |-
200211
VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or
@@ -228,6 +239,8 @@ spec:
228239
x-kubernetes-validations:
229240
- message: snapshotID or volumeSize must be defined
230241
rule: has(self.snapshotID) || has(self.volumeSize)
242+
- message: snapshotID must be set when volumeInitializationRate is set
243+
rule: '!has(self.volumeInitializationRate) || (has(self.snapshotID) && self.snapshotID != '''')'
231244
rootVolume:
232245
description: |-
233246
RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can

‎pkg/apis/v1/ec2nodeclass.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,7 @@ type BlockDeviceMapping struct {
357357
DeviceName *string `json:"deviceName,omitempty"`
358358
// EBS contains parameters used to automatically set up EBS volumes when an instance is launched.
359359
// +kubebuilder:validation:XValidation:message="snapshotID or volumeSize must be defined",rule="has(self.snapshotID) || has(self.volumeSize)"
360+
// +kubebuilder:validation:XValidation:message="snapshotID must be set when volumeInitializationRate is set",rule="!has(self.volumeInitializationRate) || (has(self.snapshotID) && self.snapshotID != '')"
360361
// +optional
361362
EBS *BlockDevice `json:"ebs,omitempty"`
362363
// RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can
@@ -405,6 +406,15 @@ type BlockDevice struct {
405406
// Valid Range: Minimum value of 125. Maximum value of 1000.
406407
// +optional
407408
Throughput *int64 `json:"throughput,omitempty"`
409+
// VolumeInitializationRate specifies the Amazon EBS Provisioned Rate for Volume Initialization,
410+
// in MiB/s, at which to download the snapshot blocks from Amazon S3 to the volume. This is also known as volume
411+
// initialization. Specifying a volume initialization rate ensures that the volume is initialized at a
412+
// predictable and consistent rate after creation. Only allowed if SnapshotID is set.
413+
// Valid Range: Minimum value of 100. Maximum value of 300.
414+
// +kubebuilder:validation:Minimum:=100
415+
// +kubebuilder:validation:Maximum:=300
416+
// +optional
417+
VolumeInitializationRate *int32 `json:"volumeInitializationRate,omitempty"`
408418
// VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or
409419
// a volume size. The following are the supported volumes sizes for each volume
410420
// type:

‎pkg/apis/v1/ec2nodeclass_validation_cel_test.go

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1145,6 +1145,74 @@ var _ = Describe("CEL/Validation", func() {
11451145
}
11461146
Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed()))
11471147
})
1148+
It("should fail if VolumeInitializationRate set but SnapshotID not specified", func() {
1149+
nodeClass := &v1.EC2NodeClass{
1150+
ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}),
1151+
Spec: v1.EC2NodeClassSpec{
1152+
AMISelectorTerms: nc.Spec.AMISelectorTerms,
1153+
SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms,
1154+
SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms,
1155+
Role: nc.Spec.Role,
1156+
BlockDeviceMappings: []*v1.BlockDeviceMapping{
1157+
{
1158+
DeviceName: aws.String("map-device-1"),
1159+
EBS: &v1.BlockDevice{
1160+
VolumeSize: &resource.Quantity{},
1161+
VolumeInitializationRate: aws.Int32(100),
1162+
},
1163+
RootVolume: false,
1164+
},
1165+
},
1166+
},
1167+
}
1168+
Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed()))
1169+
})
1170+
It("should fail if VolumeInitializationRate too low", func() {
1171+
nodeClass := &v1.EC2NodeClass{
1172+
ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}),
1173+
Spec: v1.EC2NodeClassSpec{
1174+
AMISelectorTerms: nc.Spec.AMISelectorTerms,
1175+
SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms,
1176+
SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms,
1177+
Role: nc.Spec.Role,
1178+
BlockDeviceMappings: []*v1.BlockDeviceMapping{
1179+
{
1180+
DeviceName: aws.String("map-device-1"),
1181+
EBS: &v1.BlockDevice{
1182+
VolumeSize: &resource.Quantity{},
1183+
SnapshotID: aws.String("snap-1"),
1184+
VolumeInitializationRate: aws.Int32(99),
1185+
},
1186+
RootVolume: false,
1187+
},
1188+
},
1189+
},
1190+
}
1191+
Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed()))
1192+
})
1193+
It("should fail if VolumeInitializationRate too high", func() {
1194+
nodeClass := &v1.EC2NodeClass{
1195+
ObjectMeta: test.ObjectMeta(metav1.ObjectMeta{}),
1196+
Spec: v1.EC2NodeClassSpec{
1197+
AMISelectorTerms: nc.Spec.AMISelectorTerms,
1198+
SubnetSelectorTerms: nc.Spec.SubnetSelectorTerms,
1199+
SecurityGroupSelectorTerms: nc.Spec.SecurityGroupSelectorTerms,
1200+
Role: nc.Spec.Role,
1201+
BlockDeviceMappings: []*v1.BlockDeviceMapping{
1202+
{
1203+
DeviceName: aws.String("map-device-1"),
1204+
EBS: &v1.BlockDevice{
1205+
VolumeSize: &resource.Quantity{},
1206+
SnapshotID: aws.String("snap-1"),
1207+
VolumeInitializationRate: aws.Int32(888),
1208+
},
1209+
RootVolume: false,
1210+
},
1211+
},
1212+
},
1213+
}
1214+
Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed()))
1215+
})
11481216
})
11491217
Context("Role Immutability", func() {
11501218
It("should fail if role is not defined", func() {

‎pkg/apis/v1/zz_generated.deepcopy.go

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/providers/instancetype/suite_test.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2241,7 +2241,8 @@ var _ = Describe("InstanceTypeProvider", func() {
22412241
{
22422242
DeviceName: aws.String("/dev/xvda"),
22432243
EBS: &v1.BlockDevice{
2244-
SnapshotID: aws.String("snap-xxxxxxxx"),
2244+
SnapshotID: aws.String("snap-xxxxxxxx"),
2245+
VolumeInitializationRate: aws.Int32(100),
22452246
},
22462247
},
22472248
}
@@ -2265,6 +2266,7 @@ var _ = Describe("InstanceTypeProvider", func() {
22652266
Expect(ltInput.LaunchTemplateData.BlockDeviceMappings).To(HaveLen(1))
22662267
Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].DeviceName).To(Equal("/dev/xvda"))
22672268
Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.SnapshotId).To(Equal("snap-xxxxxxxx"))
2269+
Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeInitializationRate).To(Equal(int32(100)))
22682270
})
22692271
})
22702272
It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for AL2 Root volume", func() {
@@ -2278,6 +2280,7 @@ var _ = Describe("InstanceTypeProvider", func() {
22782280
Expect(ltInput.LaunchTemplateData.BlockDeviceMappings).To(HaveLen(1))
22792281
Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].DeviceName).To(Equal("/dev/xvda"))
22802282
Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.SnapshotId).To(Equal("snap-xxxxxxxx"))
2283+
Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeInitializationRate).To(Equal(int32(100)))
22812284
})
22822285
})
22832286
It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for AL2023 Root volume", func() {
@@ -2294,6 +2297,7 @@ var _ = Describe("InstanceTypeProvider", func() {
22942297
Expect(ltInput.LaunchTemplateData.BlockDeviceMappings).To(HaveLen(1))
22952298
Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].DeviceName).To(Equal("/dev/xvda"))
22962299
Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.SnapshotId).To(Equal("snap-xxxxxxxx"))
2300+
Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeInitializationRate).To(Equal(int32(100)))
22972301
})
22982302
})
22992303
It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for Bottlerocket Root volume", func() {
@@ -2310,6 +2314,7 @@ var _ = Describe("InstanceTypeProvider", func() {
23102314
Expect(ltInput.LaunchTemplateData.BlockDeviceMappings).To(HaveLen(1))
23112315
Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].DeviceName).To(Equal("/dev/xvdb"))
23122316
Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.SnapshotId).To(Equal("snap-xxxxxxxx"))
2317+
Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeInitializationRate).To(Equal(int32(100)))
23132318
})
23142319
})
23152320
})

‎pkg/providers/launchtemplate/launchtemplate.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -368,10 +368,11 @@ func blockDeviceMappings(blockDeviceMappings []*v1.BlockDeviceMapping) []ec2type
368368
//nolint: gosec
369369
Iops: lo.EmptyableToPtr(int32(lo.FromPtr(blockDeviceMapping.EBS.IOPS))),
370370
//nolint: gosec
371-
Throughput: lo.EmptyableToPtr(int32(lo.FromPtr(blockDeviceMapping.EBS.Throughput))),
372-
KmsKeyId: blockDeviceMapping.EBS.KMSKeyID,
373-
SnapshotId: blockDeviceMapping.EBS.SnapshotID,
374-
VolumeSize: volumeSize(blockDeviceMapping.EBS.VolumeSize),
371+
Throughput: lo.EmptyableToPtr(int32(lo.FromPtr(blockDeviceMapping.EBS.Throughput))),
372+
KmsKeyId: blockDeviceMapping.EBS.KMSKeyID,
373+
SnapshotId: blockDeviceMapping.EBS.SnapshotID,
374+
VolumeInitializationRate: blockDeviceMapping.EBS.VolumeInitializationRate,
375+
VolumeSize: volumeSize(blockDeviceMapping.EBS.VolumeSize),
375376
},
376377
})
377378
}

‎website/content/en/preview/concepts/nodeclasses.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ spec:
141141
deleteOnTermination: true
142142
throughput: 125
143143
snapshotID: snap-0123456789
144+
volumeInitializationRate: 100
144145

145146
# Optional, use instance-store volumes for node ephemeral-storage
146147
instanceStorePolicy: RAID0

0 commit comments

Comments
 (0)