PWX-34271: live migrate KubeVirt VMs before deleting the pod during PX rolling update (#1533)

In order to avoid downtime for KubeVirt VMs during PX rolling update, the operator now
evicts the VMs before deleting PX pod. Live migration is used for VM eviction.

PX pod is deleted only when there are no virt-launcher pods running on the node.

There is a chance that a VM could land on the node after operator has checked for the VMs
and proceeded with the deletion of PX pod. Operator adds "unschedulable" annotation on the
node to tell Stork that it should not schedule pods on the node. However, Stork changes
are needed for the annotation to take effect.

We ignore the VMs that are not live migratable. If VM eviction fails, PX upadte is paused
on the node and an event is generated.

Signed-off-by: Neelesh Thakur <[email protected]>

Author: pureneelesh

Makefile

@@ -302,6 +302,9 @@ mockgen: $(GOPATH)/bin/mockgen
 	$(GOPATH)/bin/mockgen -destination=pkg/mock/controllercache.mock.go -package=mock sigs.k8s.io/controller-runtime/pkg/cache Cache
 	$(GOPATH)/bin/mockgen -destination=pkg/mock/preflight.mock.go -package=mock github.com/libopenstorage/operator/pkg/preflight CheckerOps
 	$(GOPATH)/bin/mockgen -destination=drivers/storage/portworx/mock/manifest.mock.go -package=mock -source=./drivers/storage/portworx/manifest/manifest.go
+	$(GOPATH)/bin/mockgen -destination=pkg/mock/kubevirtmanager.mock.go -package=mock github.com/libopenstorage/operator/pkg/controller/storagecluster KubevirtManager
+	$(GOPATH)/bin/mockgen -destination=pkg/mock/mockcore/core.ops.mock.go -package=mockcore github.com/portworx/sched-ops/k8s/core Ops
+	$(GOPATH)/bin/mockgen -destination=pkg/mock/mockkubevirtdy/kubevirt-dynamic.ops.mock.go -package=mockkubevirtdy github.com/portworx/sched-ops/k8s/kubevirt-dynamic Ops
 
 clean: clean-release-manifest clean-bundle
 	@echo "Cleaning up binaries"

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/operator-framework/api v0.17.1
 	github.com/pborman/uuid v1.2.1
 	github.com/portworx/kvdb v0.0.0-20230326003017-21a38cf82d4b
-	github.com/portworx/sched-ops v1.20.4-rc1.0.20240214081459-b1e25baf33c9
+	github.com/portworx/sched-ops v1.20.4-rc1.0.20240502044614-dc8638ea15f9
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.63.0
 	github.com/prometheus-operator/prometheus-operator/pkg/client v0.46.0
 	github.com/sirupsen/logrus v1.9.3
@@ -136,8 +136,7 @@ require (
 replace (
 	github.com/kubernetes-incubator/external-storage => github.com/libopenstorage/external-storage v5.1.1-0.20190919185747-9394ee8dd536+incompatible
 	github.com/libopenstorage/openstorage => github.com/libopenstorage/openstorage v1.0.1-0.20240221210452-7757fdc2b8ff
-	github.com/portworx/sched-ops => github.com/portworx/sched-ops v1.20.4-rc1.0.20240214081459-b1e25baf33c9
-
+	github.com/portworx/sched-ops => github.com/portworx/sched-ops v1.20.4-rc1.0.20240502044614-dc8638ea15f9
 	golang.org/x/tools => golang.org/x/tools v0.1.11
 	google.golang.org/grpc => google.golang.org/grpc v1.29.1
 	google.golang.org/grpc/examples/helloworld/helloworld => google.golang.org/grpc/examples/helloworld/helloworld v1.29.1

go.sum

@@ -3863,8 +3863,8 @@ github.com/portworx/kvdb v0.0.0-20230326003017-21a38cf82d4b/go.mod h1:Q8YyrNDvPp
 github.com/portworx/px-backup-api v1.0.1-0.20200915150042-274508e876ef/go.mod h1:puy7YVXeb6glot1vVCIePIiRLSwB//+rFtN2ZjvXeEw=
 github.com/portworx/px-object-controller v0.0.0-20220804234424-40d3b8a84987/go.mod h1:g3pw2lI2AjqAixUCRhaBdKTY98znsCPR7NGRrlpimVU=
 github.com/portworx/pxc v0.33.0/go.mod h1:Tl7hf4K2CDr0XtxzM08sr9H/KsMhscjf9ydb+MnT0U4=
-github.com/portworx/sched-ops v1.20.4-rc1.0.20240214081459-b1e25baf33c9 h1:9/WeXMlhzvB4vWrh7AO2fhcSjrpbHsSwiansYHvtw3M=
-github.com/portworx/sched-ops v1.20.4-rc1.0.20240214081459-b1e25baf33c9/go.mod h1:aBipy9Jl9IS77SFZi2RqH2UT8MeKEmja1MYembOaZrY=
+github.com/portworx/sched-ops v1.20.4-rc1.0.20240502044614-dc8638ea15f9 h1:bPpp70Ll7bc1g0zDKB/af8ckNv01YmPZhHmfW4vU1HI=
+github.com/portworx/sched-ops v1.20.4-rc1.0.20240502044614-dc8638ea15f9/go.mod h1:KOe618gr1FAzVmLNe9LFHss19df3MBZ0qom6Ct7RpyM=
 github.com/portworx/talisman v0.0.0-20210302012732-8af4564777f7/go.mod h1:e8a6uFpSbOlRpZQlW9aXYogC+GWAo065G0RL9hDkD4Q=
 github.com/portworx/torpedo v0.20.4-rc1.0.20210325154352-eb81b0cdd145/go.mod h1:CkLAs/ojTzSu3SPyeDxc3qhsbRU78H5Xz1qJlj1Ap1U=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=

pkg/constants/metadata.go

@@ -15,12 +15,18 @@ const (
 	LabelValueTrue = "true"
 	// AnnotationGarbageCollection is the annotation to let operator clean up external objects on uninstallation.
 	AnnotationGarbageCollection = OperatorPrefix + "/garbage-collection"
+	// AnnotationEvictVMsDuringUpdate is the annotation to let operator evict VMs during upgrade
+	AnnotationEvictVMsDuringUpdate = OperatorPrefix + "/evict-vms-during-update"
 	// AnnotationNodeLabels is the storage pod annotation that contains node labels
 	AnnotationNodeLabels = OperatorPrefix + "/node-labels"
 	// AnnotationDisableStorage annotation to disable the storage pods from running (default: false)
 	AnnotationDisableStorage = OperatorPrefix + "/disable-storage"
 	// AnnotationReconcileObject annotation to toggle reconciliation of operator created objects
 	AnnotationReconcileObject = OperatorPrefix + "/reconcile"
+	// AnnotationControllerRevisionHashKey has the controller revision hash for which a VM live migration was triggered
+	AnnotationControllerRevisionHashKey = OperatorPrefix + "/controller-revision-hash"
+	// AnnotationVMIMigrationSourceNode has the source node for the VM live migration triggered by the operator
+	AnnotationVMIMigrationSourceNode = OperatorPrefix + "/vmi-migration-source-node"
 	// AnnotationClusterAPIMachine is the annotation key name for the name of the
 	// machine that's backing the k8s node
 	AnnotationClusterAPIMachine = "cluster.k8s.io/machine"
@@ -39,6 +45,8 @@ const (
 	// the custom registry, there is a list of hardcoded common registries, however the list
 	// may not be complete, users can use this annotation to add more.
 	AnnotationCommonImageRegistries = OperatorPrefix + "/common-image-registries"
+	// StorkAnnotationUnschedulable tells Stork to consider the node unschedulable
+	StorkAnnotationUnschedulable = "stork.libopenstorage.org/unschedulable"
 	// OperatorLabelManagedByKey is a label key that is added to any object that is
 	// managed the Portworx operator.
 	OperatorLabelManagedByKey = OperatorPrefix + "/managed-by"