add vm graceful shutdown and machine terminating state scenarion in d…

…elete machine test
ironcore-dev · Feb 19, 2024 · 58e50ca · 58e50ca
1 parent 73fe943
commit 58e50ca
Show file tree

Hide file tree

Showing 2 changed files with 138 additions and 10 deletions.
diff --git a/provider/server/machine_delete_test.go b/provider/server/machine_delete_test.go
@@ -5,6 +5,7 @@ package server_test
 
 import (
 	"path/filepath"
+	"time"
 
 	"github.com/digitalocean/go-libvirt"
 	iri "github.com/ironcore-dev/ironcore/iri/apis/machine/v1alpha1"
@@ -17,8 +18,122 @@ import (
 // TODO: This test will require update after merge of PR: #101
 var _ = Describe("DeleteMachine", func() {
 
-	It("should delete a machine", func(ctx SpecContext) {
-		By("creating a machine")
+	It("should delete a machine with graceful shutdown", func(ctx SpecContext) {
+		By("creating a machine with ceph volume which boots properly ")
+		createResp, err := machineClient.CreateMachine(ctx, &iri.CreateMachineRequest{
+			Machine: &iri.Machine{
+				Metadata: &irimeta.ObjectMetadata{
+					Labels: map[string]string{
+						"foo": "bar",
+					},
+				},
+				Spec: &iri.MachineSpec{
+					Power: iri.Power_POWER_ON,
+					Image: &iri.ImageSpec{
+						Image: osImage,
+					},
+					Class: machineClassx3xlarge,
+					Volumes: []*iri.Volume{
+						{
+							Name: "disk-1",
+							EmptyDisk: &iri.EmptyDisk{
+								SizeBytes: emptyDiskSize,
+							},
+							Device: "oda",
+						},
+						{
+							Name:   "volume-1",
+							Device: "odc",
+							Connection: &iri.VolumeConnection{
+								Driver: "ceph",
+								Handle: "dummy",
+								Attributes: map[string]string{
+									"image":    cephImage,
+									"monitors": cephMonitors,
+								},
+								SecretData: map[string][]byte{
+									"userID":  []byte(cephUsername),
+									"userKey": []byte(cephUserkey),
+								},
+							},
+						},
+					},
+				},
+			},
+		})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(createResp).NotTo(BeNil())
+
+		By("ensuring domain and domain XML is created for machine")
+		var domain libvirt.Domain
+		Eventually(func() error {
+			domain, err = libvirtConn.DomainLookupByUUID(libvirtutils.UUIDStringToBytes(createResp.Machine.Metadata.Id))
+			return err
+		}).Should(Succeed())
+		domainXMLData, err := libvirtConn.DomainGetXMLDesc(domain, 0)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(domainXMLData).NotTo(BeEmpty())
+
+		By("ensuring domain for machine is in running state")
+		Eventually(func() libvirt.DomainState {
+			domainState, _, err := libvirtConn.DomainGetState(domain, 0)
+			Expect(err).NotTo(HaveOccurred())
+			return libvirt.DomainState(domainState)
+		}).Should(Equal(libvirt.DomainRunning))
+
+		By("ensuring machine is in running state")
+		Eventually(func() iri.MachineState {
+			listResp, err := machineClient.ListMachines(ctx, &iri.ListMachinesRequest{
+				Filter: &iri.MachineFilter{
+					Id: createResp.Machine.Metadata.Id,
+				},
+			})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(listResp.Machines).NotTo(BeEmpty())
+			return listResp.Machines[0].Status.State
+		}).Should(Equal(iri.MachineState_MACHINE_RUNNING))
+
+		By("deleting the machine")
+		_, err = machineClient.DeleteMachine(ctx, &iri.DeleteMachineRequest{
+			MachineId: createResp.Machine.Metadata.Id,
+		})
+		Expect(err).NotTo(HaveOccurred())
+
+		By("ensuring machine is in Terminating state after delete")
+		Eventually(func() iri.MachineState {
+			listResp, err := machineClient.ListMachines(ctx, &iri.ListMachinesRequest{
+				Filter: &iri.MachineFilter{
+					Id: createResp.Machine.Metadata.Id,
+				},
+			})
+			Expect(err).NotTo(HaveOccurred())
+			return listResp.Machines[0].Status.State
+		}).Should(Equal(iri.MachineState_MACHINE_TERMINATING))
+
+		By("ensuring machine is gracefully shutdown")
+		Eventually(func() int {
+			listResp, err := machineClient.ListMachines(ctx, &iri.ListMachinesRequest{
+				Filter: &iri.MachineFilter{
+					Id: createResp.Machine.Metadata.Id,
+				},
+			})
+			Expect(err).NotTo(HaveOccurred())
+			return len(listResp.Machines)
+		}).WithTimeout(gracefulShutdownTimeout).WithPolling(1 * time.Second).Should(BeZero())
+
+		By("ensuring domain and domain XML is deleted for machine")
+		domain, err = libvirtConn.DomainLookupByUUID(libvirtutils.UUIDStringToBytes(createResp.Machine.Metadata.Id))
+		Expect(libvirt.IsNotFound(err)).Should(BeTrue())
+		domainXMLData, err = libvirtConn.DomainGetXMLDesc(domain, 0)
+		Expect(domainXMLData).To(BeEmpty())
+
+		By("ensuring the respective machine's file is cleaned from machines directory")
+		machineFile := filepath.Join(tempDir, "libvirt-provider", "machines", createResp.Machine.Metadata.Id)
+		Expect(machineFile).NotTo(BeAnExistingFile())
+	})
+
+	It("should delete a machine without graceful shutdown", func(ctx SpecContext) {
+		By("creating a machine which may not boot properly")
 		createResp, err := machineClient.CreateMachine(ctx, &iri.CreateMachineRequest{
 			Machine: &iri.Machine{
 				Metadata: &irimeta.ObjectMetadata{
@@ -80,6 +195,18 @@ var _ = Describe("DeleteMachine", func() {
 		})
 		Expect(err).NotTo(HaveOccurred())
 
+		By("ensuring machine is in Terminating state after delete")
+		Eventually(func() iri.MachineState {
+			listResp, err := machineClient.ListMachines(ctx, &iri.ListMachinesRequest{
+				Filter: &iri.MachineFilter{
+					Id: createResp.Machine.Metadata.Id,
+				},
+			})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(listResp.Machines).Should(HaveLen(1))
+			return listResp.Machines[0].Status.State
+		}).WithTimeout(gracefulShutdownTimeout).WithPolling(1 * time.Second).Should(Equal(iri.MachineState_MACHINE_TERMINATING))
+
 		By("ensuring machine is deleted")
 		Eventually(func() int {
 			listResp, err := machineClient.ListMachines(ctx, &iri.ListMachinesRequest{

diff --git a/provider/server/server_suite_test.go b/provider/server/server_suite_test.go
@@ -28,13 +28,14 @@ import (
 )
 
 const (
-	eventuallyTimeout    = 30 * time.Second
-	pollingInterval      = 50 * time.Millisecond
-	consistentlyDuration = 1 * time.Second
-	machineClassx3xlarge = "x3-xlarge"
-	machineClassx2medium = "x2-medium"
-	baseURL              = "http://localhost:8080"
-	streamingAddress     = "127.0.0.1:20251"
+	eventuallyTimeout       = 30 * time.Second
+	pollingInterval         = 50 * time.Millisecond
+	gracefulShutdownTimeout = 20 * time.Second
+	consistentlyDuration    = 1 * time.Second
+	machineClassx3xlarge    = "x3-xlarge"
+	machineClassx2medium    = "x2-medium"
+	baseURL                 = "http://localhost:8080"
+	streamingAddress        = "127.0.0.1:20251"
 )
 
 var (
@@ -108,7 +109,7 @@ var _ = BeforeSuite(func() {
 		},
 		NicPlugin:                      pluginOpts,
 		ResyncIntervalMachineState:     10 * time.Second,
-		GCVMGracefulShutdownTimeout:    10 * time.Second,
+		GCVMGracefulShutdownTimeout:    gracefulShutdownTimeout,
 		ResyncIntervalGarbageCollector: 5 * time.Second,
 	}