e2e: Capture controller and driver pod logs

I found this helpful in recent PR work to catch exit logs before the pods go away. Committing it back to see if other folks are interested.

Signed-off-by: Mat Schaffer <[email protected]>

Author: matschaffer-roblox

.gitignore

@@ -5,3 +5,4 @@ codecov.yaml
 cover.out
 .DS_Store
 *.iml
+test/e2e/logs/

test/e2e/sparkapplication_test.go

@@ -62,6 +62,10 @@ var _ = Describe("Example SparkApplication", func() {
 			key := types.NamespacedName{Namespace: app.Namespace, Name: app.Name}
 			Expect(k8sClient.Get(ctx, key, app)).To(Succeed())
 
+			By("Writing driver logs")
+			driverPodName := util.GetDriverPodName(app)
+			writePodLogs(ctx, app.Namespace, driverPodName)
+
 			By("Deleting SparkApplication")
 			Expect(k8sClient.Delete(ctx, app)).To(Succeed())
 		})
@@ -116,6 +120,10 @@ var _ = Describe("Example SparkApplication", func() {
 			key := types.NamespacedName{Namespace: app.Namespace, Name: app.Name}
 			Expect(k8sClient.Get(ctx, key, app)).To(Succeed())
 
+			By("Writing driver logs")
+			driverPodName := util.GetDriverPodName(app)
+			writePodLogs(ctx, app.Namespace, driverPodName)
+
 			volumes := app.Spec.Volumes
 			By("Deleting SparkApplication")
 			Expect(k8sClient.Delete(ctx, app)).To(Succeed())
@@ -201,6 +209,10 @@ var _ = Describe("Example SparkApplication", func() {
 			key := types.NamespacedName{Namespace: app.Namespace, Name: app.Name}
 			Expect(k8sClient.Get(ctx, key, app)).To(Succeed())
 
+			By("Writing driver logs")
+			driverPodName := util.GetDriverPodName(app)
+			writePodLogs(ctx, app.Namespace, driverPodName)
+
 			By("Deleting SparkApplication")
 			Expect(k8sClient.Delete(ctx, app)).To(Succeed())
 		})
@@ -376,6 +388,10 @@ var _ = Describe("Example SparkApplication", func() {
 			key := types.NamespacedName{Namespace: app.Namespace, Name: app.Name}
 			Expect(k8sClient.Get(ctx, key, app)).To(Succeed())
 
+			By("Writing driver logs")
+			driverPodName := util.GetDriverPodName(app)
+			writePodLogs(ctx, app.Namespace, driverPodName)
+
 			By("Deleting SparkApplication")
 			Expect(k8sClient.Delete(ctx, app)).To(Succeed())
 		})

test/e2e/suit_test.go

@@ -62,6 +62,8 @@ const (
 	MutatingWebhookName   = "spark-operator-webhook"
 	ValidatingWebhookName = "spark-operator-webhook"
 
+	ControllerName = "spark-operator-controller"
+
 	PollInterval = 1 * time.Second
 	WaitTimeout  = 5 * time.Minute
 )
@@ -152,6 +154,9 @@ var _ = BeforeSuite(func() {
 })
 
 var _ = AfterSuite(func() {
+	By("Capturing the Spark operator logs")
+	writeControllerLogs(context.Background())
+
 	By("Uninstalling the Spark operator helm chart")
 	envSettings := cli.New()
 	envSettings.SetNamespace(ReleaseNamespace)
@@ -290,3 +295,28 @@ func collectSparkApplicationsUntilTermination(ctx context.Context, key types.Nam
 	})
 	return apps, err
 }
+
+func writePodLogs(ctx context.Context, podNamespace string, podName string) {
+	bytes, err := clientset.CoreV1().Pods(podNamespace).GetLogs(podName, &corev1.PodLogOptions{}).Do(ctx).Raw()
+	Expect(err).NotTo(HaveOccurred())
+	dir := filepath.Join("logs", podNamespace)
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		os.MkdirAll(dir, 0755)
+	}
+	err = os.WriteFile(filepath.Join(dir, fmt.Sprintf("%s.log", podName)), bytes, 0644)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func writeControllerLogs(ctx context.Context) {
+	deployment, err := clientset.AppsV1().Deployments(ReleaseNamespace).Get(ctx, ControllerName, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	listOptions := metav1.ListOptions{
+		LabelSelector: metav1.FormatLabelSelector(&metav1.LabelSelector{MatchLabels: deployment.Spec.Selector.MatchLabels}),
+	}
+	pods, err := clientset.CoreV1().Pods(ReleaseNamespace).List(ctx, listOptions)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(pods).NotTo(BeNil())
+	for _, pod := range pods.Items {
+		writePodLogs(ctx, ReleaseNamespace, pod.Name)
+	}
+}