feat: added support for screenshots

.github/workflows/build-darwin.yml

@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [macOS-12]
-        go: [1.19]
+        go: ['1.21']
         arch: [amd64, arm64]
 
     steps:

.github/workflows/build-windows.yml

@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [windows-2019]
-        go: [1.19]
+        go: ['1.21']
 
     steps:
       - name: Check out repository code

README.md

@@ -6,3 +6,9 @@ Small framework to interact with UX element through OS accessibility native APIs
 
 goax.exe can be detected as a threat by windows defender, as it may be identified as a [Trojan:Win32/Wacatac.B!ml](https://www.makeuseof.com/windows-wacatac-trojan/)  
 this is a **false positive** from the code you can see that goax uses OS dll libraries to access OS functionality (accessibility, messaging,...).  
+
+## Required permissions
+
+### Mac 
+
+We need to setup the accessibility and screen recording for the app (if run locally we will allow terminal app if we use ssh we need the sshd-keygen-wrapper app)

cmd/app/app.go

@@ -5,7 +5,15 @@ import (
 )
 
 const (
-	appPath = "app-path"
+	appPath         = "app-path"
+	record          = "record"
+	recordDesc      = "capture screenshot"
+	recordsPath     = "records-path"
+	recordsPathDesc = "path to store the screeshots"
+)
+
+var (
+	recordsPathDefault = "."
 )
 
 func GetCmd() *cobra.Command {

cmd/app/click.go

@@ -1,11 +1,16 @@
 package app
 
 import (
+	"fmt"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 
 	"github.com/adrianriobo/goax/pkg/goax/app"
+	"github.com/adrianriobo/goax/pkg/util/delay"
+	"github.com/adrianriobo/goax/pkg/util/logging"
+	"github.com/adrianriobo/goax/pkg/util/screenshot"
 )
 
 func getClickCmd() *cobra.Command {
@@ -27,6 +32,8 @@ func getClickCmd() *cobra.Command {
 	flagSet.StringP("element-type", "t", "", "element type to be clicked")
 	flagSet.Bool("strict", false, "to force id match exactly with the element id")
 	flagSet.Int("order", 0, "in case multiple elements with same id, we can specify the order of the element within the list of elements")
+	flagSet.Bool(record, false, recordDesc)
+	flagSet.StringP(recordsPath, "", recordsPathDefault, recordsPathDesc)
 	c.Flags().AddFlagSet(flagSet)
 	c.MarkFlagRequired("element")
 	return c
@@ -37,9 +44,25 @@ func click() error {
 	if err != nil {
 		return err
 	}
-	return a.ClickWithOrder(
+	delay.Delay(delay.LONG)
+	if viper.IsSet(record) {
+		if err := screenshot.CaptureScreen(viper.GetString(recordsPath), "clickLoadForefrontApp"); err != nil {
+			logging.Errorf("error capturing the screenshot: %v", err)
+		}
+	}
+	if err := a.ClickWithOrder(
 		viper.GetString("element"),
 		viper.GetString("element-type"),
 		viper.IsSet("strict"),
-		viper.GetInt("order"))
+		viper.GetInt("order")); err != nil {
+		return err
+	}
+	delay.Delay(delay.LONG)
+	if viper.IsSet(record) {
+		sfn := fmt.Sprintf("click-%s", viper.GetString("element"))
+		if err := screenshot.CaptureScreen(viper.GetString(recordsPath), sfn); err != nil {
+			logging.Errorf("error capturing the screenshot: %v", err)
+		}
+	}
+	return nil
 }

cmd/app/open.go

@@ -6,6 +6,9 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/adrianriobo/goax/pkg/goax/app"
+	"github.com/adrianriobo/goax/pkg/util/delay"
+	"github.com/adrianriobo/goax/pkg/util/logging"
+	"github.com/adrianriobo/goax/pkg/util/screenshot"
 )
 
 func getOpenCmd() *cobra.Command {
@@ -23,11 +26,23 @@ func getOpenCmd() *cobra.Command {
 
 	flagSet := pflag.NewFlagSet("app", pflag.ExitOnError)
 	flagSet.StringP(appPath, "p", "", "path for the application to be handle")
+	flagSet.Bool(record, false, recordDesc)
+	flagSet.StringP(recordsPath, "", recordsPathDefault, recordsPathDesc)
 	c.Flags().AddFlagSet(flagSet)
-
 	return c
 }
 
 func open() error {
-	return app.Open(viper.GetString(appPath))
+	if err := app.Open(
+		viper.GetString(appPath)); err != nil {
+		return err
+	}
+	// We open remotely so we wait for a bit
+	delay.Delay(delay.LONG)
+	if viper.IsSet(record) {
+		if err := screenshot.CaptureScreen(viper.GetString(recordsPath), "openApp"); err != nil {
+			logging.Errorf("error capturing the screenshot: %v", err)
+		}
+	}
+	return nil
 }

cmd/app/setvalue.go

@@ -1,11 +1,16 @@
 package app
 
 import (
+	"fmt"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 
 	"github.com/adrianriobo/goax/pkg/goax/app"
+	"github.com/adrianriobo/goax/pkg/util/delay"
+	"github.com/adrianriobo/goax/pkg/util/logging"
+	"github.com/adrianriobo/goax/pkg/util/screenshot"
 )
 
 func getSetValueCmd() *cobra.Command {
@@ -29,6 +34,8 @@ func getSetValueCmd() *cobra.Command {
 	flagSet.Int("order", 0, "in case multiple elements with same id, we can specify the order of the element within the list of elements")
 	flagSet.Bool("focus", false, "if focus flag is added it will add the value to the current focused textbox on the screen (if any)")
 	flagSet.StringP("value", "v", "", "value to be set on the element")
+	flagSet.Bool(record, false, recordDesc)
+	flagSet.StringP(recordsPath, "", recordsPathDefault, recordsPathDesc)
 	c.Flags().AddFlagSet(flagSet)
 	c.MarkFlagRequired("value")
 	c.MarkFlagsMutuallyExclusive("focus", "element")
@@ -40,13 +47,29 @@ func setValue() error {
 	if err != nil {
 		return err
 	}
+	delay.Delay(delay.LONG)
+	if viper.IsSet(record) {
+		if err := screenshot.CaptureScreen(viper.GetString(recordsPath), "setValueLoadForefrontApp"); err != nil {
+			logging.Errorf("error capturing the screenshot: %v", err)
+		}
+	}
 	if viper.IsSet("focus") {
 		return a.SetValueOnFocus(viper.GetString("value"))
 	}
-	return a.SetValueWithOrder(
+	if err := a.SetValueWithOrder(
 		viper.GetString("element"),
 		viper.GetString("element-type"),
 		viper.IsSet("strict"),
 		viper.GetInt("order"),
-		viper.GetString("value"))
+		viper.GetString("value")); err != nil {
+		return err
+	}
+	delay.Delay(delay.LONG)
+	if viper.IsSet(record) {
+		sfn := fmt.Sprintf("setValue-%s", viper.GetString("element"))
+		if err := screenshot.CaptureScreen(viper.GetString(recordsPath), sfn); err != nil {
+			logging.Errorf("error capturing the screenshot: %v", err)
+		}
+	}
+	return nil
 }

go.mod

@@ -4,6 +4,7 @@ go 1.19
 
 require (
 	github.com/go-ole/go-ole v1.2.6
+	github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329
 	github.com/openstandia/w32uiautomation v0.0.0-20190416091226-c987b5b65d6b
 	github.com/sirupsen/logrus v1.9.2
 	github.com/spf13/cobra v1.7.0
@@ -15,8 +16,11 @@ require (
 
 require (
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jezek/xgb v0.0.0-20210312150743-0e0f116e1240 // indirect
+	github.com/lxn/win v0.0.0-20210218163916-a377121e959e // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect

go.sum

@@ -59,6 +59,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 h1:Y5Q2mEwfzjMt5+3u70Gtw93ZOu2UuPeeeTBDntF7FoY=
+github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -127,15 +129,21 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jezek/xgb v0.0.0-20210312150743-0e0f116e1240 h1:dy+DS31tGEGCsZzB45HmJJNHjur8GDgtRNX9U7HnSX4=
+github.com/jezek/xgb v0.0.0-20210312150743-0e0f116e1240/go.mod h1:3P4UH/k22rXyHIJD2w4h2XMqPX4Of/eySEZq9L6wqc4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329 h1:qq2nCpSrXrmvDGRxW0ruW9BVEV1CN2a9YDOExdt+U0o=
+github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329/go.mod h1:2VPVQDR4wO7KXHwP+DAypEy67rXf+okUx2zjgpCxZw4=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/lxn/win v0.0.0-20210218163916-a377121e959e h1:H+t6A/QJMbhCSEH5rAuRxh+CtW96g0Or0Fxa9IKr4uc=
+github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -310,6 +318,7 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

pkg/os/windows/util/toolbar/toolbar.go

@@ -11,8 +11,8 @@ import (
 
 	win32ss "github.com/adrianriobo/goax/pkg/os/windows/api/system-services"
 	win32wam "github.com/adrianriobo/goax/pkg/os/windows/api/user-interface/windows-and-messages"
-	win32windows "github.com/adrianriobo/goax/pkg/os/windows/app/ux/windows"
-	win32process "github.com/adrianriobo/goax/pkg/os/windows/services/process"
+	win32process "github.com/adrianriobo/goax/pkg/os/windows/util/services/process"
+	win32windows "github.com/adrianriobo/goax/pkg/os/windows/util/windows"
 	"github.com/adrianriobo/goax/pkg/util/logging"
 )
 

pkg/util/screenshot/screenshot.go

@@ -0,0 +1,42 @@
+package screenshot
+
+import (
+	"fmt"
+	"image"
+	"image/png"
+	"os"
+	"path/filepath"
+
+	"github.com/kbinani/screenshot"
+)
+
+var seq int = 0
+
+func CaptureScreen(outputPath, outputFilename string) error {
+	return CaptureBounds(screenshot.GetDisplayBounds(0), outputPath, outputFilename)
+}
+
+func CaptureBounds(rect image.Rectangle, outputPath, outputFilename string) error {
+	bounds := screenshot.GetDisplayBounds(0)
+	img, err := screenshot.CaptureRect(bounds)
+	if err != nil {
+		return fmt.Errorf("error capturing the screen: %v", err)
+	}
+	fileName := fmt.Sprintf("%s-%s.png", outputFilename, screenshotSequece())
+	file, err := os.Create(filepath.Join(outputPath, fileName))
+	if err != nil {
+		return fmt.Errorf("error creating the file: %v", err)
+	}
+	defer file.Close()
+	err = png.Encode(file, img)
+	if err != nil {
+		return fmt.Errorf("error encoding the capture: %v", err)
+	}
+	return nil
+}
+
+func screenshotSequece() string {
+	s := fmt.Sprintf("%d", seq)
+	seq++
+	return s
+}