feat: add a new extension: mongodb (#278)

* feat: support to jump to a specific version address

* feat: add a new extension: mongodb

* disable the git e2e test due to unstable

* fix the code style isseus

---------

Co-authored-by: rick <[email protected]>

Author: LinuxSuRen

.gitpod.yml

@@ -1,8 +1,12 @@
-# See https://www.gitpod.io/docs/configure/workspaces
+
 tasks:
   - init: make init-env install-precheck
     before: IMG_TOOL=docker GOPROXY= make build-ext copy-ext build-image
     command: cd console/atest-ui/ && npm i
 
 ports:
   - port: 5713  # console interactive port
+
+vscode:
+  extensions:
+    - golang.goteractive port

.remarkrc

@@ -0,0 +1 @@
+listItemIndent=one

CONTRIBUTION.md

@@ -50,4 +50,9 @@ docker run -p 12800:12800 -p 9412:9412 \
     -e SW_STORAGE=banyandb \
     -e SW_STORAGE_BANYANDB_HOST=192.168.1.98 \
     docker.io/apache/skywalking-oap-server
-```
+```
+
+## FAQ
+
+* Got sum missing match error of go.
+  * Run command: `go clean -modcache && go mod tidy`

Dockerfile

@@ -36,14 +36,15 @@ COPY --from=ui /workspace/dist/assets/*.css cmd/data/index.css
 
 COPY --from=sk /usr/local/bin/skywalking-go-agent /usr/local/bin/skywalking-go-agent
 
-RUN GOPROXY=${GOPROXY} go mod download
-RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -toolexec="skywalking-go-agent" -a -ldflags "-w -s -X github.com/linuxsuren/api-testing/pkg/version.version=${VERSION}\
+# RUN GOPROXY=${GOPROXY} go mod download
+RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -v -toolexec="skywalking-go-agent" -a -ldflags "-w -s -X github.com/linuxsuren/api-testing/pkg/version.version=${VERSION}\
     -X github.com/linuxsuren/api-testing/pkg/version.date=$(date +%Y-%m-%d)" -o atest .
-RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -ldflags "-w -s" -o atest-collector extensions/collector/main.go
-RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -ldflags "-w -s" -o atest-store-orm extensions/store-orm/main.go
-RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -ldflags "-w -s" -o atest-store-s3 extensions/store-s3/main.go
-RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -ldflags "-w -s" -o atest-store-etcd extensions/store-etcd/main.go
-RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -toolexec="skywalking-go-agent" -a -ldflags "-w -s" -o atest-store-git extensions/store-git/main.go
+RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -v -ldflags "-w -s" -o atest-collector extensions/collector/main.go
+RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -v -ldflags "-w -s" -o atest-store-orm extensions/store-orm/main.go
+RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -v -ldflags "-w -s" -o atest-store-s3 extensions/store-s3/main.go
+RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -v -ldflags "-w -s" -o atest-store-etcd extensions/store-etcd/main.go
+RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -v -ldflags "-w -s" -o atest-store-mongodb extensions/store-mongodb/main.go
+RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -v -toolexec="skywalking-go-agent" -a -ldflags "-w -s" -o atest-store-git extensions/store-git/main.go
 
 FROM docker.io/library/ubuntu:23.04
 
@@ -64,6 +65,7 @@ COPY --from=builder /workspace/atest-store-orm /usr/local/bin/atest-store-orm
 COPY --from=builder /workspace/atest-store-s3 /usr/local/bin/atest-store-s3
 COPY --from=builder /workspace/atest-store-etcd /usr/local/bin/atest-store-etcd
 COPY --from=builder /workspace/atest-store-git /usr/local/bin/atest-store-git
+COPY --from=builder /workspace/atest-store-mongodb /usr/local/bin/atest-store-mongodb
 COPY --from=builder /workspace/LICENSE /LICENSE
 COPY --from=builder /workspace/README.md /README.md
 

Makefile

@@ -9,16 +9,18 @@ APP_VERSION?=v0.0.13
 HELM_REPO?=docker.io/linuxsuren
 
 fmt:
+	go mod tidy
 	go fmt ./...
 	cd extensions/store-etcd && go mod tidy && go fmt ./...
 	cd extensions/store-git && go mod tidy && go fmt ./...
 	cd extensions/store-orm && go mod tidy && go fmt ./...
 	cd extensions/store-s3 && go mod tidy && go fmt ./...
+	cd extensions/store-mongodb && go mod tidy && go fmt ./...
 build:
 	mkdir -p bin
 	rm -rf bin/atest
 	CGO_ENABLED=0 go build ${TOOLEXEC} -a ${BUILD_FLAG} -o bin/${BINARY} main.go
-build-ext: build-ext-git build-ext-orm build-ext-s3 build-ext-etcd
+build-ext: build-ext-git build-ext-orm build-ext-s3 build-ext-etcd build-ext-mongodb
 build-ext-git:
 	CGO_ENABLED=0 go build -ldflags "-w -s" -o bin/atest-store-git extensions/store-git/main.go
 build-ext-orm:
@@ -27,6 +29,8 @@ build-ext-etcd:
 	CGO_ENABLED=0 go build -ldflags "-w -s" -o bin/atest-store-etcd extensions/store-etcd/main.go
 build-ext-s3:
 	CGO_ENABLED=0 go build -ldflags "-w -s" -o bin/atest-store-s3 extensions/store-s3/main.go
+build-ext-mongodb:
+	CGO_ENABLED=0 go build -ldflags "-w -s" -o bin/atest-store-mongodb extensions/store-mongodb/main.go
 build-ui:
 	cd console/atest-ui && npm i && npm run build-only
 embed-ui:

console/atest-ui/src/App.vue

@@ -293,8 +293,22 @@ const suiteKinds = [{
 }]
 
 const appVersion = ref('')
+const appVersionLink = ref('https://github.com/LinuxSuRen/api-testing')
 API.GetVersion((d) => {
   appVersion.value = d.message
+  const version = d.message.match('^v\\d*.\\d*.\\d*')
+  const dirtyVersion = d.message.match('^v\\d*.\\d*.\\d*-\\d*-g')
+
+  if (!version && !dirtyVersion) {
+    return
+  }
+
+  console.log(dirtyVersion)
+  if (dirtyVersion && dirtyVersion.length > 0) {
+    appVersionLink.value = appVersionLink.value + '/commit/' + d.message.replace(dirtyVersion[0], '')
+  } else if (version && version.length > 0) {
+    appVersionLink.value = appVersionLink.value + '/releases/tag/' + version[0]
+  }
 })
 </script>
 
@@ -363,7 +377,7 @@ API.GetVersion((d) => {
         </el-container>
       </el-main>
     <div style="position: absolute; bottom: 0px; right: 10px;">
-      <a href="https://github.com/LinuxSuRen/api-testing" target="_blank" rel="noopener">{{appVersion}}</a>
+      <a :href=appVersionLink target="_blank" rel="noopener">{{appVersion}}</a>
     </div>
     <TemplateFunctions/>
     </el-container>

console/atest-ui/src/views/StoreManager.vue

@@ -50,11 +50,15 @@ interface Store {
   properties: Pair[]
 }
 
+const storesLoading = ref(false)
 function loadStores() {
+  storesLoading.value = true
   API.GetStores((e) => {
     stores.value = e.data
   }, (e) => {
     ElMessage.error('Oops, ' + e)
+  }, () => {
+    storesLoading.value = false
   })
 }
 loadStores()
@@ -148,7 +152,7 @@ const submitForm = async (formEl: FormInstance | undefined) => {
 function storeVerify(formEl: FormInstance | undefined) {
   if (!formEl) return
   
-  API.VerifyStore(setStoreForm.name, (e) => {
+  API.VerifyStore(storeForm.name, (e) => {
     if (e.ready) {
       ElMessage({
         message: 'Verified!',
@@ -180,7 +184,7 @@ function updateKeys() {
         <el-button type="primary" @click="addStore" :icon="Edit">{{t('button.new')}}</el-button>
         <el-button type="primary" @click="loadStores">{{t('button.refresh')}}</el-button>
     </div>
-    <el-table :data="stores" style="width: 100%">
+    <el-table :data="stores" style="width: 100%" v-loading=storesLoading>
       <el-table-column :label="t('field.name')" width="180">
         <template #default="scope">
           <el-input v-model="scope.row.name" placeholder="Name"/>

console/atest-ui/src/views/net.ts

@@ -36,7 +36,7 @@ interface AppVersion {
   message: string
 }
 
-function GetVersion(callback: (v: AppVersion) => {}) {
+function GetVersion(callback: (v: AppVersion) => void) {
   const requestOptions = {
     method: 'POST',
   }
@@ -90,7 +90,7 @@ function UpdateTestSuite(suite: any,
 }
 
 function GetTestSuite(name: string, callback: () => {},
-  errHandle: (e: any) => {}) {
+  errHandle: (e: any) => void) {
   const store = Cache.GetCurrentStore()
   const requestOptions = {
     method: 'POST',
@@ -107,7 +107,7 @@ function GetTestSuite(name: string, callback: () => {},
 }
 
 function DeleteTestSuite(name: string,
-  callback: (d: any) => {}, errHandle?: (e: any) => {} | null) {
+  callback: (d: any) => void, errHandle?: (e: any) => void | null) {
   const requestOptions = {
     method: 'POST',
     headers: {
@@ -123,7 +123,7 @@ function DeleteTestSuite(name: string,
 }
 
 function ConvertTestSuite(suiteName: string, genertor: string,
-  callback: (d: any) => {}, errHandle?: (e: any) => {} | null) {
+  callback: (d: any) => void, errHandle?: (e: any) => void | null) {
   const requestOptions = {
     method: 'POST',
     headers: {
@@ -163,7 +163,7 @@ interface TestCase {
 }
 
 function CreateTestCase(testcase: TestCase,
-  callback: () => {}, errHandle?: (e: any) => {} | null) {
+  callback: () => {}, errHandle?: (e: any) => void | null) {
   const requestOptions = {
     method: 'POST',
     headers: {
@@ -187,7 +187,7 @@ function CreateTestCase(testcase: TestCase,
 }
 
 function UpdateTestCase(testcase: any,
-  callback: () => {}, errHandle?: (e: any) => {} | null) {
+  callback: () => {}, errHandle?: (e: any) => void | null) {
     const requestOptions = {
       method: 'POST',
       headers: {
@@ -201,7 +201,7 @@ function UpdateTestCase(testcase: any,
 }
 
 function GetTestCase(req: TestCase,
-  callback: (d: any) => {}, errHandle?: (e: any) => {} | null) {
+  callback: (d: any) => void, errHandle?: (e: any) => void | null) {
   const requestOptions = {
     method: 'POST',
     headers: {
@@ -218,7 +218,7 @@ function GetTestCase(req: TestCase,
 }
 
 function ListTestCase(suite: string, store: string,
-  callback: (d: any) => {}, errHandle?: (e: any) => {} | null) {
+  callback: (d: any) => void, errHandle?: (e: any) => void | null) {
   const requestOptions = {
     method: 'POST',
     headers: {
@@ -234,7 +234,7 @@ function ListTestCase(suite: string, store: string,
 }
 
 function DeleteTestCase(testcase: TestCase,
-  callback: (d: any) => {}, errHandle?: (e: any) => {} | null) {
+  callback: (d: any) => void, errHandle?: (e: any) => void | null) {
     const requestOptions = {
       method: 'POST',
       headers: {
@@ -256,7 +256,7 @@ interface RunTestCaseRequest {
 }
 
 function RunTestCase(request: RunTestCaseRequest,
-  callback: (d: any) => {}, errHandle?: (e: any) => {} | null) {
+  callback: (d: any) => void, errHandle?: (e: any) => void | null) {
   const requestOptions = {
     method: 'POST',
     headers: {
@@ -280,7 +280,7 @@ interface GenerateRequest {
 }
 
 function GenerateCode(request: GenerateRequest,
-  callback: (d: any) => {}, errHandle?: (e: any) => {} | null) {
+  callback: (d: any) => void, errHandle?: (e: any) => void | null) {
   const requestOptions = {
     method: 'POST',
     headers: {
@@ -297,14 +297,14 @@ function GenerateCode(request: GenerateRequest,
     .then(callback).catch(errHandle)
 }
 
-function ListCodeGenerator(callback: (d: any) => {}, errHandle?: (e: any) => {} | null) {
+function ListCodeGenerator(callback: (d: any) => void, errHandle?: (e: any) => void | null) {
   fetch('/server.Runner/ListCodeGenerator', {
     method: 'POST'
   }).then(DefaultResponseProcess)
   .then(callback).catch(errHandle)
 }
 
-function PopularHeaders(callback: (d: any) => {}, errHandle?: (e: any) => {} | null) {
+function PopularHeaders(callback: (d: any) => void, errHandle?: (e: any) => void | null) {
   const requestOptions = {
     method: 'POST',
     headers: {
@@ -316,17 +316,18 @@ function PopularHeaders(callback: (d: any) => {}, errHandle?: (e: any) => {} | n
     .then(callback).catch(errHandle)
 }
 
-function GetStores(callback: (d: any) => {}, errHandle?: (e: any) => {} | null) {
+function GetStores(callback: (d: any) => void,
+  errHandle?: (e: any) => void | null, final?: () => void | null) {
   const requestOptions = {
     method: 'POST',
   }
   fetch('/server.Runner/GetStores', requestOptions)
-  .then(DefaultResponseProcess)
-    .then(callback).catch(errHandle)
+    .then(DefaultResponseProcess)
+    .then(callback).catch(errHandle).finally(final)
 }
 
 function DeleteStore(name: string,
-  callback: (d: any) => {}, errHandle?: (e: any) => {} | null) {
+  callback: (d: any) => void, errHandle?: (e: any) => void | null) {
   const requestOptions = {
     method: 'POST',
     body: JSON.stringify({
@@ -339,7 +340,7 @@ function DeleteStore(name: string,
 }
 
 function VerifyStore(name: string,
-  callback: (d: any) => {}, errHandle?: (e: any) => {} | null) {
+  callback: (d: any) => void, errHandle?: (e: any) => void | null) {
   const requestOptions = {
     method: 'POST',
     body: JSON.stringify({

console/atest-ui/src/views/store.ts

@@ -17,6 +17,10 @@ export function SupportedExtensions() {
         {
           value: 'atest-store-etcd',
           key: 'atest-store-etcd'
+        },
+        {
+          value: 'atest-store-mongodb',
+          key: 'atest-store-mongodb'
         }
     ] as Pair[]
 }

docs/README.md

@@ -160,9 +160,10 @@ There are multiple storage backends supported. See the status from the list:
 |---|---|
 | Local Storage | Ready |
 | S3 | Ready |
-| ORM DataBase | Developing |
+| ORM DataBase | Ready |
 | Git Repository | Ready |
-| Etcd DataBase | Ready |
+| Etcd | Ready |
+| MongoDB | Devloping |
 
 ### Local Storage
 Local storage is the built-in solution. You can run it with the following command:
@@ -271,6 +272,23 @@ Have a look at the expected configuration below:
     insecure: false               # whether to use insecure
 ```
 
+### MongoDB Storage
+You can use a MongoDB as the storage backend.
+
+Have a look at the expected configuration below:
+
+```yaml
+- name: mongodb
+  url: 172.11.0.13:27017   # address of the mongodb
+  username: linuxsuren
+  password: linuxsuren
+  kind:
+    name: atest-store-mongodb     # the extension binary file name
+  properties:                     # optional properties for specific features
+    database: testing             # the database name
+    collection: atest             # the collection name
+```
+
 ## Secret Server
 You can put sensitive information into a secret server. For example, [Vault](https://www.github.com/hashicorp/vault).
 

e2e/compose.yaml

@@ -13,6 +13,8 @@ services:
         condition: service_healthy
       mariadb:
         condition: service_healthy
+      mongo:
+        condition: service_healthy
       # postgres:
       #   condition: service_healthy
       # clickhouse:
@@ -23,6 +25,7 @@ services:
       - etcd
       - mysql
       - mariadb
+      - mongo
       # - minio
       # - postgres
       # - clickhouse
@@ -68,6 +71,17 @@ services:
       MINIO_ROOT_PASSWORD: root
       MINIO_SERVER_HOST: minio
       MINIO_DEFAULT_BUCKETS: bucket
+  mongo:
+    image: mongo
+    healthcheck:
+      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/27017"]
+      interval: 3s
+      timeout: 60s
+      retries: 10
+      start_period: 3s
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: root
   # postgres:
   #   image: postgres:16.0
   #   environment: