Merge pull request #5 from devtron-labs/webhook_integration

feat: Webhook integration in CI

Author: nishant-d

App.go

@@ -18,10 +18,10 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"github.com/devtron-labs/git-sensor/api"
 	"github.com/devtron-labs/git-sensor/internal/middleware"
 	"github.com/devtron-labs/git-sensor/pkg/git"
-	"fmt"
 	"github.com/go-pg/pg"
 	"github.com/gorilla/handlers"
 	"github.com/nats-io/stan"

Gopkg.lock

@@ -74,4 +74,7 @@
 
 [[constraint]]
   name = "github.com/prometheus/client_golang"
-  version = "1.1.0"
+  version = "1.1.0"
+[[constraint]]
+  name = "github.com/tidwall/gjson"
+  version = "1.8.0"

Gopkg.toml

@@ -17,10 +17,10 @@
 package api
 
 import (
+	"encoding/json"
 	"github.com/devtron-labs/git-sensor/internal/sql"
 	"github.com/devtron-labs/git-sensor/pkg"
 	"github.com/devtron-labs/git-sensor/pkg/git"
-	"encoding/json"
 	"github.com/gorilla/mux"
 	"go.uber.org/zap"
 	"net/http"
@@ -40,16 +40,20 @@ type RestHandler interface {
 	GetChangesInRelease(w http.ResponseWriter, r *http.Request)
 	GetCommitInfoForTag(w http.ResponseWriter, r *http.Request)
 	RefreshGitMaterial(w http.ResponseWriter, r *http.Request)
+	GetWebhookData(w http.ResponseWriter, r *http.Request)
+	GetAllWebhookEventConfigForHost(w http.ResponseWriter, r *http.Request)
+	GetWebhookEventConfig(w http.ResponseWriter, r *http.Request)
 }
 
 func NewRestHandlerImpl(repositoryManager pkg.RepoManager, logger *zap.SugaredLogger) *RestHandlerImpl {
 	return &RestHandlerImpl{repositoryManager: repositoryManager, logger: logger}
 }
 
 type RestHandlerImpl struct {
-	repositoryManager pkg.RepoManager
-	logger            *zap.SugaredLogger
+	repositoryManager   pkg.RepoManager
+	logger              *zap.SugaredLogger
 }
+
 type Response struct {
 	Code   int         `json:"code,omitempty"`
 	Status string      `json:"status,omitempty"`
@@ -235,6 +239,8 @@ func (handler RestHandlerImpl) GetCommitMetadata(w http.ResponseWriter, r *http.
 	var commits *git.GitCommit
 	if len(material.GitTag) > 0 {
 		commits, err = handler.repositoryManager.GetCommitInfoForTag(material)
+	} else if len(material.BranchName) > 0 {
+		commits, err = handler.repositoryManager.GetLatestCommitForBranch(material.PipelineMaterialId, material.BranchName)
 	} else {
 		commits, err = handler.repositoryManager.GetCommitMetadata(material.PipelineMaterialId, material.GitHash)
 	}
@@ -299,3 +305,64 @@ func (handler RestHandlerImpl) RefreshGitMaterial(w http.ResponseWriter, r *http
 		handler.writeJsonResp(w, err, resp, http.StatusOK)
 	}
 }
+
+func (handler RestHandlerImpl) GetWebhookData(w http.ResponseWriter, r *http.Request) {
+	handler.logger.Debug("GetWebhookData API call")
+	decoder := json.NewDecoder(r.Body)
+	request := &git.WebhookDataRequest{}
+	err := decoder.Decode(request)
+	if err != nil {
+		handler.logger.Errorw("error in decoding request of GetWebhookData ", "err", err)
+		handler.writeJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+	handler.logger.Debugw("webhook data request ", "req", request)
+	webhookData, err := handler.repositoryManager.GetWebhookDataById(request.Id)
+
+	if err != nil {
+		handler.writeJsonResp(w, err, nil, http.StatusInternalServerError)
+	} else {
+		handler.writeJsonResp(w, err, webhookData, http.StatusOK)
+	}
+}
+
+func (handler RestHandlerImpl) GetAllWebhookEventConfigForHost(w http.ResponseWriter, r *http.Request) {
+	handler.logger.Debug("GetAllWebhookEventConfigForHost API call")
+	decoder := json.NewDecoder(r.Body)
+	request := &git.WebhookEventConfigRequest{}
+	err := decoder.Decode(request)
+	if err != nil {
+		handler.logger.Errorw("error in decoding request of GetAllWebhookEventConfigForHost ", "err", err)
+		handler.writeJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+	handler.logger.Infow("webhook event config request ", "req", request)
+
+	webhookEventConfigArr, err := handler.repositoryManager.GetAllWebhookEventConfigForHost(request.GitHostId)
+	if err != nil {
+		handler.writeJsonResp(w, err, nil, http.StatusInternalServerError)
+	} else {
+		handler.writeJsonResp(w, err, webhookEventConfigArr, http.StatusOK)
+	}
+}
+
+func (handler RestHandlerImpl) GetWebhookEventConfig(w http.ResponseWriter, r *http.Request) {
+	handler.logger.Debug("GetWebhookEventConfig API call")
+	decoder := json.NewDecoder(r.Body)
+	request := &git.WebhookEventConfigRequest{}
+	err := decoder.Decode(request)
+	if err != nil {
+		handler.logger.Errorw("error in decoding request of GetWebhookEventConfig ", "err", err)
+		handler.writeJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
+	handler.logger.Infow("webhook event config request ", "req", request)
+
+	webhookEventConfig, err := handler.repositoryManager.GetWebhookEventConfig(request.EventId)
+	if err != nil {
+		handler.writeJsonResp(w, err, nil, http.StatusInternalServerError)
+	} else {
+		handler.writeJsonResp(w, err, webhookEventConfig, http.StatusOK)
+	}
+
+}

api/RestHandler.go

@@ -17,12 +17,12 @@
 package api
 
 import (
-	"github.com/devtron-labs/git-sensor/util"
 	"encoding/json"
+	"github.com/devtron-labs/git-sensor/util"
 	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.uber.org/zap"
 	"net/http"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 type MuxRouter struct {
@@ -70,4 +70,8 @@ func (r MuxRouter) Init() {
 	r.Router.Path("/admin/reload/{materialId}").HandlerFunc(r.restHandler.ReloadMaterial).Methods("POST")
 
 	r.Router.Path("/release/changes").HandlerFunc(r.restHandler.GetChangesInRelease).Methods("POST")
+
+	r.Router.Path("/webhook/data").HandlerFunc(r.restHandler.GetWebhookData).Methods("GET")
+	r.Router.Path("/webhook/host/events").HandlerFunc(r.restHandler.GetAllWebhookEventConfigForHost).Methods("GET")
+	r.Router.Path("/webhook/host/event").HandlerFunc(r.restHandler.GetWebhookEventConfig).Methods("GET")
 }

api/Router.go

@@ -26,6 +26,7 @@ import (
 const (
 	NEW_CI_MATERIAL_TOPIC = "GIT-SENSOR.NEW-CI-MATERIAL" //{publisher-app-name}-{topic-name}
 	POLL_CI_TOPIC ="GIT-SENSOR.PULL"
+	WEBHOOK_EVENT_TOPIC = "ORCHESTRATOR.WEBHOOK_EVENT"
 )
 
 type PubSubConfig struct {

internal/NatsClient.go

@@ -38,6 +38,7 @@ type CiPipelineMaterial struct {
 	ErrorMsg      string     `sql:"error_msg,notnull"`
 }
 
+
 type CiPipelineMaterialRepository interface {
 	FindByGitMaterialId(gitMaterialId int) ([]*CiPipelineMaterial, error)
 	Update(material []*CiPipelineMaterial) error

internal/sql/CiPipelineMaterial.go

@@ -28,7 +28,7 @@ const (
 	SOURCE_TYPE_BRANCH_FIXED SourceType = "SOURCE_TYPE_BRANCH_FIXED"
 	SOURCE_TYPE_BRANCH_REGEX SourceType = "SOURCE_TYPE_BRANCH_REGEX"
 	SOURCE_TYPE_TAG_ANY      SourceType = "SOURCE_TYPE_TAG_ANY"
-	SOURCE_TYPE_TAG_REGEX    SourceType = "SOURCE_TYPE_TAG_REGEX"
+	SOURCE_TYPE_WEBHOOK 	 SourceType = "WEBHOOK"
 )
 
 //TODO: add support for submodule
@@ -57,6 +57,7 @@ type MaterialRepository interface {
 	Save(material *GitMaterial) error
 	FindActive() ([]*GitMaterial, error)
 	FindAll() ([]*GitMaterial, error)
+	FindAllActiveByUrls(urls []string) ([]*GitMaterial, error)
 }
 type MaterialRepositoryImpl struct {
 	dbConnection *pg.DB
@@ -108,3 +109,15 @@ func (repo MaterialRepositoryImpl) FindById(id int) (*GitMaterial, error) {
 		Select()
 	return &material, err
 }
+
+func (repo MaterialRepositoryImpl) FindAllActiveByUrls(urls[] string) ([]*GitMaterial, error) {
+	var materials []*GitMaterial
+	err := repo.dbConnection.Model(&materials).
+		Relation("CiPipelineMaterials", func(q *orm.Query) (*orm.Query, error) {
+			return q.Where("active IS TRUE"), nil
+		}).
+		Where("deleted =? ", false).
+		Where("url in (?) ", pg.In(urls)).
+		Select()
+	return materials, err
+}

internal/sql/GitMaterial.go

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2020 Devtron Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package sql
+
+import (
+	"github.com/devtron-labs/git-sensor/util"
+	"github.com/go-pg/pg"
+)
+
+type CiPipelineMaterialWebhookDataMapping struct {
+	tableName            struct{} `sql:"ci_pipeline_material_webhook_data_mapping"`
+	Id                   int      `sql:"id,pk"`
+	CiPipelineMaterialId int      `sql:"ci_pipeline_material_id"`
+	WebhookDataId        int      `sql:"webhook_data_id"`
+	ConditionMatched     bool     `sql:"condition_matched,notnull"`
+}
+
+type WebhookEventDataMappingRepository interface {
+	GetCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialId int, webhookParsedDataId int) (*CiPipelineMaterialWebhookDataMapping, error)
+	SaveCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialWebhookDataMapping *CiPipelineMaterialWebhookDataMapping) error
+	UpdateCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialWebhookDataMapping *CiPipelineMaterialWebhookDataMapping) error
+	GetCiPipelineMaterialWebhookDataMappingForPipelineMaterial(ciPipelineMaterialId int) ([]*CiPipelineMaterialWebhookDataMapping, error)
+}
+
+type WebhookEventDataMappingRepositoryImpl struct {
+	dbConnection *pg.DB
+}
+
+func NewWebhookEventDataMappingRepositoryImpl(dbConnection *pg.DB) *WebhookEventDataMappingRepositoryImpl {
+	return &WebhookEventDataMappingRepositoryImpl{dbConnection: dbConnection}
+}
+
+func (impl WebhookEventDataMappingRepositoryImpl) GetCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialId int, webhookParsedDataId int) (*CiPipelineMaterialWebhookDataMapping, error) {
+	var mapping CiPipelineMaterialWebhookDataMapping
+	err := impl.dbConnection.Model(&mapping).
+		Where("ci_pipeline_material_id =? ", ciPipelineMaterialId).
+		Where("webhook_data_id =? ", webhookParsedDataId).
+		Select()
+
+	if err != nil {
+		if util.IsErrNoRows(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return &mapping, nil
+}
+
+func (impl WebhookEventDataMappingRepositoryImpl) SaveCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialWebhookDataMapping *CiPipelineMaterialWebhookDataMapping) error {
+	_, err := impl.dbConnection.Model(ciPipelineMaterialWebhookDataMapping).Insert()
+	return err
+}
+
+func (impl WebhookEventDataMappingRepositoryImpl) UpdateCiPipelineMaterialWebhookDataMapping(ciPipelineMaterialWebhookDataMapping *CiPipelineMaterialWebhookDataMapping) error {
+	_, err := impl.dbConnection.Model(ciPipelineMaterialWebhookDataMapping).WherePK().Update()
+	return err
+}
+
+func (impl WebhookEventDataMappingRepositoryImpl) GetCiPipelineMaterialWebhookDataMappingForPipelineMaterial(ciPipelineMaterialId int) ([]*CiPipelineMaterialWebhookDataMapping, error) {
+	var pipelineMaterials []*CiPipelineMaterialWebhookDataMapping
+	err := impl.dbConnection.Model(&pipelineMaterials).
+		Where("ci_pipeline_material_id =? ", ciPipelineMaterialId).
+		Select()
+
+	if err != nil {
+		if util.IsErrNoRows(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return pipelineMaterials, nil
+}