Skip to content

Commit 9192fd7

Browse files
Ash-expAsh-exp
authored
fix: reload all API context handling and added range support for git material ids (#109)
* fix: reload all API context handling and added range support for git material ids * vendor files for gorilla/schema * fix: FindInRage query
1 parent b832ab4 commit 9192fd7

File tree

17 files changed

+1533
-16
lines changed

17 files changed

+1533
-16
lines changed

‎api/GrpcHandler.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -464,8 +464,8 @@ func (impl *GrpcHandlerImpl) RefreshGitMaterial(ctx context.Context, req *pb.Ref
464464

465465
func (impl *GrpcHandlerImpl) ReloadAllMaterial(ctx context.Context, req *pb.Empty) (*pb.Empty, error) {
466466
gitCtx := git.BuildGitContext(ctx)
467-
impl.repositoryManager.ReloadAllRepo(gitCtx)
468-
return &pb.Empty{}, nil
467+
err := impl.repositoryManager.ReloadAllRepo(gitCtx, nil)
468+
return &pb.Empty{}, err
469469
}
470470

471471
func (impl *GrpcHandlerImpl) ReloadMaterial(ctx context.Context, req *pb.ReloadMaterialRequest) (

‎api/RestHandler.go

+26-3
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,13 @@ package api
1818

1919
import (
2020
"encoding/json"
21+
"fmt"
22+
"github.com/devtron-labs/git-sensor/bean"
2123
"github.com/devtron-labs/git-sensor/internals/sql"
2224
"github.com/devtron-labs/git-sensor/pkg"
2325
"github.com/devtron-labs/git-sensor/pkg/git"
2426
"github.com/gorilla/mux"
27+
"github.com/gorilla/schema"
2528
"go.uber.org/zap"
2629
"net/http"
2730
"strconv"
@@ -179,10 +182,30 @@ func (handler RestHandlerImpl) SavePipelineMaterial(w http.ResponseWriter, r *ht
179182
}
180183

181184
func (handler RestHandlerImpl) ReloadAllMaterial(w http.ResponseWriter, r *http.Request) {
182-
handler.logger.Infow("reload all pipelineMaterial request")
185+
handler.logger.Infow(bean.GetReloadAllLog("received reload all pipelineMaterial request"))
186+
decoder := schema.NewDecoder()
187+
decoder.IgnoreUnknownKeys(true)
188+
req := &bean.ReloadAllMaterialQuery{}
189+
err := decoder.Decode(req, r.URL.Query())
190+
if err != nil {
191+
handler.logger.Errorw(bean.GetReloadAllLog("invalid query params, ReloadAllMaterial"), "err", err, "query", r.URL.Query())
192+
handler.writeJsonResp(w, err, nil, http.StatusBadRequest)
193+
return
194+
}
195+
if req.Start < 0 || req.End < 0 || req.Start > req.End {
196+
handler.logger.Errorw(bean.GetReloadAllLog("invalid request, ReloadAllMaterial"), "query", r.URL.Query())
197+
handler.writeJsonResp(w, fmt.Errorf("invalid query params"), nil, http.StatusBadRequest)
198+
return
199+
}
183200
gitCtx := git.BuildGitContext(r.Context())
184-
handler.repositoryManager.ReloadAllRepo(gitCtx)
185-
handler.writeJsonResp(w, nil, "reloaded se logs for detail", http.StatusOK)
201+
err = handler.repositoryManager.ReloadAllRepo(gitCtx, req)
202+
if err != nil {
203+
handler.logger.Errorw(bean.GetReloadAllLog("error in request, ReloadAllMaterial"), "query", r.URL.Query(), "err", err)
204+
handler.writeJsonResp(w, err, nil, http.StatusInternalServerError)
205+
return
206+
}
207+
handler.logger.Infow(bean.GetReloadAllLog("Reloaded materials successfully!"), "query", r.URL.Query())
208+
handler.writeJsonResp(w, nil, "Reloaded materials successfully!", http.StatusOK)
186209
}
187210

188211
func (handler RestHandlerImpl) ReloadMaterials(w http.ResponseWriter, r *http.Request) {

‎bean/bean.go

+13
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,20 @@
1616

1717
package bean
1818

19+
import "fmt"
20+
1921
type StartupConfig struct {
2022
RestPort int `env:"SERVER_REST_PORT" envDefault:"8080"`
2123
GrpcPort int `env:"SERVER_GRPC_PORT" envDefault:"8081"`
2224
}
25+
26+
const ReloadAllLogPrefix = "RELOAD_ALL_LOG"
27+
28+
func GetReloadAllLog(message string) string {
29+
return fmt.Sprintf("%s: %s", ReloadAllLogPrefix, message)
30+
}
31+
32+
type ReloadAllMaterialQuery struct {
33+
Start int `json:"start"`
34+
End int `json:"end"`
35+
}

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ require (
1212
github.com/google/wire v0.6.0
1313
github.com/gorilla/handlers v1.4.2
1414
github.com/gorilla/mux v1.8.0
15+
github.com/gorilla/schema v1.1.0
1516
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1
1617
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
1718
github.com/prometheus/client_golang v1.16.0

‎go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YAR
6464
github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
6565
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
6666
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
67+
github.com/gorilla/schema v1.1.0 h1:CamqUDOFUBqzrvxuz2vEwo8+SUdwsluFh7IlzJh30LY=
68+
github.com/gorilla/schema v1.1.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
6769
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
6870
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
6971
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 h1:HcUWd006luQPljE73d5sk+/VgYPGUReEVz2y1/qylwY=

‎internals/sql/GitMaterial.go

+16
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ type MaterialRepository interface {
6060
Save(material *GitMaterial) error
6161
FindActive() ([]*GitMaterial, error)
6262
FindAll() ([]*GitMaterial, error)
63+
FindInRage(startFrom int, endAt int) ([]*GitMaterial, error)
6364
FindAllActiveByUrls(urls []string) ([]*GitMaterial, error)
6465
}
6566
type MaterialRepositoryImpl struct {
@@ -103,6 +104,21 @@ func (repo MaterialRepositoryImpl) FindAll() ([]*GitMaterial, error) {
103104
return materials, err
104105
}
105106

107+
func (repo MaterialRepositoryImpl) FindInRage(startFrom int, endAt int) ([]*GitMaterial, error) {
108+
var materials []*GitMaterial
109+
query := repo.dbConnection.Model(&materials).
110+
Column("git_material.*", "GitProvider").
111+
Where("deleted =? ", false)
112+
if startFrom != 0 {
113+
query.Where("git_material.id >= ?", startFrom)
114+
}
115+
if endAt != 0 {
116+
query.Where("git_material.id <= ?", endAt)
117+
}
118+
err := query.Select()
119+
return materials, err
120+
}
121+
106122
func (repo MaterialRepositoryImpl) FindById(id int) (*GitMaterial, error) {
107123
var material GitMaterial
108124
err := repo.dbConnection.Model(&material).

‎pkg/RepoManages.go

+37-10
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"encoding/json"
2121
"errors"
2222
"fmt"
23+
"github.com/devtron-labs/git-sensor/bean"
2324
"github.com/devtron-labs/git-sensor/internals"
2425
"github.com/devtron-labs/git-sensor/internals/sql"
2526
"github.com/devtron-labs/git-sensor/internals/util"
@@ -39,7 +40,7 @@ type RepoManager interface {
3940
AddRepo(gitCtx git.GitContext, material []*sql.GitMaterial) ([]*sql.GitMaterial, error)
4041
UpdateRepo(gitCtx git.GitContext, material *sql.GitMaterial) (*sql.GitMaterial, error)
4142
SavePipelineMaterial(gitCtx git.GitContext, material []*sql.CiPipelineMaterial) ([]*sql.CiPipelineMaterial, error)
42-
ReloadAllRepo(gitCtx git.GitContext)
43+
ReloadAllRepo(gitCtx git.GitContext, req *bean.ReloadAllMaterialQuery) (err error)
4344
ResetRepo(gitCtx git.GitContext, materialId int) error
4445
GetReleaseChanges(gitCtx git.GitContext, request *ReleaseChangesRequest) (*git.GitChanges, error)
4546
GetCommitInfoForTag(gitCtx git.GitContext, request *git.CommitMetadataRequest) (*git.GitCommitBase, error)
@@ -202,7 +203,10 @@ func (impl RepoManagerImpl) updatePipelineMaterialCommit(gitCtx git.GitContext,
202203
var repository *git.GitRepository
203204
commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repository, pipelineMaterial.Value, "", "", fetchCount, material.CheckoutLocation, true)
204205
//commits, err := impl.FetchChanges(pipelineMaterial.Id, "", "", 0)
205-
if err == nil {
206+
if gitCtx.Err() != nil {
207+
impl.logger.Errorw("context error in getting commits", "err", gitCtx.Err())
208+
return gitCtx.Err()
209+
} else if err == nil {
206210
impl.logger.Infow("commits found", "commit", commits)
207211
b, err := json.Marshal(commits)
208212
if err == nil {
@@ -361,7 +365,10 @@ func (impl RepoManagerImpl) checkoutMaterial(gitCtx git.GitContext, material *sq
361365
checkoutLocationForFetching := impl.repositoryManager.GetCheckoutLocation(gitCtx, material, gitProvider.Url, checkoutPath)
362366

363367
err = impl.repositoryManager.Add(gitCtx, material.GitProviderId, checkoutPath, material.Url, gitProvider.AuthMode, gitProvider.SshPrivateKey)
364-
if err == nil {
368+
if gitCtx.Err() != nil {
369+
impl.logger.Errorw("context error in git checkout", "err", gitCtx.Err())
370+
return material, gitCtx.Err()
371+
} else if err == nil {
365372
material.CheckoutLocation = checkoutLocationForFetching
366373
material.CheckoutStatus = true
367374
} else {
@@ -386,18 +393,38 @@ func (impl RepoManagerImpl) checkoutMaterial(gitCtx git.GitContext, material *sq
386393
return material, nil
387394
}
388395

389-
func (impl RepoManagerImpl) ReloadAllRepo(gitCtx git.GitContext) {
390-
materials, err := impl.materialRepository.FindAll()
391-
if err != nil {
392-
impl.logger.Errorw("error in reloading materials")
396+
func (impl RepoManagerImpl) ReloadAllRepo(gitCtx git.GitContext, req *bean.ReloadAllMaterialQuery) (err error) {
397+
var materials []*sql.GitMaterial
398+
if req != nil {
399+
materials, err = impl.materialRepository.FindInRage(req.Start, req.End)
400+
if err != nil {
401+
impl.logger.Errorw(bean.GetReloadAllLog("error in getting reload materials"), "err", err)
402+
return err
403+
}
404+
} else {
405+
materials, err = impl.materialRepository.FindAll()
406+
if err != nil {
407+
impl.logger.Errorw(bean.GetReloadAllLog("error in getting reload materials"), "err", err)
408+
return err
409+
}
393410
}
411+
394412
for _, material := range materials {
395-
if _, err := impl.checkoutRepo(gitCtx, material); err != nil {
396-
impl.logger.Errorw("error in checkout", "material", material, "err", err)
413+
impl.logger.Infow(bean.GetReloadAllLog("performing material checkout for"), "materialId", material.Id)
414+
_, err = impl.checkoutRepo(gitCtx, material)
415+
if gitCtx.Err() != nil {
416+
impl.logger.Errorw(bean.GetReloadAllLog("error in material checkout"), "materialId", material.Id, "err", gitCtx.Err())
417+
return gitCtx.Err()
418+
} else if err != nil {
419+
impl.logger.Errorw(bean.GetReloadAllLog("context error in material while checkout"), "materialId", material.Id, "err", err)
420+
// skipping for other materials to be processed
421+
} else {
422+
impl.logger.Infow(bean.GetReloadAllLog("successfully checked out for material"), "materialId", material.Id)
397423
}
398-
399424
}
425+
return nil
400426
}
427+
401428
func (impl RepoManagerImpl) ResetRepo(gitCtx git.GitContext, materialId int) error {
402429
material, err := impl.materialRepository.FindById(materialId)
403430
if err != nil {

‎vendor/github.com/gorilla/schema/.travis.yml

+18
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/gorilla/schema/LICENSE

+27
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/gorilla/schema/README.md

+90
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)