Skip to content

Commit 7c95bec

Browse files
zhengkunwang223zhengkunwang223
authored
feat: Optimize the OpenResty module compilation logic. (#13355)
* refactor: separate openresty module catalog and state * feat: Optimize the OpenResty module compilation logic.
1 parent 6b1480c commit 7c95bec

25 files changed

Lines changed: 1098 additions & 1101 deletions

File tree

‎agent/app/dto/nginx.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -89,22 +89,22 @@ var LBAlgorithms = map[string]struct{}{"ip_hash": {}, "least_conn": {}}
8989
var RealIPKeys = map[string]struct{}{"X-Forwarded-For": {}, "X-Real-IP": {}, "CF-Connecting-IP": {}}
9090

9191
type NginxModule struct {
92-
Name string `json:"name"`
93-
Script string `json:"script"`
94-
Packages []string `json:"packages"`
95-
Params string `json:"params"`
96-
Enable bool `json:"enable"`
97-
Deleted bool `json:"deleted,omitempty"`
98-
BuildMode string `json:"buildMode,omitempty"`
99-
Provider string `json:"provider,omitempty"`
100-
DynamicSupport string `json:"dynamicSupport,omitempty"`
101-
LoadOrder int `json:"loadOrder,omitempty"`
102-
Builds []NginxModuleBuild `json:"builds,omitempty"`
103-
LastError string `json:"lastError,omitempty"`
92+
Name string `json:"name"`
93+
Custom bool `json:"custom,omitempty"`
94+
Script string `json:"script"`
95+
Packages []string `json:"packages"`
96+
Params string `json:"params"`
97+
Enable bool `json:"enable"`
98+
BuildMode string `json:"buildMode,omitempty"`
99+
Provider string `json:"provider,omitempty"`
100+
LoadOrder int `json:"loadOrder,omitempty"`
101+
Builds []NginxModuleBuild `json:"builds,omitempty"`
102+
LastError string `json:"lastError,omitempty"`
104103
}
105104

106105
type NginxModuleBuild struct {
107106
Provider string `json:"provider"`
107+
BuildMode string `json:"buildMode"`
108108
Status string `json:"status"`
109109
Hash string `json:"hash"`
110110
Target NginxModuleTarget `json:"target"`

‎agent/app/dto/request/nginx.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ type NginxModuleUpdate struct {
127127
Packages string `json:"packages"`
128128
Enable bool `json:"enable"`
129129
Params string `json:"params"`
130-
BuildMode string `json:"buildMode" validate:"omitempty,oneof=auto dynamic static"`
130+
BuildMode string `json:"buildMode" validate:"omitempty,oneof=dynamic static"`
131131
Provider string `json:"provider" validate:"omitempty,oneof=local prebuilt"`
132132
LoadOrder int `json:"loadOrder" validate:"omitempty,min=0,max=9999"`
133133
}

‎agent/app/dto/response/nginx.go‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -69,20 +69,19 @@ type NginxProxyCache struct {
6969
}
7070

7171
type NginxModule struct {
72-
Name string `json:"name"`
73-
Script string `json:"script"`
74-
Packages string `json:"packages"`
75-
Params string `json:"params"`
76-
Enable bool `json:"enable"`
77-
BuildMode string `json:"buildMode"`
78-
Provider string `json:"provider"`
79-
DynamicSupport string `json:"dynamicSupport"`
80-
LoadOrder int `json:"loadOrder"`
81-
BuildStatus string `json:"buildStatus"`
82-
LoadStatus string `json:"loadStatus"`
83-
Compatibility string `json:"compatibility"`
84-
Artifacts []dto.NginxModuleArtifact `json:"artifacts"`
85-
LastError string `json:"lastError"`
72+
Name string `json:"name"`
73+
Custom bool `json:"custom"`
74+
Script string `json:"script"`
75+
Packages string `json:"packages"`
76+
Params string `json:"params"`
77+
Enable bool `json:"enable"`
78+
BuildMode string `json:"buildMode"`
79+
Provider string `json:"provider"`
80+
LoadOrder int `json:"loadOrder"`
81+
BuildStatus string `json:"buildStatus"`
82+
LoadStatus string `json:"loadStatus"`
83+
Artifacts []dto.NginxModuleArtifact `json:"artifacts"`
84+
LastError string `json:"lastError"`
8685
}
8786

8887
type NginxBuildConfig struct {

‎agent/app/service/app_utils.go‎

Lines changed: 111 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -752,13 +752,13 @@ func getUpgradeCompose(install model.AppInstall, detail model.AppDetail) (string
752752
return string(composeByte), nil
753753
}
754754

755-
func buildNginx(parentTask *task.Task, nginxInstall model.AppInstall) error {
755+
func buildNginx(parentTask *task.Task, nginxInstall model.AppInstall, catalogPath string) error {
756756
fileOp := files.NewFileOp()
757757
buildPath := path.Join(nginxInstall.GetPath(), nginxModuleBuildDir)
758758
if !fileOp.Stat(buildPath) {
759759
return buserr.New("ErrBuildDirNotFound")
760760
}
761-
modules, err := loadNginxModules(nginxInstall)
761+
modules, err := loadNginxModulesWithCatalog(nginxInstall, catalogPath)
762762
if err != nil {
763763
return err
764764
}
@@ -776,18 +776,19 @@ func buildNginx(parentTask *task.Task, nginxInstall model.AppInstall) error {
776776
}
777777
parentTask.LogSuccess(logStr)
778778
}
779-
modules, err = buildDynamicNginxModules(nginxInstall, modules, nil, false, "", parentTask)
779+
modules, err = buildDynamicNginxModules(nginxInstall, modules, nil, false, "", catalogPath, parentTask)
780780
if err != nil {
781781
return err
782782
}
783-
return commitNginxModuleBuilds(nginxInstall, previousModules, modules, false)
783+
return commitNginxModuleBuilds(nginxInstall, previousModules, modules, false, catalogPath)
784784
}
785785

786786
func upgradeInstall(req request.AppInstallUpgrade) error {
787787
install, err := appInstallRepo.GetFirst(repo.WithByID(req.InstallID))
788788
if err != nil {
789789
return err
790790
}
791+
originalInstall := install
791792
oldVersion := install.Version
792793
detail, err := appDetailRepo.GetFirst(repo.WithByID(req.DetailID))
793794
if err != nil {
@@ -807,8 +808,9 @@ func upgradeInstall(req request.AppInstallUpgrade) error {
807808
install.Status = constant.StatusUpgrading
808809

809810
var (
810-
upErr error
811-
backupFile string
811+
upErr error
812+
backupFile string
813+
nginxUpgradeSnapshot *openrestyUpgradeSnapshot
812814
)
813815
backUpApp := func(t *task.Task) error {
814816
backupService := NewIBackupService()
@@ -864,6 +866,13 @@ func upgradeInstall(req request.AppInstallUpgrade) error {
864866
}
865867
oldEnvContent := append([]byte(nil), content...)
866868
oldDockerCompose := install.DockerCompose
869+
targetNginxCatalogPath := ""
870+
if install.App.Key == constant.AppOpenresty {
871+
nginxUpgradeSnapshot, err = createOpenrestyUpgradeSnapshot(install.GetPath())
872+
if err != nil {
873+
return err
874+
}
875+
}
867876
if install.App.Key == vllmAppKeyForUpgrade {
868877
envs := make(map[string]interface{})
869878
if err = json.Unmarshal([]byte(install.Env), &envs); err != nil {
@@ -896,15 +905,16 @@ func upgradeInstall(req request.AppInstallUpgrade) error {
896905
if err := fileOp.CopyFile(path.Join(detailBuildDir, "Dockerfile"), installBuildDir); err != nil {
897906
return err
898907
}
899-
if fileOp.Stat(path.Join(detailBuildDir, nginxModuleBuilderFile)) {
900-
if err := fileOp.CopyFile(path.Join(detailBuildDir, nginxModuleBuilderFile), installBuildDir); err != nil {
901-
return err
902-
}
908+
if err := syncNginxModuleBuilder(detailBuildDir, installBuildDir); err != nil {
909+
return err
903910
}
904-
if fileOp.Stat(path.Join(detailBuildDir, nginxModuleCatalogFile)) {
905-
if err := fileOp.CopyFile(path.Join(detailBuildDir, nginxModuleCatalogFile), installBuildDir); err != nil {
906-
return err
907-
}
911+
targetCatalogSource := path.Join(detailBuildDir, nginxModuleCatalogFile)
912+
if !fileOp.Stat(targetCatalogSource) {
913+
return fmt.Errorf("target OpenResty module catalog not found: %s", targetCatalogSource)
914+
}
915+
targetNginxCatalogPath = path.Join(installBuildDir, nginxModuleCatalogPendingFile)
916+
if err := stageNginxModuleCatalog(targetCatalogSource, targetNginxCatalogPath); err != nil {
917+
return err
908918
}
909919
if err := fileOp.CopyFile(path.Join(detailBuildDir, "nginx.conf"), installBuildDir); err != nil {
910920
return err
@@ -984,19 +994,19 @@ func upgradeInstall(req request.AppInstallUpgrade) error {
984994
}
985995

986996
if install.App.Key == constant.AppOpenresty {
987-
modules, moduleErr := loadNginxModules(install)
997+
modules, moduleErr := loadNginxModulesWithCatalog(install, targetNginxCatalogPath)
988998
if moduleErr != nil {
989999
return moduleErr
9901000
}
9911001
// Build dynamic modules for the target version before stopping the
9921002
// current container. Static modules retain the full rebuild path.
9931003
if !hasEnabledStaticNginxModules(modules) {
9941004
previousModules := cloneNginxModules(modules)
995-
modules, moduleErr = buildDynamicNginxModules(install, modules, nil, false, "", t)
1005+
modules, moduleErr = buildDynamicNginxModules(install, modules, nil, false, "", targetNginxCatalogPath, t)
9961006
if moduleErr != nil {
9971007
return moduleErr
9981008
}
999-
if moduleErr = saveNginxModules(install, modules); moduleErr != nil {
1009+
if moduleErr = saveNginxModulesWithCatalog(install, modules, targetNginxCatalogPath); moduleErr != nil {
10001010
removeNginxModuleOutputsNotReferenced(install, modules, previousModules)
10011011
return moduleErr
10021012
}
@@ -1037,7 +1047,7 @@ func upgradeInstall(req request.AppInstallUpgrade) error {
10371047
}
10381048

10391049
if install.App.Key == constant.AppOpenresty {
1040-
if err = buildNginx(t, install); err != nil {
1050+
if err = buildNginx(t, install, targetNginxCatalogPath); err != nil {
10411051
t.Log(err.Error())
10421052
return err
10431053
}
@@ -1053,8 +1063,24 @@ func upgradeInstall(req request.AppInstallUpgrade) error {
10531063
}
10541064
t.LogSuccess(logStr)
10551065
install.Status = constant.StatusRunning
1056-
if err = appInstallRepo.Save(context.Background(), &install); err != nil {
1057-
return err
1066+
if install.App.Key == constant.AppOpenresty {
1067+
if err = commitStaticNginxModuleBuilds(install, targetNginxCatalogPath, t); err != nil {
1068+
return err
1069+
}
1070+
activeCatalogPath := path.Join(install.GetPath(), nginxModuleBuildDir, nginxModuleCatalogFile)
1071+
if err = activateNginxModuleCatalogAndCommit(targetNginxCatalogPath, activeCatalogPath, func() error {
1072+
return appInstallRepo.Save(context.Background(), &install)
1073+
}); err != nil {
1074+
return err
1075+
}
1076+
} else {
1077+
if err = appInstallRepo.Save(context.Background(), &install); err != nil {
1078+
return err
1079+
}
1080+
}
1081+
if nginxUpgradeSnapshot != nil {
1082+
nginxUpgradeSnapshot.Cleanup()
1083+
nginxUpgradeSnapshot = nil
10581084
}
10591085
if req.DeleteImage {
10601086
newEnvContent, err := fileOp.GetContent(install.GetEnvPath())
@@ -1083,30 +1109,88 @@ func upgradeInstall(req request.AppInstallUpgrade) error {
10831109
rollBackApp := func(t *task.Task) {
10841110
if req.Backup {
10851111
t.Log(i18n.GetWithName("AppRecover", install.Name))
1086-
if err := NewIBackupService().AppRecover(dto.CommonRecover{Name: install.App.Key, DetailName: install.Name, Type: "app", DownloadAccountID: 1, File: backupFile}); err != nil {
1087-
t.LogFailedWithErr(i18n.GetWithName("AppRecover", install.Name), err)
1112+
recoverErr := NewIBackupService().AppRecover(dto.CommonRecover{
1113+
Name: install.App.Key, DetailName: install.Name, Type: "app", DownloadAccountID: 1, File: backupFile,
1114+
})
1115+
if recoverErr == nil {
1116+
if nginxUpgradeSnapshot != nil {
1117+
nginxUpgradeSnapshot.Cleanup()
1118+
nginxUpgradeSnapshot = nil
1119+
}
1120+
t.LogSuccess(i18n.GetWithName("AppRecover", install.Name))
10881121
return
10891122
}
1123+
t.LogFailedWithErr(i18n.GetWithName("AppRecover", install.Name), recoverErr)
1124+
if install.App.Key != constant.AppOpenresty {
1125+
return
1126+
}
1127+
}
1128+
if install.App.Key == constant.AppOpenresty && nginxUpgradeSnapshot != nil {
1129+
if out, rollbackErr := compose.Down(install.GetComposePath()); rollbackErr != nil {
1130+
if out != "" {
1131+
rollbackErr = fmt.Errorf("%s: %w", out, rollbackErr)
1132+
}
1133+
t.LogFailedWithErr(i18n.GetWithName("AppRecover", install.Name), rollbackErr)
1134+
}
1135+
if rollbackErr := nginxUpgradeSnapshot.Restore(); rollbackErr != nil {
1136+
t.LogFailedWithErr(i18n.GetWithName("AppRecover", install.Name), rollbackErr)
1137+
return
1138+
}
1139+
nginxUpgradeSnapshot.Cleanup()
1140+
nginxUpgradeSnapshot = nil
1141+
if out, rollbackErr := compose.Up(originalInstall.GetComposePath()); rollbackErr != nil {
1142+
if out != "" {
1143+
rollbackErr = fmt.Errorf("%s: %w", out, rollbackErr)
1144+
}
1145+
t.LogFailedWithErr(i18n.GetWithName("AppRecover", install.Name), rollbackErr)
1146+
return
1147+
}
1148+
originalInstall.Status = constant.StatusRunning
1149+
originalInstall.Message = ""
1150+
if rollbackErr := appInstallRepo.Save(context.Background(), &originalInstall); rollbackErr != nil {
1151+
t.LogFailedWithErr(i18n.GetWithName("AppRecover", install.Name), rollbackErr)
1152+
return
1153+
}
1154+
install = originalInstall
10901155
t.LogSuccess(i18n.GetWithName("AppRecover", install.Name))
10911156
return
10921157
}
1158+
if install.App.Key == constant.AppOpenresty {
1159+
if rollbackErr := appInstallRepo.Save(context.Background(), &originalInstall); rollbackErr != nil {
1160+
t.LogFailedWithErr(i18n.GetWithName("AppRecover", install.Name), rollbackErr)
1161+
return
1162+
}
1163+
install = originalInstall
1164+
t.LogSuccess(i18n.GetWithName("AppRecover", install.Name))
1165+
}
10931166
}
10941167

1095-
upgradeTask.AddSubTaskWithOps(task.GetTaskName(install.Name, task.TaskUpgrade, task.TaskScopeApp), upgradeApp, rollBackApp, 0, 1*time.Hour)
1168+
upgradeTimeout := 1 * time.Hour
1169+
if install.App.Key == constant.AppOpenresty {
1170+
// Dynamic modules are built serially and each Docker build has its own
1171+
// timeout. An outer deadline would start rollback while upgradeApp is
1172+
// still mutating the installation because SubTask does not stop its
1173+
// action goroutine on timeout.
1174+
upgradeTimeout = 0
1175+
}
1176+
upgradeTask.AddSubTaskWithOps(task.GetTaskName(install.Name, task.TaskUpgrade, task.TaskScopeApp), upgradeApp, rollBackApp, 0, upgradeTimeout)
10961177

1178+
upgradingInstall := install
1179+
if err = appInstallRepo.Save(context.Background(), &upgradingInstall); err != nil {
1180+
return err
1181+
}
10971182
go func() {
1098-
err = upgradeTask.Execute()
1099-
if err != nil {
1183+
if taskErr := upgradeTask.Execute(); taskErr != nil {
11001184
existInstall, _ := appInstallRepo.GetFirst(repo.WithByID(req.InstallID))
11011185
if existInstall.ID > 0 && existInstall.Status != constant.StatusRunning {
11021186
existInstall.Status = constant.StatusUpgradeErr
1103-
existInstall.Message = err.Error()
1187+
existInstall.Message = taskErr.Error()
11041188
_ = appInstallRepo.Save(context.Background(), &existInstall)
11051189
}
11061190
}
11071191
}()
11081192

1109-
return appInstallRepo.Save(context.Background(), &install)
1193+
return nil
11101194
}
11111195

11121196
func skipCheckStatus(service types.ServiceConfig) bool {

0 commit comments

Comments
 (0)