Skip to content

Commit 0247eee

Browse files
committed
refactor(model): move repository tag management APIs for Docker registry (#779)
**Because** - Repository tag management for Docker registry operations belongs in model-backend rather than artifact-backend, as it's specifically for model versioning and deployment workflows - The api-gateway registry plugin needs to record tag metadata (digest, timestamps) after successful Docker image pushes to enable proper model version tracking - The Docker Registry V2 API doesn't persist tag metadata, requiring separate storage in the model service database **This commit** - Adds `RepositoryTag` message to `model/model/v1alpha/model.proto` with fields: name (resource name), id (tag identifier), digest (manifest identifier), and update_time - Adds request/response message pairs for all tag operations: - `ListRepositoryTagsRequest`/`ListRepositoryTagsResponse` - paginated tag listing with filters - `GetRepositoryTagRequest`/`GetRepositoryTagResponse` - retrieve single tag details - `CreateRepositoryTagRequest`/`CreateRepositoryTagResponse` - create/upsert tag with digest - `DeleteRepositoryTagRequest`/`DeleteRepositoryTagResponse` - remove tag from database and registry - Adds four RPC methods to `ModelPrivateService` in `model/model/v1alpha/model_private_service.proto`: - `ListRepositoryTags` - list tags for a repository - `GetRepositoryTag` - get specific tag details - `CreateRepositoryTag` - create/update tag (called by api-gateway after manifest push) - `DeleteRepositoryTag` - delete tag from both database and Docker registry - Provides private service endpoints enabling api-gateway registry plugin integration with model-backend
1 parent c726c96 commit 0247eee

19 files changed

+6971
-1897
lines changed

.github/workflows/integration-test.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ jobs:
9898
- name: Launch Instill Core CE (commit hash)
9999
working-directory: instill-core
100100
run: |
101-
make compose-dev EDITION=docker-ce:test ENV_SECRETS_COMPONENT=.env.secrets.component.test MODEL_BACKEND_VERSION=${{ env.COMMIT_SHORT_SHA }}
101+
make compose-dev EDITION=docker-ce:test CI=true MODEL_BACKEND_VERSION=${{ env.COMMIT_SHORT_SHA }}
102102
103103
- name: Run integration-test
104104
working-directory: model-backend

go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ require (
77
github.com/alicebob/miniredis/v2 v2.33.0
88
github.com/frankban/quicktest v1.14.6
99
github.com/gabriel-vasile/mimetype v1.4.5
10+
github.com/go-resty/resty/v2 v2.16.5
1011
github.com/gofrs/uuid v4.4.0+incompatible
1112
github.com/gogo/status v1.1.1
1213
github.com/gojuno/minimock/v3 v3.4.5
@@ -16,7 +17,7 @@ require (
1617
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1
1718
github.com/iancoleman/strcase v0.3.0
1819
github.com/influxdata/influxdb-client-go/v2 v2.14.0
19-
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20250707160902-77023eb2f033
20+
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20251029195138-d623a821ff95
2021
github.com/instill-ai/usage-client v0.4.0
2122
github.com/instill-ai/x v0.10.0-alpha
2223
github.com/jackc/pgx/v5 v5.6.0

go.sum

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
127127
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
128128
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
129129
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
130+
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
131+
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
130132
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
131133
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
132134
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
@@ -248,8 +250,8 @@ github.com/influxdata/influxdb-client-go/v2 v2.14.0 h1:AjbBfJuq+QoaXNcrova8smSjw
248250
github.com/influxdata/influxdb-client-go/v2 v2.14.0/go.mod h1:Ahpm3QXKMJslpXl3IftVLVezreAUtBOTZssDrjZEFHI=
249251
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
250252
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
251-
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20250707160902-77023eb2f033 h1:jhP9Gz7tw57rTTHQ7WhHOWf7z/worMcfr/n3NAcjbD4=
252-
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20250707160902-77023eb2f033/go.mod h1:bCnBosofpaUxKBuTTJM3/I3thAK37kvfBnKByjnLsl4=
253+
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20251029195138-d623a821ff95 h1:FRySdN/mdGDyNZNZAzueGPDPetMEuP3wtbyhg3cu0XE=
254+
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20251029195138-d623a821ff95/go.mod h1:bCnBosofpaUxKBuTTJM3/I3thAK37kvfBnKByjnLsl4=
253255
github.com/instill-ai/usage-client v0.4.0 h1:xf1hAlO4a8lZwZzz9bprZOJqU3ghIcIsavUUB7UURyg=
254256
github.com/instill-ai/usage-client v0.4.0/go.mod h1:zZ9LRoXps2u63ARYPAbR2YvqTib3dWJLObZn+9YqhF0=
255257
github.com/instill-ai/x v0.10.0-alpha h1:I83WJc+21J+IgI4aJDn755ON/BX4cDvKCVVguI77r14=

pkg/client/http/registry.go

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package httpclient
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net/http"
7+
"time"
8+
9+
"github.com/go-resty/resty/v2"
10+
11+
logx "github.com/instill-ai/x/log"
12+
)
13+
14+
const (
15+
reqTimeout = time.Second * 30
16+
maxRetryCount = 3
17+
retryDelay = 100 * time.Millisecond
18+
)
19+
20+
// RegistryClient interacts with the Docker Registry HTTP V2 API.
21+
type RegistryClient struct {
22+
*resty.Client
23+
}
24+
25+
// NewRegistryClient returns an initialized registry HTTP client.
26+
func NewRegistryClient(ctx context.Context, registryHost string, registryPort int) *RegistryClient {
27+
logger, _ := logx.GetZapLogger(ctx)
28+
baseURL := fmt.Sprintf("http://%s:%d", registryHost, registryPort)
29+
30+
r := resty.New().
31+
SetLogger(logger.Sugar()).
32+
SetBaseURL(baseURL).
33+
SetTimeout(reqTimeout).
34+
SetTransport(&http.Transport{
35+
DisableKeepAlives: true,
36+
}).
37+
SetRetryCount(maxRetryCount).
38+
SetRetryWaitTime(retryDelay)
39+
40+
return &RegistryClient{Client: r}
41+
}
42+
43+
type tagList struct {
44+
Tags []string `json:"tags"`
45+
}
46+
47+
// ListTags calls the GET /v2/<name>/tags/list endpoint, where <name> is a
48+
// repository.
49+
func (c *RegistryClient) ListTags(ctx context.Context, repository string) ([]string, error) {
50+
var resp tagList
51+
52+
tagsPath := fmt.Sprintf("/v2/%s/tags/list", repository)
53+
r := c.R().SetContext(ctx).SetResult(&resp)
54+
if _, err := r.Get(tagsPath); err != nil {
55+
return nil, fmt.Errorf("couldn't connect with registry: %w", err)
56+
}
57+
58+
return resp.Tags, nil
59+
}
60+
61+
// DeleteTag calls the DELETE /v2/<name>/manifests/<reference> endpoint, where <name> is a
62+
// repository, and <reference> is the digest
63+
func (c *RegistryClient) DeleteTag(ctx context.Context, repository string, digest string) error {
64+
65+
deletePath := fmt.Sprintf("/v2/%s/manifests/%s", repository, digest)
66+
r := c.R().SetContext(ctx)
67+
if _, err := r.Delete(deletePath); err != nil {
68+
return fmt.Errorf("couldn't delete the image with registry: %w", err)
69+
}
70+
71+
return nil
72+
}
73+
74+
// GetTagDigest calls the HEAD /v2/<name>/manifests/<reference> endpoint, where <name> is a
75+
// repository, and <reference> is the tag
76+
func (c *RegistryClient) GetTagDigest(ctx context.Context, repository string, tag string) (string, error) {
77+
78+
digestPath := fmt.Sprintf("/v2/%s/manifests/%s", repository, tag)
79+
r := c.R().SetContext(ctx).SetHeader("Accept", "application/vnd.docker.distribution.manifest.v2+json")
80+
resp, err := r.Head(digestPath)
81+
if err != nil {
82+
return "", fmt.Errorf("couldn't get the image digest: %w", err)
83+
}
84+
85+
return resp.Header().Get("Docker-Content-Digest"), nil
86+
}

pkg/datamodel/datamodel.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,3 +211,11 @@ const (
211211
FieldLastRunTime = "last_run_time"
212212
FieldNumberOfRuns = "number_of_runs"
213213
)
214+
215+
// Tag represents a repository tag (domain model) for Docker registry versioning
216+
type Tag struct {
217+
Name string // The name of the tag (e.g. "repositories/{repo}/tags/{id}")
218+
ID string // The tag identifier
219+
Digest string // Unique identifier from the manifest
220+
UpdateTime time.Time // Tag update time
221+
}
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
BEGIN;
2+
DROP TABLE IF EXISTS repository_tag CASCADE;
3+
COMMIT;
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
BEGIN;
2+
CREATE TABLE IF NOT EXISTS repository_tag (
3+
-- "<repository>:tag", e.g. "melancholic-wombat/llava-34b:latest"
4+
name VARCHAR(255) PRIMARY KEY,
5+
digest VARCHAR(255) NOT NULL,
6+
create_time TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL,
7+
update_time TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL
8+
);
9+
COMMIT;

pkg/db/migration/migration.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import (
1515
)
1616

1717
// TargetSchemaVersion is the target database schema version
18-
const TargetSchemaVersion = 12
18+
const TargetSchemaVersion = 13
1919

2020
type migration interface {
2121
Migrate() error

pkg/handler/private.go

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import (
1414
"github.com/instill-ai/model-backend/pkg/ray"
1515
"github.com/instill-ai/model-backend/pkg/resource"
1616

17-
artifactpb "github.com/instill-ai/protogen-go/artifact/artifact/v1alpha"
1817
modelpb "github.com/instill-ai/protogen-go/model/model/v1alpha"
1918
)
2019

@@ -126,7 +125,7 @@ func (h *PrivateHandler) DeployNamespaceModelAdmin(ctx context.Context, req *mod
126125
}
127126
}
128127

129-
if _, err := h.service.GetArtifactPrivateServiceClient().GetRepositoryTag(ctx, &artifactpb.GetRepositoryTagRequest{
128+
if _, err := h.service.GetRepositoryTag(ctx, &modelpb.GetRepositoryTagRequest{
130129
Name: fmt.Sprintf("repositories/%s/%s/tags/%s", ns.NsID, req.GetModelId(), version.Version),
131130
}); err != nil {
132131
return nil, err
@@ -189,3 +188,25 @@ func (h *PrivateHandler) UndeployNamespaceModelAdmin(ctx context.Context, req *m
189188

190189
return &modelpb.UndeployNamespaceModelAdminResponse{}, nil
191190
}
191+
192+
// Repository Tag Management handlers
193+
194+
// ListRepositoryTags lists tags in a repository
195+
func (h *PrivateHandler) ListRepositoryTags(ctx context.Context, req *modelpb.ListRepositoryTagsRequest) (*modelpb.ListRepositoryTagsResponse, error) {
196+
return h.service.ListRepositoryTags(ctx, req)
197+
}
198+
199+
// GetRepositoryTag gets details of a repository tag
200+
func (h *PrivateHandler) GetRepositoryTag(ctx context.Context, req *modelpb.GetRepositoryTagRequest) (*modelpb.GetRepositoryTagResponse, error) {
201+
return h.service.GetRepositoryTag(ctx, req)
202+
}
203+
204+
// CreateRepositoryTag creates a new repository tag
205+
func (h *PrivateHandler) CreateRepositoryTag(ctx context.Context, req *modelpb.CreateRepositoryTagRequest) (*modelpb.CreateRepositoryTagResponse, error) {
206+
return h.service.CreateRepositoryTag(ctx, req)
207+
}
208+
209+
// DeleteRepositoryTag deletes a repository tag
210+
func (h *PrivateHandler) DeleteRepositoryTag(ctx context.Context, req *modelpb.DeleteRepositoryTagRequest) (*modelpb.DeleteRepositoryTagResponse, error) {
211+
return h.service.DeleteRepositoryTag(ctx, req)
212+
}

pkg/mock/acl_client_interface_mock.gen.go

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)