Populate Version.Integrity for pub, julia and nuget (#23)

pub.dev returns archive_sha256 on each version entry, julia's
Versions.toml already gives us git-tree-sha1, and nuget's catalog
leaf carries packageHash/packageHashAlgorithm (sha512, base64). The
nuget hash is not in the registration index so this fetches the
catalog leaf per version, same pattern hex already uses.

deno is left out: the apiland /v2/modules endpoint only returns bare
version strings with no per-version detail endpoint, so there is no
checksum to read without changing data source.

Related to ecosyste-ms/packages#1630.

Author: andrew

internal/julia/julia.go

@@ -148,8 +148,13 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi
 	versions := make([]core.Version, 0, len(versionNumbers))
 	for _, v := range versionNumbers {
 		info := versionMap[v]
+		var integrity string
+		if info.gitTreeSha1 != "" {
+			integrity = "sha1-" + info.gitTreeSha1
+		}
 		versions = append(versions, core.Version{
-			Number: v,
+			Number:    v,
+			Integrity: integrity,
 			Metadata: map[string]any{
 				"git-tree-sha1": info.gitTreeSha1,
 			},

internal/julia/julia_test.go

@@ -89,6 +89,9 @@ func TestFetchVersions(t *testing.T) {
 	if versions[0].Metadata["git-tree-sha1"] != "3043b8e5c7c7f4b6f6f5e3b4b4c5d6e7f8a9b0c1" {
 		t.Errorf("unexpected git-tree-sha1: %v", versions[0].Metadata["git-tree-sha1"])
 	}
+	if versions[0].Integrity != "sha1-3043b8e5c7c7f4b6f6f5e3b4b4c5d6e7f8a9b0c1" {
+		t.Errorf("unexpected integrity: %q", versions[0].Integrity)
+	}
 }
 
 func TestFetchDependencies(t *testing.T) {

internal/nuget/nuget.go

@@ -61,6 +61,7 @@ type registrationLeaf struct {
 }
 
 type catalogEntry struct {
+	CatalogURL    string   `json:"@id"`
 	ID            string   `json:"id"`
 	Version       string   `json:"version"`
 	Description   string   `json:"description"`
@@ -92,6 +93,11 @@ type dependency struct {
 	Range string `json:"range"`
 }
 
+type catalogLeaf struct {
+	PackageHash          string `json:"packageHash"`
+	PackageHashAlgorithm string `json:"packageHashAlgorithm"`
+}
+
 func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package, error) {
 	// NuGet IDs are case-insensitive, lowercase for URL
 	lowerName := strings.ToLower(name)
@@ -191,6 +197,7 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi
 				Number:      entry.Version,
 				PublishedAt: publishedAt,
 				Licenses:    licenses,
+				Integrity:   r.fetchIntegrity(ctx, entry.CatalogURL),
 				Status:      status,
 				Metadata: map[string]any{
 					"listed":      entry.Listed,
@@ -203,6 +210,27 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi
 	return versions, nil
 }
 
+// fetchIntegrity fetches the catalog leaf to extract the package hash.
+// The registration index does not include packageHash, only the full
+// catalog leaf does. Errors are swallowed since integrity is best-effort.
+func (r *Registry) fetchIntegrity(ctx context.Context, catalogURL string) string {
+	if catalogURL == "" {
+		return ""
+	}
+	var leaf catalogLeaf
+	if err := r.client.GetJSON(ctx, catalogURL, &leaf); err != nil {
+		return ""
+	}
+	if leaf.PackageHash == "" {
+		return ""
+	}
+	algo := strings.ToLower(leaf.PackageHashAlgorithm)
+	if algo == "" {
+		algo = "sha512"
+	}
+	return algo + "-" + leaf.PackageHash
+}
+
 func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) {
 	lowerName := strings.ToLower(name)
 	url := fmt.Sprintf("%s/registration5-semver1/%s/index.json", r.baseURL, lowerName)

internal/nuget/nuget_test.go

@@ -95,24 +95,39 @@ func TestFetchPackageWithGitHubRepository(t *testing.T) {
 
 func TestFetchVersions(t *testing.T) {
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/catalog/xunit.2.6.0.json":
+			_ = json.NewEncoder(w).Encode(catalogLeaf{
+				PackageHash:          "ExN13Ybp8c12hwOzvHnrVJeb+I9sR2YgdHUGmIQvU3h+2AHGEU0JieFQ0c8TECX4BgZ/+6aSMAe5ea8LMLykeg==",
+				PackageHashAlgorithm: "SHA512",
+			})
+			return
+		case "/catalog/xunit.2.5.0.json":
+			w.WriteHeader(404)
+			return
+		}
+
+		base := "http://" + r.Host
 		resp := registrationResponse{
 			Items: []registrationPage{
 				{
 					Items: []registrationLeaf{
 						{
 							CatalogEntry: catalogEntry{
-								ID:        "xunit",
-								Version:   "2.6.0",
-								Published: "2023-10-15T12:00:00Z",
-								Listed:    true,
+								CatalogURL: base + "/catalog/xunit.2.6.0.json",
+								ID:         "xunit",
+								Version:    "2.6.0",
+								Published:  "2023-10-15T12:00:00Z",
+								Listed:     true,
 							},
 						},
 						{
 							CatalogEntry: catalogEntry{
-								ID:        "xunit",
-								Version:   "2.5.0",
-								Published: "2023-07-01T12:00:00Z",
-								Listed:    false,
+								CatalogURL: base + "/catalog/xunit.2.5.0.json",
+								ID:         "xunit",
+								Version:    "2.5.0",
+								Published:  "2023-07-01T12:00:00Z",
+								Listed:     false,
 							},
 						},
 						{
@@ -160,6 +175,20 @@ func TestFetchVersions(t *testing.T) {
 	if statusMap["2.4.0"] != core.StatusDeprecated {
 		t.Errorf("expected deprecated status for 2.4.0, got %q", statusMap["2.4.0"])
 	}
+
+	integrityMap := make(map[string]string)
+	for _, v := range versions {
+		integrityMap[v.Number] = v.Integrity
+	}
+	if integrityMap["2.6.0"] != "sha512-ExN13Ybp8c12hwOzvHnrVJeb+I9sR2YgdHUGmIQvU3h+2AHGEU0JieFQ0c8TECX4BgZ/+6aSMAe5ea8LMLykeg==" {
+		t.Errorf("unexpected integrity for 2.6.0: %q", integrityMap["2.6.0"])
+	}
+	if integrityMap["2.5.0"] != "" {
+		t.Errorf("expected empty integrity for 2.5.0 (catalog 404), got %q", integrityMap["2.5.0"])
+	}
+	if integrityMap["2.4.0"] != "" {
+		t.Errorf("expected empty integrity for 2.4.0 (no @id), got %q", integrityMap["2.4.0"])
+	}
 }
 
 func TestFetchDependencies(t *testing.T) {

internal/pub/pub.go

@@ -55,9 +55,10 @@ type packageResponse struct {
 }
 
 type versionInfo struct {
-	Version   string    `json:"version"`
-	Published time.Time `json:"published"`
-	Pubspec   pubspec   `json:"pubspec"`
+	Version       string    `json:"version"`
+	Published     time.Time `json:"published"`
+	ArchiveSHA256 string    `json:"archive_sha256"`
+	Pubspec       pubspec   `json:"pubspec"`
 }
 
 type pubspec struct {
@@ -111,10 +112,15 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi
 
 	versions := make([]core.Version, len(resp.Versions))
 	for i, v := range resp.Versions {
+		var integrity string
+		if v.ArchiveSHA256 != "" {
+			integrity = "sha256-" + v.ArchiveSHA256
+		}
 		versions[i] = core.Version{
 			Number:      v.Version,
 			PublishedAt: v.Published,
 			Licenses:    v.Pubspec.License,
+			Integrity:   integrity,
 		}
 	}
 

internal/pub/pub_test.go

@@ -59,7 +59,7 @@ func TestFetchVersions(t *testing.T) {
 		resp := packageResponse{
 			Name: "provider",
 			Versions: []versionInfo{
-				{Version: "6.1.0", Pubspec: pubspec{License: "MIT"}},
+				{Version: "6.1.0", ArchiveSHA256: "cc4fedb40b3c48d1a7558fcfe8d0f479046f017337d7b6b883208d65dbf8724d", Pubspec: pubspec{License: "MIT"}},
 				{Version: "6.0.0", Pubspec: pubspec{License: "MIT"}},
 				{Version: "5.0.0", Pubspec: pubspec{License: "MIT"}},
 			},
@@ -84,6 +84,12 @@ func TestFetchVersions(t *testing.T) {
 	if versions[0].Licenses != "MIT" {
 		t.Errorf("expected MIT license, got %q", versions[0].Licenses)
 	}
+	if versions[0].Integrity != "sha256-cc4fedb40b3c48d1a7558fcfe8d0f479046f017337d7b6b883208d65dbf8724d" {
+		t.Errorf("unexpected integrity: %q", versions[0].Integrity)
+	}
+	if versions[1].Integrity != "" {
+		t.Errorf("expected empty integrity when archive_sha256 absent, got %q", versions[1].Integrity)
+	}
 }
 
 func TestFetchDependencies(t *testing.T) {