Skip to content

Commit 8739ec4

Browse files
rdimitrovclaude
rdimitrov
and
claude
authored
Add registry format converters and support for the Official MCP Registry (#2469)
* Initial PoC of supporting the official MCP registry Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Temporarily copy the converters from toolhive-registry Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Complete the initial implementation Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Support setting the registry API via REST Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Move the whole converters package from toolhive-registry Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Fix the imports in the tests Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Config changes now take effect immediately without a restart Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Fix a small issue with converting the names Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Normalise the full name instead of simplifying it Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Fix imports after rebase - update to use pkg/registry/types * Fix imports after package reorganization rebase Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Fix CodeQL warning and regenerate swagger docs Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Fix another CodeQL error Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Update MCP Registry API version from v0 to v0.1 in docs The API endpoints use v0.1 as the version path component. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Add User-Agent header to MCP Registry API client Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Extract environment variables from runtime arguments in MCP registry converter Support extracting environment variables from both sources in the MCP Registry API format: 1. The dedicated environmentVariables field (preferred way) 2. The -e/--env flags in runtimeArguments (Docker CLI pattern) Many servers in the MCP registry (like github-mcp-server) use the Docker CLI pattern of specifying environment variables as runtime arguments with -e flags, rather than using the dedicated environmentVariables field. This is a valid approach according to the MCP registry schema. Changes: - Add extractEnvironmentVariables() to handle both sources - Add extractEnvFromRuntimeArgs() to parse -e/--env flags - Add parseEnvVarFromValue() to extract variable metadata - Parse variable references like {token} and extract metadata (isSecret, isRequired, etc.) - Handle static values and complex values with multiple equals signs - Add comprehensive test suite with 15+ test cases covering all scenarios - Add integration test with realistic GitHub MCP server data This ensures ToolHive can properly consume all servers from the official MCP registry, regardless of which format they use for environment variables. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * Implement caching Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Update CLI documentation for registry commands Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Regenerate the swagger files Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> * Consolidate set-registry-api into set-registry Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> --------- Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com> Co-authored-by: Claude <noreply@anthropic.com>
1 parent cd66940 commit 8739ec4

File tree

85 files changed

+6427
-1056
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

85 files changed

+6427
-1056
lines changed

‎cmd/regup/app/update.go‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616

1717
"github.com/stacklok/toolhive/pkg/container/verifier"
1818
"github.com/stacklok/toolhive/pkg/logger"
19-
"github.com/stacklok/toolhive/pkg/registry"
19+
regtypes "github.com/stacklok/toolhive/pkg/registry/types"
2020
)
2121

2222
var (
@@ -29,7 +29,7 @@ var (
2929

3030
type serverWithName struct {
3131
name string
32-
server *registry.ImageMetadata
32+
server *regtypes.ImageMetadata
3333
}
3434

3535
// ProvenanceVerificationError represents an error during provenance verification
@@ -96,31 +96,31 @@ func updateCmdFunc(_ *cobra.Command, _ []string) error {
9696
return saveResults(reg, updatedServers, failedServers)
9797
}
9898

99-
func loadRegistry() (*registry.Registry, error) {
99+
func loadRegistry() (*regtypes.Registry, error) {
100100
registryPath := filepath.Join("pkg", "registry", "data", "registry.json")
101101
// #nosec G304 -- This is a known file path
102102
data, err := os.ReadFile(registryPath)
103103
if err != nil {
104104
return nil, fmt.Errorf("failed to read registry file: %w", err)
105105
}
106106

107-
var reg registry.Registry
107+
var reg regtypes.Registry
108108
if err := json.Unmarshal(data, &reg); err != nil {
109109
return nil, fmt.Errorf("failed to parse registry: %w", err)
110110
}
111111

112112
return &reg, nil
113113
}
114114

115-
func selectServersToUpdate(reg *registry.Registry) ([]serverWithName, error) {
115+
func selectServersToUpdate(reg *regtypes.Registry) ([]serverWithName, error) {
116116
if serverName != "" {
117117
return selectSpecificServer(reg, serverName)
118118
}
119119

120120
return selectOldestServers(reg)
121121
}
122122

123-
func selectSpecificServer(reg *registry.Registry, name string) ([]serverWithName, error) {
123+
func selectSpecificServer(reg *regtypes.Registry, name string) ([]serverWithName, error) {
124124
server, exists := reg.Servers[name]
125125
if !exists {
126126
return nil, fmt.Errorf("server '%s' not found in registry", name)
@@ -129,7 +129,7 @@ func selectSpecificServer(reg *registry.Registry, name string) ([]serverWithName
129129
return []serverWithName{{name: name, server: server}}, nil
130130
}
131131

132-
func selectOldestServers(reg *registry.Registry) ([]serverWithName, error) {
132+
func selectOldestServers(reg *regtypes.Registry) ([]serverWithName, error) {
133133
servers := make([]serverWithName, 0, len(reg.Servers))
134134
for name, server := range reg.Servers {
135135
server.Name = name
@@ -151,7 +151,7 @@ func selectOldestServers(reg *registry.Registry) ([]serverWithName, error) {
151151
return servers[:limit], nil
152152
}
153153

154-
func isOlder(serverI, serverJ *registry.ImageMetadata) bool {
154+
func isOlder(serverI, serverJ *regtypes.ImageMetadata) bool {
155155
var lastUpdatedI, lastUpdatedJ string
156156

157157
if serverI.Metadata != nil {
@@ -180,7 +180,7 @@ func isOlder(serverI, serverJ *registry.ImageMetadata) bool {
180180
return timeI.Before(timeJ)
181181
}
182182

183-
func updateServers(servers []serverWithName, reg *registry.Registry) ([]string, []string) {
183+
func updateServers(servers []serverWithName, reg *regtypes.Registry) ([]string, []string) {
184184
updatedServers := make([]string, 0, len(servers))
185185
failedServers := make([]string, 0)
186186

@@ -209,7 +209,7 @@ func updateServers(servers []serverWithName, reg *registry.Registry) ([]string,
209209
return updatedServers, failedServers
210210
}
211211

212-
func saveResults(reg *registry.Registry, updatedServers []string, failedServers []string) error {
212+
func saveResults(reg *regtypes.Registry, updatedServers []string, failedServers []string) error {
213213
// If we're in dry run mode, don't save changes
214214
if dryRun {
215215
logger.Info("Dry run completed, no changes made")
@@ -235,7 +235,7 @@ func saveResults(reg *registry.Registry, updatedServers []string, failedServers
235235
}
236236

237237
// updateServerInfo updates the GitHub stars and pulls for a server
238-
func updateServerInfo(name string, server *registry.ImageMetadata) error {
238+
func updateServerInfo(name string, server *regtypes.ImageMetadata) error {
239239
// Verify provenance if requested
240240
if verifyProvenance {
241241
if err := verifyServerProvenance(name, server); err != nil {
@@ -254,7 +254,7 @@ func updateServerInfo(name string, server *registry.ImageMetadata) error {
254254

255255
// Initialize metadata if it's nil
256256
if server.Metadata == nil {
257-
server.Metadata = &registry.Metadata{}
257+
server.Metadata = &regtypes.Metadata{}
258258
}
259259

260260
// Extract owner and repo from repository URL
@@ -289,7 +289,7 @@ func updateServerInfo(name string, server *registry.ImageMetadata) error {
289289
}
290290

291291
// verifyServerProvenance verifies the provenance information for a server
292-
func verifyServerProvenance(name string, server *registry.ImageMetadata) error {
292+
func verifyServerProvenance(name string, server *regtypes.ImageMetadata) error {
293293
// Skip if no provenance information
294294
if server.Provenance == nil {
295295
logger.Warnf("Server %s has no provenance information, skipping verification", name)
@@ -325,7 +325,7 @@ func verifyServerProvenance(name string, server *registry.ImageMetadata) error {
325325
}
326326

327327
// removeFailedServers removes servers that failed provenance verification from the registry
328-
func removeFailedServers(reg *registry.Registry, failedServers []string) {
328+
func removeFailedServers(reg *regtypes.Registry, failedServers []string) {
329329
for _, serverName := range failedServers {
330330
logger.Warnf("Removing server %s from registry due to provenance verification failure", serverName)
331331
delete(reg.Servers, serverName)
@@ -420,7 +420,7 @@ func getGitHubRepoInfo(owner, repo, serverName string, currentPulls int) (stars
420420
}
421421

422422
// saveRegistry saves the registry to the filesystem while preserving the order of entries
423-
func saveRegistry(reg *registry.Registry, updatedServers []string, failedServers []string) error {
423+
func saveRegistry(reg *regtypes.Registry, updatedServers []string, failedServers []string) error {
424424
// Find the registry file path
425425
registryPath := filepath.Join("pkg", "registry", "data", "registry.json")
426426

‎cmd/regup/app/update_test.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"github.com/stretchr/testify/require"
1111

1212
"github.com/stacklok/toolhive/pkg/logger"
13-
"github.com/stacklok/toolhive/pkg/registry"
13+
regtypes "github.com/stacklok/toolhive/pkg/registry/types"
1414
)
1515

1616
//nolint:paralleltest // This test manages temporary directories and cannot run in parallel
@@ -141,7 +141,7 @@ func TestServerSelection(t *testing.T) {
141141
data, err := os.ReadFile(registryPath)
142142
require.NoError(t, err)
143143

144-
var reg registry.Registry
144+
var reg regtypes.Registry
145145
err = json.Unmarshal(data, &reg)
146146
require.NoError(t, err)
147147

@@ -170,15 +170,15 @@ func setupTestRegistryWithMultipleServers(t *testing.T) (string, func()) {
170170
require.NoError(t, err)
171171

172172
// Create test registry with multiple servers
173-
testRegistry := &registry.Registry{
173+
testRegistry := &regtypes.Registry{
174174
LastUpdated: "2025-06-16T12:00:00Z",
175-
Servers: map[string]*registry.ImageMetadata{
175+
Servers: map[string]*regtypes.ImageMetadata{
176176
"github": {
177-
BaseServerMetadata: registry.BaseServerMetadata{
177+
BaseServerMetadata: regtypes.BaseServerMetadata{
178178
Name: "github",
179179
Description: "GitHub MCP server",
180180
RepositoryURL: "https://github.com/github/github-mcp-server",
181-
Metadata: &registry.Metadata{
181+
Metadata: &regtypes.Metadata{
182182
Stars: 100,
183183
Pulls: 5000,
184184
LastUpdated: "2025-06-16T12:00:00Z", // Older
@@ -187,11 +187,11 @@ func setupTestRegistryWithMultipleServers(t *testing.T) (string, func()) {
187187
Image: "ghcr.io/github/github-mcp-server:latest",
188188
},
189189
"gitlab": {
190-
BaseServerMetadata: registry.BaseServerMetadata{
190+
BaseServerMetadata: regtypes.BaseServerMetadata{
191191
Name: "gitlab",
192192
Description: "GitLab MCP server",
193193
RepositoryURL: "https://github.com/example/gitlab-mcp-server",
194-
Metadata: &registry.Metadata{
194+
Metadata: &regtypes.Metadata{
195195
Stars: 50,
196196
Pulls: 2000,
197197
LastUpdated: "2025-06-17T12:00:00Z", // Newer
@@ -200,11 +200,11 @@ func setupTestRegistryWithMultipleServers(t *testing.T) (string, func()) {
200200
Image: "mcp/gitlab:latest",
201201
},
202202
"fetch": {
203-
BaseServerMetadata: registry.BaseServerMetadata{
203+
BaseServerMetadata: regtypes.BaseServerMetadata{
204204
Name: "fetch",
205205
Description: "Fetch MCP server",
206206
RepositoryURL: "https://github.com/example/fetch-mcp-server",
207-
Metadata: &registry.Metadata{
207+
Metadata: &regtypes.Metadata{
208208
Stars: 25,
209209
Pulls: 1000,
210210
LastUpdated: "2025-06-15T12:00:00Z", // Oldest
@@ -241,9 +241,9 @@ func setupEmptyTestRegistry(t *testing.T) (string, func()) {
241241
require.NoError(t, err)
242242

243243
// Create empty test registry
244-
testRegistry := &registry.Registry{
244+
testRegistry := &regtypes.Registry{
245245
LastUpdated: "2025-06-16T12:00:00Z",
246-
Servers: map[string]*registry.ImageMetadata{},
246+
Servers: map[string]*regtypes.ImageMetadata{},
247247
}
248248

249249
// Write registry file

‎cmd/thv-operator/pkg/filtering/filter_service.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,13 @@ import (
88
"sigs.k8s.io/controller-runtime/pkg/log"
99

1010
mcpv1alpha1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1alpha1"
11-
"github.com/stacklok/toolhive/pkg/registry"
11+
regtypes "github.com/stacklok/toolhive/pkg/registry/types"
1212
)
1313

1414
// FilterService coordinates name and tag filtering to apply registry filters
1515
type FilterService interface {
1616
// ApplyFilters filters the registry based on MCPRegistry filter configuration
17-
ApplyFilters(ctx context.Context, reg *registry.Registry, filter *mcpv1alpha1.RegistryFilter) (*registry.Registry, error)
17+
ApplyFilters(ctx context.Context, reg *regtypes.Registry, filter *mcpv1alpha1.RegistryFilter) (*regtypes.Registry, error)
1818
}
1919

2020
// DefaultFilterService implements filtering coordination using name and tag filters
@@ -49,8 +49,8 @@ func NewFilterService(nameFilter NameFilter, tagFilter TagFilter) *DefaultFilter
4949
// 5. Return the filtered registry
5050
func (s *DefaultFilterService) ApplyFilters(
5151
ctx context.Context,
52-
reg *registry.Registry,
53-
filter *mcpv1alpha1.RegistryFilter) (*registry.Registry, error) {
52+
reg *regtypes.Registry,
53+
filter *mcpv1alpha1.RegistryFilter) (*regtypes.Registry, error) {
5454
ctxLogger := log.FromContext(ctx)
5555

5656
// If no filter is specified, return original registry
@@ -64,11 +64,11 @@ func (s *DefaultFilterService) ApplyFilters(
6464
"originalRemoteServerCount", len(reg.RemoteServers))
6565

6666
// Create a new filtered registry with same metadata
67-
filteredRegistry := &registry.Registry{
67+
filteredRegistry := &regtypes.Registry{
6868
Version: reg.Version,
6969
LastUpdated: reg.LastUpdated,
70-
Servers: make(map[string]*registry.ImageMetadata),
71-
RemoteServers: make(map[string]*registry.RemoteServerMetadata),
70+
Servers: make(map[string]*regtypes.ImageMetadata),
71+
RemoteServers: make(map[string]*regtypes.RemoteServerMetadata),
7272
Groups: reg.Groups, // Groups are not filtered for now
7373
}
7474

0 commit comments

Comments
 (0)