This commit is contained in:
mangomqy
2025-11-13 02:54:06 +00:00
commit c5e51ed069
254 changed files with 54901 additions and 0 deletions

View File

@ -0,0 +1,284 @@
package mock
import (
"context"
"fmt"
"strings"
"time"
"github.com/ocdp/cluster-service/internal/domain/entity"
"github.com/ocdp/cluster-service/internal/domain/repository"
)
// OCIClientMock OCI Registry 客户端 Mock 实现
type OCIClientMock struct {
// Mock 数据存储
repositories map[string][]string // registryID -> []repositoryName
artifacts map[string]map[string][]*entity.Artifact // registryID -> repository -> []artifact
}
// NewOCIClientMock 创建 Mock 实现
func NewOCIClientMock() repository.OCIClient {
mock := &OCIClientMock{
repositories: make(map[string][]string),
artifacts: make(map[string]map[string][]*entity.Artifact),
}
// 初始化一些测试数据
mock.initMockData()
return mock
}
func (c *OCIClientMock) initMockData() {
// Note: This method intentionally left empty
// Mock data will be generated dynamically per registry to support any registry ID
}
// initArtifactsForRegistry initializes mock artifacts for a given registry ID
func (c *OCIClientMock) initArtifactsForRegistry(registryID string) {
c.artifacts[registryID] = make(map[string][]*entity.Artifact)
// vllm-serve artifacts (OCI 格式的 Helm Chart)
c.artifacts[registryID]["charts/vllm-serve"] = []*entity.Artifact{
{
RegistryID: registryID,
Repository: "charts/vllm-serve",
Tag: "0.1.0",
Digest: "sha256:abc123def456",
Type: entity.ArtifactTypeChart,
Size: 12345678,
MediaType: "application/vnd.oci.image.manifest.v1+json",
ConfigType: "application/vnd.cncf.helm.config.v1+json", // Helm Chart 的 config type
Annotations: map[string]string{
"org.opencontainers.image.title": "vllm-serve",
"org.opencontainers.image.version": "0.1.0",
},
CreatedAt: time.Now().Add(-24 * time.Hour),
},
{
RegistryID: registryID,
Repository: "charts/vllm-serve",
Tag: "0.2.0",
Digest: "sha256:xyz789uvw012",
Type: entity.ArtifactTypeChart,
Size: 13456789,
MediaType: "application/vnd.oci.image.manifest.v1+json",
ConfigType: "application/vnd.cncf.helm.config.v1+json", // Helm Chart 的 config type
Annotations: map[string]string{
"org.opencontainers.image.title": "vllm-serve",
"org.opencontainers.image.version": "0.2.0",
},
CreatedAt: time.Now(),
},
}
// nginx artifacts (OCI 格式的 Helm Chart)
c.artifacts[registryID]["charts/nginx"] = []*entity.Artifact{
{
RegistryID: registryID,
Repository: "charts/nginx",
Tag: "1.0.0",
Digest: "sha256:nginx123456",
Type: entity.ArtifactTypeChart,
Size: 5678901,
MediaType: "application/vnd.oci.image.manifest.v1+json",
ConfigType: "application/vnd.cncf.helm.config.v1+json", // Helm Chart 的 config type
Annotations: map[string]string{
"org.opencontainers.image.title": "nginx",
},
CreatedAt: time.Now().Add(-48 * time.Hour),
},
}
// redis artifacts (OCI 格式的 Helm Chart)
c.artifacts[registryID]["charts/redis"] = []*entity.Artifact{
{
RegistryID: registryID,
Repository: "charts/redis",
Tag: "6.2.0",
Digest: "sha256:redis789abc",
Type: entity.ArtifactTypeChart,
Size: 8901234,
MediaType: "application/vnd.oci.image.manifest.v1+json",
ConfigType: "application/vnd.cncf.helm.config.v1+json", // Helm Chart 的 config type
Annotations: map[string]string{
"org.opencontainers.image.title": "redis",
"org.opencontainers.image.version": "6.2.0",
},
CreatedAt: time.Now().Add(-72 * time.Hour),
},
}
// alpine artifacts (Docker Image)
c.artifacts[registryID]["library/alpine"] = []*entity.Artifact{
{
RegistryID: registryID,
Repository: "library/alpine",
Tag: "3.18",
Digest: "sha256:alpine123",
Type: entity.ArtifactTypeImage,
Size: 2345678,
MediaType: "application/vnd.docker.distribution.manifest.v2+json",
ConfigType: "application/vnd.docker.container.image.v1+json", // Docker Image 的 config type
Annotations: map[string]string{
"org.opencontainers.image.title": "alpine",
"org.opencontainers.image.version": "3.18",
},
CreatedAt: time.Now().Add(-96 * time.Hour),
},
{
RegistryID: registryID,
Repository: "library/alpine",
Tag: "latest",
Digest: "sha256:alpine456",
Type: entity.ArtifactTypeImage,
Size: 2456789,
MediaType: "application/vnd.docker.distribution.manifest.v2+json",
ConfigType: "application/vnd.docker.container.image.v1+json", // Docker Image 的 config type
Annotations: map[string]string{
"org.opencontainers.image.title": "alpine",
},
CreatedAt: time.Now().Add(-24 * time.Hour),
},
}
}
func (c *OCIClientMock) ListRepositories(ctx context.Context, registry *entity.Registry) ([]string, error) {
// Check if we have cached data for this registry
repos, exists := c.repositories[registry.ID]
if !exists {
// Generate mock data dynamically for any registry
repos = []string{
"charts/vllm-serve",
"charts/nginx",
"charts/redis",
"library/alpine",
}
c.repositories[registry.ID] = repos
// Also initialize artifacts for this registry
c.initArtifactsForRegistry(registry.ID)
}
return repos, nil
}
func (c *OCIClientMock) ListArtifacts(ctx context.Context, registry *entity.Registry, repository, mediaTypeFilter string) ([]*entity.Artifact, error) {
regArtifacts, exists := c.artifacts[registry.ID]
if !exists {
// Initialize artifacts for this registry if not exists
c.initArtifactsForRegistry(registry.ID)
regArtifacts = c.artifacts[registry.ID]
}
artifacts, exists := regArtifacts[repository]
if !exists {
return []*entity.Artifact{}, nil
}
// 应用 mediaType 过滤
if mediaTypeFilter == "" || mediaTypeFilter == "all" {
return artifacts, nil
}
filtered := make([]*entity.Artifact, 0)
filter := strings.ToLower(strings.TrimSpace(mediaTypeFilter))
for _, artifact := range artifacts {
switch filter {
case "chart":
if artifact.Type == entity.ArtifactTypeChart {
filtered = append(filtered, artifact)
}
case "image":
if artifact.Type == entity.ArtifactTypeImage {
filtered = append(filtered, artifact)
}
case "other":
if artifact.Type == entity.ArtifactTypeOther {
filtered = append(filtered, artifact)
}
}
}
return filtered, nil
}
func (c *OCIClientMock) GetArtifact(ctx context.Context, registry *entity.Registry, repository, reference string) (*entity.Artifact, error) {
regArtifacts, exists := c.artifacts[registry.ID]
if !exists {
// Initialize artifacts for this registry if not exists
c.initArtifactsForRegistry(registry.ID)
regArtifacts = c.artifacts[registry.ID]
}
artifacts, exists := regArtifacts[repository]
if !exists {
return nil, entity.ErrArtifactNotFound
}
// 根据 tag 或 digest 查找
for _, artifact := range artifacts {
if artifact.Tag == reference || artifact.Digest == reference {
return artifact, nil
}
}
return nil, entity.ErrArtifactNotFound
}
func (c *OCIClientMock) GetValuesSchema(ctx context.Context, registry *entity.Registry, repository, reference string) (string, error) {
artifact, err := c.GetArtifact(ctx, registry, repository, reference)
if err != nil {
return "", err
}
if !artifact.IsChart() {
return "", fmt.Errorf("not a helm chart")
}
// 返回 Mock values schema
mockSchema := `{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"properties": {
"replicaCount": {
"type": "integer",
"default": 1
},
"image": {
"type": "object",
"properties": {
"repository": {
"type": "string"
},
"tag": {
"type": "string"
}
}
}
}
}`
return mockSchema, nil
}
func (c *OCIClientMock) PullArtifact(ctx context.Context, registry *entity.Registry, repository, reference, destPath string) error {
_, err := c.GetArtifact(ctx, registry, repository, reference)
if err != nil {
return err
}
// Mock 实现,不实际下载
return nil
}
func (c *OCIClientMock) PushArtifact(ctx context.Context, registry *entity.Registry, repository, tag, sourcePath string) error {
// Mock 实现,不实际上传
return nil
}
func (c *OCIClientMock) CheckHealth(ctx context.Context, registry *entity.Registry) error {
// Mock 实现,总是返回健康
return nil
}

View File

@ -0,0 +1,468 @@
package real
import (
"archive/tar"
"compress/gzip"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/ocdp/cluster-service/internal/domain/entity"
"github.com/ocdp/cluster-service/internal/domain/repository"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/registry/remote"
"oras.land/oras-go/v2/registry/remote/auth"
)
// OCIClient 真实的 OCI 客户端实现(使用 ORAS
type OCIClient struct {
httpClient *http.Client
}
// NewOCIClient 创建真实的 OCI 客户端
func NewOCIClient() repository.OCIClient {
return &OCIClient{
httpClient: &http.Client{},
}
}
// getRegistry 创建 ORAS Registry 客户端
func (c *OCIClient) getRegistry(reg *entity.Registry) (*remote.Registry, error) {
// 解析 Registry URL
registryURL := strings.TrimPrefix(reg.URL, "https://")
registryURL = strings.TrimPrefix(registryURL, "http://")
registry, err := remote.NewRegistry(registryURL)
if err != nil {
return nil, fmt.Errorf("failed to create registry client: %w", err)
}
// 设置认证
if reg.Username != "" && reg.Password != "" {
registry.Client = &auth.Client{
Client: c.httpClient,
Credential: auth.StaticCredential(registryURL, auth.Credential{
Username: reg.Username,
Password: reg.Password,
}),
}
}
// 设置 PlainHTTP如果是 insecure
registry.PlainHTTP = reg.Insecure
return registry, nil
}
// ListRepositories 列出 Registry 中的所有 repositories
func (c *OCIClient) ListRepositories(ctx context.Context, registry *entity.Registry) ([]string, error) {
reg, err := c.getRegistry(registry)
if err != nil {
return nil, err
}
repositories := make([]string, 0)
err = reg.Repositories(ctx, "", func(repos []string) error {
repositories = append(repositories, repos...)
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to list repositories: %w", err)
}
return repositories, nil
}
// ListArtifacts 列出指定 repository 的所有 artifacts
// mediaTypeFilter: "all", "image", "chart", "other" - 使用模糊匹配过滤
func (c *OCIClient) ListArtifacts(ctx context.Context, registry *entity.Registry, repository, mediaTypeFilter string) ([]*entity.Artifact, error) {
reg, err := c.getRegistry(registry)
if err != nil {
return nil, err
}
repo, err := reg.Repository(ctx, repository)
if err != nil {
return nil, fmt.Errorf("failed to get repository: %w", err)
}
artifacts := make([]*entity.Artifact, 0)
err = repo.Tags(ctx, "", func(tags []string) error {
for _, tag := range tags {
// 获取 manifest 以获取更多信息
desc, err := repo.Resolve(ctx, tag)
if err != nil {
// 跳过无法解析的 tag
continue
}
artifact := &entity.Artifact{
Repository: repository,
Tag: tag,
Digest: desc.Digest.String(),
MediaType: desc.MediaType,
Size: desc.Size,
}
// 尝试获取 config.mediaType 以更准确判断类型
if manifestBytes, err := repo.Fetch(ctx, desc); err == nil {
defer manifestBytes.Close()
if manifestData, err := io.ReadAll(manifestBytes); err == nil {
var manifest map[string]interface{}
if err := json.Unmarshal(manifestData, &manifest); err == nil {
// 获取 config.mediaType
if config, ok := manifest["config"].(map[string]interface{}); ok {
if configMediaType, ok := config["mediaType"].(string); ok {
artifact.ConfigType = configMediaType
}
}
}
}
}
// 使用智能类型判断(综合多种信息)
artifact.DetermineType()
// 应用 mediaType 过滤
if c.shouldIncludeArtifact(artifact, mediaTypeFilter) {
artifacts = append(artifacts, artifact)
}
}
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to list artifacts: %w", err)
}
return artifacts, nil
}
// shouldIncludeArtifact 判断是否应该包含该 artifact
func (c *OCIClient) shouldIncludeArtifact(artifact *entity.Artifact, filter string) bool {
// 默认或 "all" 返回所有
if filter == "" || filter == "all" {
return true
}
filter = strings.ToLower(strings.TrimSpace(filter))
switch filter {
case "chart":
// 只返回 Helm Charts
return artifact.Type == entity.ArtifactTypeChart
case "image":
// 返回 Docker 或 OCI images
return artifact.Type == entity.ArtifactTypeImage
case "other":
// 返回其他类型
return artifact.Type == entity.ArtifactTypeOther
default:
// 未知的 filter返回所有
return true
}
}
// GetArtifact 获取指定 artifact 的详细信息
func (c *OCIClient) GetArtifact(ctx context.Context, registry *entity.Registry, repository, reference string) (*entity.Artifact, error) {
reg, err := c.getRegistry(registry)
if err != nil {
return nil, err
}
repo, err := reg.Repository(ctx, repository)
if err != nil {
return nil, fmt.Errorf("failed to get repository: %w", err)
}
// 解析 reference
desc, err := repo.Resolve(ctx, reference)
if err != nil {
return nil, fmt.Errorf("failed to resolve artifact: %w", err)
}
// 获取 manifest
manifestBytes, err := repo.Fetch(ctx, desc)
if err != nil {
return nil, fmt.Errorf("failed to fetch manifest: %w", err)
}
defer manifestBytes.Close()
manifestData, err := io.ReadAll(manifestBytes)
if err != nil {
return nil, fmt.Errorf("failed to read manifest: %w", err)
}
// 解析 manifest 获取配置信息
var manifest map[string]interface{}
if err := json.Unmarshal(manifestData, &manifest); err != nil {
return nil, fmt.Errorf("failed to unmarshal manifest: %w", err)
}
artifact := &entity.Artifact{
Repository: repository,
Tag: reference,
Digest: desc.Digest.String(),
MediaType: desc.MediaType,
Size: desc.Size,
Annotations: make(map[string]string),
}
// 获取 config.mediaType 和 annotations
if config, ok := manifest["config"].(map[string]interface{}); ok {
// 获取 config.mediaType用于准确的类型判断
if configMediaType, ok := config["mediaType"].(string); ok {
artifact.ConfigType = configMediaType
}
// 获取 annotations
if annotations, ok := config["annotations"].(map[string]interface{}); ok {
for k, v := range annotations {
if str, ok := v.(string); ok {
artifact.Annotations[k] = str
}
}
}
}
// 使用智能类型判断(综合 ConfigType, Annotations, Repository 名称等)
artifact.DetermineType()
return artifact, nil
}
// GetValuesSchema 获取 Helm Chart 的 values schema
func (c *OCIClient) GetValuesSchema(ctx context.Context, registry *entity.Registry, repository, reference string) (string, error) {
reg, err := c.getRegistry(registry)
if err != nil {
return "", err
}
repo, err := reg.Repository(ctx, repository)
if err != nil {
return "", fmt.Errorf("failed to get repository: %w", err)
}
// 解析 reference (tag 或 digest)
desc, err := repo.Resolve(ctx, reference)
if err != nil {
return "", fmt.Errorf("failed to resolve artifact: %w", err)
}
manifestReader, err := repo.Fetch(ctx, desc)
if err != nil {
return "", fmt.Errorf("failed to fetch manifest: %w", err)
}
defer manifestReader.Close()
manifestBytes, err := io.ReadAll(manifestReader)
if err != nil {
return "", fmt.Errorf("failed to read manifest: %w", err)
}
var manifest ocispec.Manifest
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
return "", fmt.Errorf("failed to unmarshal manifest: %w", err)
}
// 优先查找是否存在独立的 values schema layer一些 registry 会将 values.schema.json 作为单独的 layer 存储)
var valuesSchemaLayer *ocispec.Descriptor
for i := range manifest.Layers {
layer := manifest.Layers[i]
mediaType := strings.ToLower(layer.MediaType)
if strings.Contains(mediaType, "helm.values.schema") ||
strings.Contains(mediaType, "values.schema") {
valuesSchemaLayer = &manifest.Layers[i]
break
}
}
// 如果存在独立的 values schema layer直接返回
if valuesSchemaLayer != nil {
reader, err := repo.Fetch(ctx, *valuesSchemaLayer)
if err != nil {
return "", fmt.Errorf("failed to fetch values schema layer: %w", err)
}
defer reader.Close()
data, err := io.ReadAll(reader)
if err != nil {
return "", fmt.Errorf("failed to read values schema layer: %w", err)
}
if len(data) == 0 {
return "", entity.ErrValuesSchemaNotFound
}
return string(data), nil
}
// 回退:查找 Helm Chart layertar+gzip 包含 chart 内容)并从中读取 values.schema.json
var chartLayer *ocispec.Descriptor
for i := range manifest.Layers {
layer := manifest.Layers[i]
if strings.Contains(layer.MediaType, "cncf.helm.chart") ||
strings.Contains(layer.MediaType, "helm.chart.content") {
chartLayer = &manifest.Layers[i]
break
}
}
if chartLayer == nil {
return "", entity.ErrValuesSchemaNotFound
}
if chartLayer.Digest == "" {
return "", fmt.Errorf("chart layer digest is empty")
}
if _, err := digest.Parse(string(chartLayer.Digest)); err != nil {
return "", fmt.Errorf("invalid chart layer digest: %w", err)
}
layerReader, err := repo.Fetch(ctx, *chartLayer)
if err != nil {
return "", fmt.Errorf("failed to fetch chart layer: %w", err)
}
defer layerReader.Close()
gzipReader, err := gzip.NewReader(layerReader)
if err != nil {
return "", fmt.Errorf("failed to create gzip reader: %w", err)
}
defer gzipReader.Close()
tarReader := tar.NewReader(gzipReader)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return "", fmt.Errorf("failed to read chart archive: %w", err)
}
if header.Typeflag != tar.TypeReg {
continue
}
if strings.HasSuffix(header.Name, "values.schema.json") {
data, err := io.ReadAll(tarReader)
if err != nil {
return "", fmt.Errorf("failed to read values.schema.json: %w", err)
}
if len(data) == 0 {
return "", entity.ErrValuesSchemaNotFound
}
return string(data), nil
}
}
return "", entity.ErrValuesSchemaNotFound
}
// PullArtifact 下载 artifact 到本地
func (c *OCIClient) PullArtifact(ctx context.Context, registry *entity.Registry, repository, reference, destPath string) error {
reg, err := c.getRegistry(registry)
if err != nil {
return err
}
repo, err := reg.Repository(ctx, repository)
if err != nil {
return fmt.Errorf("failed to get repository: %w", err)
}
// 解析 reference
desc, err := repo.Resolve(ctx, reference)
if err != nil {
return fmt.Errorf("failed to resolve artifact: %w", err)
}
// 获取 manifest 内容
manifestReader, err := repo.Fetch(ctx, desc)
if err != nil {
return fmt.Errorf("failed to fetch manifest: %w", err)
}
defer manifestReader.Close()
manifestBytes, err := io.ReadAll(manifestReader)
if err != nil {
return fmt.Errorf("failed to read manifest: %w", err)
}
var manifest ocispec.Manifest
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
return fmt.Errorf("failed to unmarshal manifest: %w", err)
}
var chartLayer *ocispec.Descriptor
for i := range manifest.Layers {
layer := manifest.Layers[i]
if strings.Contains(layer.MediaType, "cncf.helm.chart") ||
strings.Contains(layer.MediaType, "helm.chart.content") {
chartLayer = &layer
break
}
}
if chartLayer == nil {
return fmt.Errorf("helm chart layer not found in manifest")
}
content, err := repo.Fetch(ctx, *chartLayer)
if err != nil {
return fmt.Errorf("failed to fetch chart layer: %w", err)
}
defer content.Close()
// 确保目标目录存在
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
return fmt.Errorf("failed to create destination directory: %w", err)
}
// 写入文件
file, err := os.Create(destPath)
if err != nil {
return fmt.Errorf("failed to create file: %w", err)
}
defer file.Close()
if _, err := io.Copy(file, content); err != nil {
return fmt.Errorf("failed to write artifact: %w", err)
}
return nil
}
// PushArtifact 推送 artifact 到 Registry
func (c *OCIClient) PushArtifact(ctx context.Context, registry *entity.Registry, repository, tag, sourcePath string) error {
// 这是一个简化实现
// 实际应该实现完整的 OCI artifact push 流程
return fmt.Errorf("push artifact not fully implemented yet")
}
// CheckHealth 检查 Registry 健康状态
func (c *OCIClient) CheckHealth(ctx context.Context, registry *entity.Registry) error {
reg, err := c.getRegistry(registry)
if err != nil {
return err
}
// 尝试 ping registry
err = reg.Ping(ctx)
if err != nil {
return fmt.Errorf("registry health check failed: %w", err)
}
return nil
}