From f55af4675cbcb0170693caaeb5faa7707836de4c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 7 Aug 2022 12:09:54 +0200 Subject: [PATCH] Add support for Pub packages (#20560) * Added support for Pub packages. * Update docs/content/doc/packages/overview.en-us.md Co-authored-by: Gergely Nagy Co-authored-by: Lunny Xiao Co-authored-by: Gergely Nagy Co-authored-by: Lauris BH --- docs/content/doc/packages/overview.en-us.md | 1 + docs/content/doc/packages/pub.en-us.md | 83 ++++++ docs/content/doc/packages/pypi.en-us.md | 2 +- docs/content/doc/packages/rubygems.en-us.md | 2 +- integrations/api_packages_pub_test.go | 179 +++++++++++++ models/packages/descriptor.go | 3 + models/packages/package.go | 5 + modules/packages/pub/metadata.go | 154 +++++++++++ modules/packages/pub/metadata_test.go | 136 ++++++++++ options/locale/locale_en-US.ini | 4 + public/img/svg/gitea-pub.svg | 1 + routers/api/packages/api.go | 15 ++ routers/api/packages/nuget/nuget.go | 2 +- routers/api/packages/pub/pub.go | 275 ++++++++++++++++++++ routers/api/v1/packages/package.go | 2 +- templates/admin/packages/list.tmpl | 1 + templates/package/content/pub.tmpl | 19 ++ templates/package/metadata/pub.tmpl | 5 + templates/package/shared/list.tmpl | 1 + templates/package/view.tmpl | 2 + templates/swagger/v1_json.tmpl | 1 + web_src/svg/gitea-pub.svg | 19 ++ 22 files changed, 908 insertions(+), 4 deletions(-) create mode 100644 docs/content/doc/packages/pub.en-us.md create mode 100644 integrations/api_packages_pub_test.go create mode 100644 modules/packages/pub/metadata.go create mode 100644 modules/packages/pub/metadata_test.go create mode 100644 public/img/svg/gitea-pub.svg create mode 100644 routers/api/packages/pub/pub.go create mode 100644 templates/package/content/pub.tmpl create mode 100644 templates/package/metadata/pub.tmpl create mode 100644 web_src/svg/gitea-pub.svg diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index 81575b9ade..5e03b71017 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -34,6 +34,7 @@ The following package managers are currently supported: | [Maven]({{< relref "doc/packages/maven.en-us.md" >}}) | Java | `mvn`, `gradle` | | [npm]({{< relref "doc/packages/npm.en-us.md" >}}) | JavaScript | `npm`, `yarn` | | [NuGet]({{< relref "doc/packages/nuget.en-us.md" >}}) | .NET | `nuget` | +| [Pub]({{< relref "doc/packages/pub.en-us.md" >}}) | Dart | `dart`, `flutter` | | [PyPI]({{< relref "doc/packages/pypi.en-us.md" >}}) | Python | `pip`, `twine` | | [RubyGems]({{< relref "doc/packages/rubygems.en-us.md" >}}) | Ruby | `gem`, `Bundler` | diff --git a/docs/content/doc/packages/pub.en-us.md b/docs/content/doc/packages/pub.en-us.md new file mode 100644 index 0000000000..4d37662208 --- /dev/null +++ b/docs/content/doc/packages/pub.en-us.md @@ -0,0 +1,83 @@ +--- +date: "2022-07-31T00:00:00+00:00" +title: "Pub Packages Repository" +slug: "packages/pub" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Pub" + weight: 90 + identifier: "pub" +--- + +# Pub Packages Repository + +Publish [Pub](https://dart.dev/guides/packages) packages for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Pub package registry, you need to use the tools [dart](https://dart.dev/tools/dart-tool) and/or [flutter](https://docs.flutter.dev/reference/flutter-cli). + +The following examples use dart. + +## Configuring the package registry + +To register the package registry and provide credentials, execute: + +```shell +dart pub token add https://gitea.example.com/api/packages/{owner}/pub +``` + +| Placeholder | Description | +| ------------ | ----------- | +| `owner` | The owner of the package. | + +You need to provide your [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}). + +## Publish a package + +To publish a package, edit the `pubspec.yaml` and add the following line: + +```yaml +publish_to: https://gitea.example.com/api/packages/{owner}/pub +``` + +| Placeholder | Description | +| ------------ | ----------- | +| `owner` | The owner of the package. | + +Now you can publish the package by running the following command: + +```shell +dart pub publish +``` + +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +## Install a package + +To install a Pub package from the package registry, execute the following command: + +```shell +dart pub add {package_name} --hosted-url=https://gitea.example.com/api/packages/{owner}/pub/ +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `owner` | The owner of the package. | +| `package_name` | The package name. | + +For example: + +```shell +# use latest version +dart pub add mypackage --hosted-url=https://gitea.example.com/api/packages/testuser/pub/ +# specify version +dart pub add mypackage:1.0.8 --hosted-url=https://gitea.example.com/api/packages/testuser/pub/ +``` diff --git a/docs/content/doc/packages/pypi.en-us.md b/docs/content/doc/packages/pypi.en-us.md index af17fe8310..588df71d60 100644 --- a/docs/content/doc/packages/pypi.en-us.md +++ b/docs/content/doc/packages/pypi.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "PyPI" - weight: 90 + weight: 100 identifier: "pypi" --- diff --git a/docs/content/doc/packages/rubygems.en-us.md b/docs/content/doc/packages/rubygems.en-us.md index dd7ac9ee7e..d4ae30bbce 100644 --- a/docs/content/doc/packages/rubygems.en-us.md +++ b/docs/content/doc/packages/rubygems.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "RubyGems" - weight: 100 + weight: 110 identifier: "rubygems" --- diff --git a/integrations/api_packages_pub_test.go b/integrations/api_packages_pub_test.go new file mode 100644 index 0000000000..d64f88def7 --- /dev/null +++ b/integrations/api_packages_pub_test.go @@ -0,0 +1,179 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + pub_module "code.gitea.io/gitea/modules/packages/pub" + + "github.com/stretchr/testify/assert" +) + +func TestPackagePub(t *testing.T) { + defer prepareTestEnv(t)() + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}).(*user_model.User) + + token := "Bearer " + getUserToken(t, user.Name) + + packageName := "test_package" + packageVersion := "1.0.1" + packageDescription := "Test Description" + + filename := fmt.Sprintf("%s.tar.gz", packageVersion) + + pubspecContent := `name: ` + packageName + ` +version: ` + packageVersion + ` +description: ` + packageDescription + + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + archive := tar.NewWriter(zw) + archive.WriteHeader(&tar.Header{ + Name: "pubspec.yaml", + Mode: 0o600, + Size: int64(len(pubspecContent)), + }) + archive.Write([]byte(pubspecContent)) + archive.Close() + zw.Close() + content := buf.Bytes() + + root := fmt.Sprintf("/api/packages/%s/pub", user.Name) + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + uploadURL := root + "/api/packages/versions/new" + + req := NewRequest(t, "GET", uploadURL) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequest(t, "GET", uploadURL) + addTokenAuthHeader(req, token) + resp := MakeRequest(t, req, http.StatusOK) + + type UploadRequest struct { + URL string `json:"url"` + Fields map[string]string `json:"fields"` + } + + var result UploadRequest + DecodeJSON(t, resp, &result) + + assert.Empty(t, result.Fields) + + uploadFile := func(t *testing.T, url string, content []byte, expectedStatus int) *httptest.ResponseRecorder { + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, _ := writer.CreateFormFile("file", "dummy.tar.gz") + _, _ = io.Copy(part, bytes.NewReader(content)) + + _ = writer.Close() + + req := NewRequestWithBody(t, "POST", url, body) + req.Header.Add("Content-Type", writer.FormDataContentType()) + addTokenAuthHeader(req, token) + return MakeRequest(t, req, expectedStatus) + } + + resp = uploadFile(t, result.URL, content, http.StatusNoContent) + + req = NewRequest(t, "GET", resp.Header().Get("Location")) + addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusOK) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypePub) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &pub_module.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, filename, pfs[0].Name) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), pb.Size) + + resp = uploadFile(t, result.URL, content, http.StatusBadRequest) + }) + + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/api/packages/%s/%s", root, packageName, packageVersion)) + resp := MakeRequest(t, req, http.StatusOK) + + type VersionMetadata struct { + Version string `json:"version"` + ArchiveURL string `json:"archive_url"` + Published time.Time `json:"published"` + Pubspec interface{} `json:"pubspec,omitempty"` + } + + var result VersionMetadata + DecodeJSON(t, resp, &result) + + assert.Equal(t, packageVersion, result.Version) + assert.NotNil(t, result.Pubspec) + + req = NewRequest(t, "GET", result.ArchiveURL) + resp = MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, content, resp.Body.Bytes()) + }) + + t.Run("EnumeratePackageVersions", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/api/packages/%s", root, packageName)) + resp := MakeRequest(t, req, http.StatusOK) + + type VersionMetadata struct { + Version string `json:"version"` + ArchiveURL string `json:"archive_url"` + Published time.Time `json:"published"` + Pubspec interface{} `json:"pubspec,omitempty"` + } + + type PackageVersions struct { + Name string `json:"name"` + Latest *VersionMetadata `json:"latest"` + Versions []*VersionMetadata `json:"versions"` + } + + var result PackageVersions + DecodeJSON(t, resp, &result) + + assert.Equal(t, packageName, result.Name) + assert.NotNil(t, result.Latest) + assert.Len(t, result.Versions, 1) + assert.Equal(t, result.Latest.Version, result.Versions[0].Version) + assert.Equal(t, packageVersion, result.Latest.Version) + assert.NotNil(t, result.Latest.Pubspec) + }) +} diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 31819ccca1..dc753421d0 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/packages/maven" "code.gitea.io/gitea/modules/packages/npm" "code.gitea.io/gitea/modules/packages/nuget" + "code.gitea.io/gitea/modules/packages/pub" "code.gitea.io/gitea/modules/packages/pypi" "code.gitea.io/gitea/modules/packages/rubygems" @@ -143,6 +144,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc metadata = &npm.Metadata{} case TypeMaven: metadata = &maven.Metadata{} + case TypePub: + metadata = &pub.Metadata{} case TypePyPI: metadata = &pypi.Metadata{} case TypeRubyGems: diff --git a/models/packages/package.go b/models/packages/package.go index 97cfbc6cad..39b1c83cfa 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -39,6 +39,7 @@ const ( TypeMaven Type = "maven" TypeNpm Type = "npm" TypeNuGet Type = "nuget" + TypePub Type = "pub" TypePyPI Type = "pypi" TypeRubyGems Type = "rubygems" ) @@ -62,6 +63,8 @@ func (pt Type) Name() string { return "npm" case TypeNuGet: return "NuGet" + case TypePub: + return "Pub" case TypePyPI: return "PyPI" case TypeRubyGems: @@ -89,6 +92,8 @@ func (pt Type) SVGName() string { return "gitea-npm" case TypeNuGet: return "gitea-nuget" + case TypePub: + return "gitea-pub" case TypePyPI: return "gitea-python" case TypeRubyGems: diff --git a/modules/packages/pub/metadata.go b/modules/packages/pub/metadata.go new file mode 100644 index 0000000000..1fc4908b91 --- /dev/null +++ b/modules/packages/pub/metadata.go @@ -0,0 +1,154 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pub + +import ( + "archive/tar" + "compress/gzip" + "errors" + "io" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" + "gopkg.in/yaml.v2" +) + +var ( + ErrMissingPubspecFile = errors.New("Pubspec file is missing") + ErrPubspecFileTooLarge = errors.New("Pubspec file is too large") + ErrInvalidName = errors.New("Package name is invalid") + ErrInvalidVersion = errors.New("Package version is invalid") +) + +var namePattern = regexp.MustCompile(`\A[a-zA-Z_][a-zA-Z0-9_]*\z`) + +// https://github.com/dart-lang/pub-dev/blob/4d582302a8d10152a5cd6129f65bf4f4dbca239d/pkg/pub_package_reader/lib/pub_package_reader.dart#L143 +const maxPubspecFileSize = 128 * 1024 + +// Package represents a Pub package +type Package struct { + Name string + Version string + Metadata *Metadata +} + +// Metadata represents the metadata of a Pub package +type Metadata struct { + Description string `json:"description,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + Readme string `json:"readme,omitempty"` + Pubspec interface{} `json:"pubspec"` +} + +type pubspecPackage struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Description string `yaml:"description"` + Homepage string `yaml:"homepage"` + Repository string `yaml:"repository"` + Documentation string `yaml:"documentation"` +} + +// ParsePackage parses the Pub package file +func ParsePackage(r io.Reader) (*Package, error) { + gzr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer gzr.Close() + + var p *Package + var readme string + + tr := tar.NewReader(gzr) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Typeflag != tar.TypeReg { + continue + } + + if hd.Name == "pubspec.yaml" { + if hd.Size > maxPubspecFileSize { + return nil, ErrPubspecFileTooLarge + } + p, err = ParsePubspecMetadata(tr) + if err != nil { + return nil, err + } + } else if strings.ToLower(hd.Name) == "readme.md" { + data, err := io.ReadAll(tr) + if err != nil { + return nil, err + } + readme = string(data) + } + } + + if p == nil { + return nil, ErrMissingPubspecFile + } + + p.Metadata.Readme = readme + + return p, nil +} + +// ParsePubspecMetadata parses a Pubspec file to retrieve the metadata of a Pub package +func ParsePubspecMetadata(r io.Reader) (*Package, error) { + buf, err := io.ReadAll(io.LimitReader(r, maxPubspecFileSize)) + if err != nil { + return nil, err + } + + var p pubspecPackage + if err := yaml.Unmarshal(buf, &p); err != nil { + return nil, err + } + + if !namePattern.MatchString(p.Name) { + return nil, ErrInvalidName + } + + v, err := version.NewSemver(p.Version) + if err != nil { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(p.Homepage) { + p.Homepage = "" + } + if !validation.IsValidURL(p.Repository) { + p.Repository = "" + } + + var pubspec interface{} + if err := yaml.Unmarshal(buf, &pubspec); err != nil { + return nil, err + } + + return &Package{ + Name: p.Name, + Version: v.String(), + Metadata: &Metadata{ + Description: p.Description, + ProjectURL: p.Homepage, + RepositoryURL: p.Repository, + DocumentationURL: p.Documentation, + Pubspec: pubspec, + }, + }, nil +} diff --git a/modules/packages/pub/metadata_test.go b/modules/packages/pub/metadata_test.go new file mode 100644 index 0000000000..e43ed64fc6 --- /dev/null +++ b/modules/packages/pub/metadata_test.go @@ -0,0 +1,136 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pub + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + packageName = "gitea" + packageVersion = "1.0.1" + description = "Package Description" + projectURL = "https://gitea.io" + repositoryURL = "https://gitea.io/gitea/gitea" + documentationURL = "https://docs.gitea.io" +) + +const pubspecContent = `name: ` + packageName + ` +version: ` + packageVersion + ` +description: ` + description + ` +homepage: ` + projectURL + ` +repository: ` + repositoryURL + ` +documentation: ` + documentationURL + ` + +environment: + sdk: '>=2.16.0 <3.0.0' + +dependencies: + flutter: + sdk: flutter + path: '>=1.8.0 <3.0.0' + +dev_dependencies: + http: '>=0.13.0'` + +func TestParsePackage(t *testing.T) { + createArchive := func(files map[string][]byte) io.Reader { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for filename, content := range files { + hdr := &tar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + } + tw.Close() + zw.Close() + return &buf + } + + t.Run("MissingPubspecFile", func(t *testing.T) { + data := createArchive(map[string][]byte{"dummy.txt": {}}) + + pp, err := ParsePackage(data) + assert.Nil(t, pp) + assert.ErrorIs(t, err, ErrMissingPubspecFile) + }) + + t.Run("PubspecFileTooLarge", func(t *testing.T) { + data := createArchive(map[string][]byte{"pubspec.yaml": make([]byte, 200*1024)}) + + pp, err := ParsePackage(data) + assert.Nil(t, pp) + assert.ErrorIs(t, err, ErrPubspecFileTooLarge) + }) + + t.Run("InvalidPubspecFile", func(t *testing.T) { + data := createArchive(map[string][]byte{"pubspec.yaml": {}}) + + pp, err := ParsePackage(data) + assert.Nil(t, pp) + assert.Error(t, err) + }) + + t.Run("Valid", func(t *testing.T) { + data := createArchive(map[string][]byte{"pubspec.yaml": []byte(pubspecContent)}) + + pp, err := ParsePackage(data) + assert.NoError(t, err) + assert.NotNil(t, pp) + assert.Empty(t, pp.Metadata.Readme) + }) + + t.Run("ValidWithReadme", func(t *testing.T) { + data := createArchive(map[string][]byte{"pubspec.yaml": []byte(pubspecContent), "README.md": []byte("readme")}) + + pp, err := ParsePackage(data) + assert.NoError(t, err) + assert.NotNil(t, pp) + assert.Equal(t, "readme", pp.Metadata.Readme) + }) +} + +func TestParsePubspecMetadata(t *testing.T) { + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{"123abc", "ab-cd"} { + pp, err := ParsePubspecMetadata(strings.NewReader(`name: ` + name)) + assert.Nil(t, pp) + assert.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + pp, err := ParsePubspecMetadata(strings.NewReader(`name: dummy +version: invalid`)) + assert.Nil(t, pp) + assert.ErrorIs(t, err, ErrInvalidVersion) + }) + + t.Run("Valid", func(t *testing.T) { + pp, err := ParsePubspecMetadata(strings.NewReader(pubspecContent)) + assert.NoError(t, err) + assert.NotNil(t, pp) + + assert.Equal(t, packageName, pp.Name) + assert.Equal(t, packageVersion, pp.Version) + assert.Equal(t, description, pp.Metadata.Description) + assert.Equal(t, projectURL, pp.Metadata.ProjectURL) + assert.Equal(t, repositoryURL, pp.Metadata.RepositoryURL) + assert.Equal(t, documentationURL, pp.Metadata.DocumentationURL) + assert.NotNil(t, pp.Metadata.Pubspec) + }) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index b6f00727ea..56d6d41916 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3111,6 +3111,10 @@ npm.dependencies.development = Development Dependencies npm.dependencies.peer = Peer Dependencies npm.dependencies.optional = Optional Dependencies npm.details.tag = Tag +pub.install = To install the package using Dart, run the following command: +pub.documentation = For more information on the Pub registry, see the documentation. +pub.details.repository_site = Repository Site +pub.details.documentation_site = Documentation Site pypi.requires = Requires Python pypi.install = To install the package using pip, run the following command: pypi.documentation = For more information on the PyPI registry, see the documentation. diff --git a/public/img/svg/gitea-pub.svg b/public/img/svg/gitea-pub.svg new file mode 100644 index 0000000000..4a750c7082 --- /dev/null +++ b/public/img/svg/gitea-pub.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index bb9a42e33d..4b274860dc 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -21,6 +21,7 @@ import ( "code.gitea.io/gitea/routers/api/packages/maven" "code.gitea.io/gitea/routers/api/packages/npm" "code.gitea.io/gitea/routers/api/packages/nuget" + "code.gitea.io/gitea/routers/api/packages/pub" "code.gitea.io/gitea/routers/api/packages/pypi" "code.gitea.io/gitea/routers/api/packages/rubygems" "code.gitea.io/gitea/services/auth" @@ -216,6 +217,20 @@ func Routes() *web.Route { }, reqPackageAccess(perm.AccessModeWrite)) }) }) + r.Group("/pub", func() { + r.Group("/api/packages", func() { + r.Group("/versions/new", func() { + r.Get("", pub.RequestUpload) + r.Post("/upload", pub.UploadPackageFile) + r.Get("/finalize/{id}/{version}", pub.FinalizePackage) + }, reqPackageAccess(perm.AccessModeWrite)) + r.Group("/{id}", func() { + r.Get("", pub.EnumeratePackageVersions) + r.Get("/files/{version}", pub.DownloadPackageFile) + r.Get("/{version}", pub.PackageVersionMetadata) + }) + }) + }) r.Group("/pypi", func() { r.Post("/", reqPackageAccess(perm.AccessModeWrite), pypi.UploadPackageFile) r.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index 4d630708b9..81ea28bcad 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -102,7 +102,7 @@ func RegistrationLeaf(ctx *context.Context) { packageName := ctx.Params("id") packageVersion := strings.TrimSuffix(ctx.Params("version"), ".json") - pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages_model.TypeNuGet, packageName, packageVersion) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeNuGet, packageName, packageVersion) if err != nil { if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) diff --git a/routers/api/packages/pub/pub.go b/routers/api/packages/pub/pub.go new file mode 100644 index 0000000000..470f446238 --- /dev/null +++ b/routers/api/packages/pub/pub.go @@ -0,0 +1,275 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pub + +import ( + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + pub_module "code.gitea.io/gitea/modules/packages/pub" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" +) + +func jsonResponse(ctx *context.Context, status int, obj interface{}) { + resp := ctx.Resp + resp.Header().Set("Content-Type", "application/vnd.pub.v2+json") + resp.WriteHeader(status) + if err := json.NewEncoder(resp).Encode(obj); err != nil { + log.Error("JSON encode: %v", err) + } +} + +func apiError(ctx *context.Context, status int, obj interface{}) { + type Error struct { + Code string `json:"code"` + Message string `json:"message"` + } + type ErrorWrapper struct { + Error Error `json:"error"` + } + + helper.LogAndProcessError(ctx, status, obj, func(message string) { + jsonResponse(ctx, status, ErrorWrapper{ + Error: Error{ + Code: http.StatusText(status), + Message: message, + }, + }) + }) +} + +type packageVersions struct { + Name string `json:"name"` + Latest *versionMetadata `json:"latest"` + Versions []*versionMetadata `json:"versions"` +} + +type versionMetadata struct { + Version string `json:"version"` + ArchiveURL string `json:"archive_url"` + Published time.Time `json:"published"` + Pubspec interface{} `json:"pubspec,omitempty"` +} + +func packageDescriptorToMetadata(baseURL string, pd *packages_model.PackageDescriptor) *versionMetadata { + return &versionMetadata{ + Version: pd.Version.Version, + ArchiveURL: fmt.Sprintf("%s/files/%s.tar.gz", baseURL, url.PathEscape(pd.Version.Version)), + Published: time.Unix(int64(pd.Version.CreatedUnix), 0), + Pubspec: pd.Metadata.(*pub_module.Metadata).Pubspec, + } +} + +func baseURL(ctx *context.Context) string { + return setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pub/api/packages" +} + +// https://github.com/dart-lang/pub/blob/master/doc/repository-spec-v2.md#list-all-versions-of-a-package +func EnumeratePackageVersions(ctx *context.Context) { + packageName := ctx.Params("id") + + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypePub, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + sort.Slice(pds, func(i, j int) bool { + return pds[i].SemVer.LessThan(pds[j].SemVer) + }) + + baseURL := fmt.Sprintf("%s/%s", baseURL(ctx), url.PathEscape(pds[0].Package.Name)) + + versions := make([]*versionMetadata, 0, len(pds)) + for _, pd := range pds { + versions = append(versions, packageDescriptorToMetadata(baseURL, pd)) + } + + jsonResponse(ctx, http.StatusOK, &packageVersions{ + Name: pds[0].Package.Name, + Latest: packageDescriptorToMetadata(baseURL, pds[0]), + Versions: versions, + }) +} + +// https://github.com/dart-lang/pub/blob/master/doc/repository-spec-v2.md#deprecated-inspect-a-specific-version-of-a-package +func PackageVersionMetadata(ctx *context.Context) { + packageName := ctx.Params("id") + packageVersion := ctx.Params("version") + + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypePub, packageName, packageVersion) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pd, err := packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + jsonResponse(ctx, http.StatusOK, packageDescriptorToMetadata( + fmt.Sprintf("%s/%s", baseURL(ctx), url.PathEscape(pd.Package.Name)), + pd, + )) +} + +// https://github.com/dart-lang/pub/blob/master/doc/repository-spec-v2.md#publishing-packages +func RequestUpload(ctx *context.Context) { + type UploadRequest struct { + URL string `json:"url"` + Fields map[string]string `json:"fields"` + } + + jsonResponse(ctx, http.StatusOK, UploadRequest{ + URL: baseURL(ctx) + "/versions/new/upload", + Fields: make(map[string]string), + }) +} + +// https://github.com/dart-lang/pub/blob/master/doc/repository-spec-v2.md#publishing-packages +func UploadPackageFile(ctx *context.Context) { + file, _, err := ctx.Req.FormFile("file") + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + defer file.Close() + + buf, err := packages_module.CreateHashedBufferFromReader(file, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + pck, err := pub_module.ParsePackage(buf) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + _, _, err = packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypePub, + Name: pck.Name, + Version: pck.Version, + }, + SemverCompatible: true, + Creator: ctx.Doer, + Metadata: pck.Metadata, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(pck.Version + ".tar.gz"), + }, + Data: buf, + IsLead: true, + }, + ) + if err != nil { + if err == packages_model.ErrDuplicatePackageVersion { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Resp.Header().Set("Location", fmt.Sprintf("%s/versions/new/finalize/%s/%s", baseURL(ctx), url.PathEscape(pck.Name), url.PathEscape(pck.Version))) + ctx.Status(http.StatusNoContent) +} + +// https://github.com/dart-lang/pub/blob/master/doc/repository-spec-v2.md#publishing-packages +func FinalizePackage(ctx *context.Context) { + packageName := ctx.Params("id") + packageVersion := ctx.Params("version") + + _, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypePub, packageName, packageVersion) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + type Success struct { + Message string `json:"message"` + } + type SuccessWrapper struct { + Success Success `json:"success"` + } + + jsonResponse(ctx, http.StatusOK, SuccessWrapper{Success{}}) +} + +// https://github.com/dart-lang/pub/blob/master/doc/repository-spec-v2.md#deprecated-download-a-specific-version-of-a-package +func DownloadPackageFile(ctx *context.Context) { + packageName := ctx.Params("id") + packageVersion := strings.TrimSuffix(ctx.Params("version"), ".tar.gz") + + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypePub, packageName, packageVersion) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pd, err := packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pf := pd.Files[0].File + + s, _, err := packages_service.GetPackageFileStream(ctx, pf) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 5a9c93b3ca..2c02389102 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -41,7 +41,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [composer, conan, container, generic, helm, maven, npm, nuget, pypi, rubygems] + // enum: [composer, conan, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems] // - name: q // in: query // description: name filter diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index 18e8c5fed8..61721532a4 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -21,6 +21,7 @@ + diff --git a/templates/package/content/pub.tmpl b/templates/package/content/pub.tmpl new file mode 100644 index 0000000000..999e48ce82 --- /dev/null +++ b/templates/package/content/pub.tmpl @@ -0,0 +1,19 @@ +{{if eq .PackageDescriptor.Package.Type "pub"}} +

{{.locale.Tr "packages.installation"}}

+
+
+
+ + 
dart pub add {{.PackageDescriptor.Package.Name}}:{{.PackageDescriptor.Version.Version}} --hosted-url={{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/pub/
+
+
+ +
+
+
+ {{if or .PackageDescriptor.Metadata.Description .PackageDescriptor.Metadata.Readme}} +

{{.locale.Tr "packages.about"}}

+ {{if .PackageDescriptor.Metadata.Description}}
{{.PackageDescriptor.Metadata.Description}}
{{end}} + {{if .PackageDescriptor.Metadata.Readme}}
{{RenderMarkdownToHtml .PackageDescriptor.Metadata.Readme}}
{{end}} + {{end}} +{{end}} diff --git a/templates/package/metadata/pub.tmpl b/templates/package/metadata/pub.tmpl new file mode 100644 index 0000000000..8bd65b49d6 --- /dev/null +++ b/templates/package/metadata/pub.tmpl @@ -0,0 +1,5 @@ +{{if eq .PackageDescriptor.Package.Type "pub"}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.project_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.RepositoryURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.pub.details.repository_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.DocumentationURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.pub.details.documentation_site"}}
{{end}} +{{end}} diff --git a/templates/package/shared/list.tmpl b/templates/package/shared/list.tmpl index 189b75b59e..020acba9d7 100644 --- a/templates/package/shared/list.tmpl +++ b/templates/package/shared/list.tmpl @@ -14,6 +14,7 @@ + diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index ba309b6a77..0116da53b3 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -27,6 +27,7 @@ {{template "package/content/maven" .}} {{template "package/content/npm" .}} {{template "package/content/nuget" .}} + {{template "package/content/pub" .}} {{template "package/content/pypi" .}} {{template "package/content/rubygems" .}} @@ -48,6 +49,7 @@ {{template "package/metadata/maven" .}} {{template "package/metadata/npm" .}} {{template "package/metadata/nuget" .}} + {{template "package/metadata/pub" .}} {{template "package/metadata/pypi" .}} {{template "package/metadata/rubygems" .}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index daec38044f..19f16b5c1c 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -1960,6 +1960,7 @@ "maven", "npm", "nuget", + "pub", "pypi", "rubygems" ], diff --git a/web_src/svg/gitea-pub.svg b/web_src/svg/gitea-pub.svg new file mode 100644 index 0000000000..5ec2b8818a --- /dev/null +++ b/web_src/svg/gitea-pub.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + +