feat: template unmarshal

modules/context/repo.go

@@ -9,11 +9,9 @@ import (
 	"context"
 	"fmt"
 	"html"
-	"io"
 	"net/http"
 	"net/url"
 	"path"
-	"path/filepath"
 	"strings"
 
 	"code.gitea.io/gitea/models"
@@ -27,6 +25,7 @@ import (
 	"code.gitea.io/gitea/modules/cache"
 	"code.gitea.io/gitea/modules/git"
 	code_indexer "code.gitea.io/gitea/modules/indexer/code"
+	"code.gitea.io/gitea/modules/issue/template"
 	"code.gitea.io/gitea/modules/log"
 	repo_module "code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
@@ -1035,8 +1034,8 @@ func UnitTypes() func(ctx *Context) {
 }
 
 // IssueTemplatesFromDefaultBranch checks for issue templates in the repo's default branch
-func (ctx *Context) IssueTemplatesFromDefaultBranch() []api.IssueTemplate {
-	var issueTemplates []api.IssueTemplate
+func (ctx *Context) IssueTemplatesFromDefaultBranch() []*api.IssueTemplate {
+	var issueTemplates []*api.IssueTemplate
 
 	if ctx.Repo.Repository.IsEmpty {
 		return issueTemplates
@@ -1060,52 +1059,19 @@ func (ctx *Context) IssueTemplatesFromDefaultBranch() []api.IssueTemplate {
 			return issueTemplates
 		}
 		for _, entry := range entries {
-			if t := ctx.extractIssueTemplate(entry); t != nil {
-				issueTemplates = append(issueTemplates, *t)
+			it := &api.IssueTemplate{
+				FileName: entry.Name(),
+			}
+			if it.Type() == "" {
+				continue
+			}
+			it, err := template.UnmarshalFromEntry(entry)
+			if err != nil {
+				log.Debug("unmarshal template from %s: %v", entry.Name(), err)
+			} else if it.Valid() {
+				issueTemplates = append(issueTemplates, it)
 			}
 		}
 	}
 	return issueTemplates
 }
-
-func (ctx *Context) extractIssueTemplate(entry *git.TreeEntry) *api.IssueTemplate {
-	it := &api.IssueTemplate{
-		FileName: entry.Name(),
-	}
-	if it.Type() == "" {
-		return nil
-	}
-
-	if name := filepath.Base(it.FileName); name == "config.yaml" || name == "config.yml" {
-		// ignore config.yaml which is a special configuration file
-		return nil
-	}
-
-	if entry.Blob().Size() >= setting.UI.MaxDisplayFileSize {
-		log.Debug("Issue template is too large: %s", entry.Name())
-		return nil
-	}
-
-	r, err := entry.Blob().DataAsync()
-	if err != nil {
-		log.Debug("DataAsync: %v", err)
-		return nil
-	}
-	defer r.Close()
-
-	data, err := io.ReadAll(r)
-	if err != nil {
-		log.Debug("ReadAll: %v", err)
-		return nil
-	}
-
-	if err := it.Fill(data); err != nil {
-		log.Debug("fill template: %v", err)
-		return nil
-	}
-
-	if !it.Valid() {
-		return nil
-	}
-	return it
-}

modules/issue/template/unmarshal.go

@@ -0,0 +1,92 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package template
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/markup/markdown"
+	"code.gitea.io/gitea/modules/setting"
+	api "code.gitea.io/gitea/modules/structs"
+
+	"gopkg.in/yaml.v2"
+)
+
+// TODO
+func Unmarshal(filename string, content []byte) (*api.IssueTemplate, error) {
+	it := &api.IssueTemplate{
+		FileName: filename,
+	}
+
+	if typ := it.Type(); typ == "md" {
+		templateBody, err := markdown.ExtractMetadata(string(content), it)
+		if err != nil {
+			return nil, fmt.Errorf("extract metadata: %w", err)
+		}
+		it.Content = templateBody
+	} else if typ == "yaml" {
+		if err := yaml.Unmarshal(content, it); err != nil {
+			return nil, fmt.Errorf("yaml unmarshal: %w", err)
+		}
+		if it.About == "" {
+			// Compatible with treating description as about
+			compatibleTemplate := &struct {
+				About string `yaml:"description"`
+			}{}
+			if err := yaml.Unmarshal(content, compatibleTemplate); err == nil && compatibleTemplate.About != "" {
+				it.About = compatibleTemplate.About
+			}
+		}
+		for i, v := range it.Fields {
+			if v.ID == "" {
+				v.ID = strconv.Itoa(i)
+			}
+		}
+	}
+
+	return it, nil
+}
+
+// TODO
+func UnmarshalFromEntry(entry *git.TreeEntry) (*api.IssueTemplate, error) {
+	if size := entry.Blob().Size(); size > setting.UI.MaxDisplayFileSize {
+		return nil, fmt.Errorf("too large: %v > MaxDisplayFileSize", size)
+	}
+
+	r, err := entry.Blob().DataAsync()
+	if err != nil {
+		return nil, fmt.Errorf("data async: %w", err)
+	}
+	defer r.Close()
+
+	content, err := io.ReadAll(r)
+	if err != nil {
+		return nil, fmt.Errorf("read all: %w", err)
+	}
+
+	return Unmarshal(entry.Name(), content)
+}
+
+// TODO
+func UnmarshalFromCommit(commit *git.Commit, filename string) (*api.IssueTemplate, error) {
+	entry, err := commit.GetTreeEntryByPath(filename)
+	if err != nil {
+		return nil, fmt.Errorf("get entry for %q: %w", filename, err)
+	}
+	return UnmarshalFromEntry(entry)
+}
+
+// TODO
+func UnmarshalFromRepo(repo *git.Repository, branch, filename string) (*api.IssueTemplate, error) {
+	commit, err := repo.GetBranchCommit(branch)
+	if err != nil {
+		return nil, fmt.Errorf("get commit on branch %q: %w", branch, err)
+	}
+
+	return UnmarshalFromCommit(commit, filename)
+}

modules/structs/issue.go

@@ -7,11 +7,8 @@ package structs
 import (
 	"fmt"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"time"
-
-	"gopkg.in/yaml.v2"
 )
 
 // StateType issue state type
@@ -229,79 +226,15 @@ func (it *IssueTemplate) Valid() bool {
 
 // Type returns the type of IssueTemplate, it could be "md", "yaml" or empty for known
 func (it *IssueTemplate) Type() string {
+	if it.Name == "config.yaml" || it.Name == "config.yml" {
+		// TODO: should it be?
+		// ignore config.yaml which is a special configuration file
+		return ""
+	}
 	if ext := filepath.Ext(it.FileName); ext == ".md" {
 		return "md"
 	} else if ext == ".yaml" || ext == ".yml" {
 		return "yaml"
 	}
 	return ""
 }
-
-func (it *IssueTemplate) Fill(content []byte) error {
-	if typ := it.Type(); typ == "md" {
-		templateBody, err := it.extractMetadata(string(content), it)
-		if err != nil {
-			return fmt.Errorf("extract metadata: %w", err)
-		}
-		it.Content = templateBody
-	} else if typ == "yaml" {
-		if err := yaml.Unmarshal(content, it); err != nil {
-			return fmt.Errorf("yaml unmarshal: %w", err)
-		}
-		if it.About == "" {
-			// Compatible with treating description as about
-			compatibleTemplate := &struct {
-				About string `yaml:"description"`
-			}{}
-			if err := yaml.Unmarshal(content, compatibleTemplate); err == nil && compatibleTemplate.About != "" {
-				it.About = compatibleTemplate.About
-			}
-		}
-		for i, v := range it.Fields {
-			if v.ID == "" {
-				v.ID = strconv.Itoa(i)
-			}
-		}
-	}
-	return nil
-}
-
-// extractMetadata consumes a markdown file, parses YAML frontmatter,
-// and returns the frontmatter metadata separated from the markdown content.
-// Copy from markdown.ExtractMetadata to avoid import cycle.
-func (*IssueTemplate) extractMetadata(contents string, out interface{}) (string, error) {
-	isYAMLSeparator := func(line string) bool {
-		line = strings.TrimSpace(line)
-		for i := 0; i < len(line); i++ {
-			if line[i] != '-' {
-				return false
-			}
-		}
-		return len(line) > 2
-	}
-
-	var front, body []string
-	lines := strings.Split(contents, "\n")
-	for idx, line := range lines {
-		if idx == 0 {
-			// First line has to be a separator
-			if !isYAMLSeparator(line) {
-				return "", fmt.Errorf("frontmatter must start with a separator line")
-			}
-			continue
-		}
-		if isYAMLSeparator(line) {
-			front, body = lines[1:idx], lines[idx+1:]
-			break
-		}
-	}
-
-	if len(front) == 0 {
-		return "", fmt.Errorf("could not determine metadata")
-	}
-
-	if err := yaml.Unmarshal([]byte(strings.Join(front, "\n")), out); err != nil {
-		return "", err
-	}
-	return strings.Join(body, "\n"), nil
-}

routers/web/repo/issue.go

@@ -10,7 +10,6 @@ import (
 	stdCtx "context"
 	"errors"
 	"fmt"
-	"io"
 	"math/big"
 	"net/http"
 	"net/url"
@@ -35,6 +34,7 @@ import (
 	"code.gitea.io/gitea/modules/convert"
 	"code.gitea.io/gitea/modules/git"
 	issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
+	issue_template "code.gitea.io/gitea/modules/issue/template"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
@@ -734,34 +734,6 @@ func RetrieveRepoMetas(ctx *context.Context, repo *repo_model.Repository, isPull
 	return labels
 }
 
-func getFileContentFromDefaultBranch(ctx *context.Context, filename string) (string, bool) {
-	if ctx.Repo.Commit == nil {
-		var err error
-		ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(ctx.Repo.Repository.DefaultBranch)
-		if err != nil {
-			return "", false
-		}
-	}
-
-	entry, err := ctx.Repo.Commit.GetTreeEntryByPath(filename)
-	if err != nil {
-		return "", false
-	}
-	if entry.Blob().Size() >= setting.UI.MaxDisplayFileSize {
-		return "", false
-	}
-	r, err := entry.Blob().DataAsync()
-	if err != nil {
-		return "", false
-	}
-	defer r.Close()
-	bytes, err := io.ReadAll(r)
-	if err != nil {
-		return "", false
-	}
-	return string(bytes), true
-}
-
 func setTemplateIfExists(ctx *context.Context, ctxDataKey string, possibleDirs, possibleFiles []string) {
 	templateCandidates := make([]string, 0, len(possibleDirs)+len(possibleFiles))
 	if t := ctx.FormString("template"); t != "" {
@@ -771,23 +743,19 @@ func setTemplateIfExists(ctx *context.Context, ctxDataKey string, possibleDirs,
 	}
 	templateCandidates = append(templateCandidates, possibleFiles...) // Append files to the end because they should be fallback
 	for _, filename := range templateCandidates {
-		templateContent, found := getFileContentFromDefaultBranch(ctx, filename)
-		if !found {
+		template, err := issue_template.UnmarshalFromRepo(ctx.Repo.GitRepo, ctx.Repo.Repository.DefaultBranch, filename)
+		if err == nil {
 			continue
 		}
-		template := api.IssueTemplate{
-			FileName: filename,
-		}
-		if err := template.Fill([]byte(templateContent)); err != nil {
-			log.Debug("could fill template from %s [%s]: %v", filename, ctx.Repo.Repository.FullName(), err)
-			ctx.Data[ctxDataKey] = templateContent
-			return
+		if !template.Valid() {
+			continue
 		}
 
 		ctx.Data[issueTemplateTitleKey] = template.Title
 		ctx.Data[ctxDataKey] = template.Content
+
 		if template.Type() == "yaml" {
-			ctx.Data["TemplateForm"] = template // TODO: maybe use template.Body
+			ctx.Data["Fields"] = template.Fields
 		}
 		labelIDs := make([]string, 0, len(template.Labels))
 		if repoLabels, err := issues_model.GetLabelsByRepoID(ctx, ctx.Repo.Repository.ID, "", db.ListOptions{}); err == nil {
@@ -1107,13 +1075,11 @@ func NewIssuePost(ctx *context.Context) {
 // Returns an empty string if user submitted a non-form issue
 func renderIssueFormValues(ctx *context.Context, form *url.Values) (string, error) {
 	filename := form.Get("template-file")
-	template := api.IssueTemplate{
-		FileName: filename,
-	}
-	if templateContent, found := getFileContentFromDefaultBranch(ctx, form.Get("template-file")); !found {
-		return "", fmt.Errorf("template file %q not found", filename)
-	} else if err := template.Fill([]byte(templateContent)); err != nil {
-		return "", fmt.Errorf("fill template with %q: %w", filename, err)
+	template, err := issue_template.UnmarshalFromRepo(ctx.Repo.GitRepo, ctx.Repo.Repository.DefaultBranch, form.Get("template-file"))
+	if err != nil {
+		return "", fmt.Errorf("unmarshal template %q: %w", filename, err)
+	} else if !template.Valid() {
+		return "", fmt.Errorf("invalid template %q", filename)
 	}
 
 	// Render values