upload assets and replace the links in issue/comment/pullrequest

models/migrate.go

@@ -91,6 +91,16 @@ func insertIssue(ctx context.Context, issue *Issue) error {
 		}
 	}
 
+	for _, attach := range issue.Attachments {
+		attach.IssueID = issue.ID
+	}
+
+	if len(issue.Attachments) > 0 {
+		if _, err := sess.Insert(issue.Attachments); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -124,6 +134,16 @@ func InsertIssueComments(comments []*Comment) error {
 				return err
 			}
 		}
+
+		for _, attach := range comment.Attachments {
+			attach.IssueID = comment.IssueID
+			attach.CommentID = comment.ID
+		}
+		if len(comment.Attachments) > 0 {
+			if err := db.Insert(ctx, comment.Attachments); err != nil {
+				return err
+			}
+		}
 	}
 
 	for issueID := range issueIDs {

modules/migration/asset.go

@@ -0,0 +1,25 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import (
+	"io"
+	"time"
+)
+
+// Asset represents an asset for issue, comment, release
+type Asset struct {
+	ID            int64
+	Name          string
+	ContentType   *string `yaml:"content_type"`
+	Size          *int
+	DownloadCount *int `yaml:"download_count"`
+	Created       time.Time
+	Updated       time.Time
+	DownloadURL   *string `yaml:"download_url"`
+	OriginalURL   string
+	// if DownloadURL is nil, the function should be invoked
+	DownloadFunc func() (io.ReadCloser, error) `yaml:"-"`
+}

modules/migration/comment.go

@@ -63,6 +63,7 @@ type Comment struct {
 	Updated     time.Time
 	Content     string
 	Reactions   []*Reaction
+	Assets      []*Asset
 }
 
 // GetExternalName ExternalUserMigrated interface

modules/migration/pullrequest.go

@@ -37,6 +37,7 @@ type PullRequest struct {
 	Reactions      []*Reaction
 	ForeignIndex   int64
 	Context        DownloaderContext `yaml:"-"`
+	Assets         []*Asset
 }
 
 func (p *PullRequest) GetLocalIndex() int64          { return p.Number }

modules/migration/release.go

@@ -5,24 +5,9 @@
 package migration
 
 import (
-	"io"
 	"time"
 )
 
-// Asset represents an asset for issue, comment, release
-type Asset struct {
-	ID            int64
-	Name          string
-	ContentType   *string `yaml:"content_type"`
-	Size          *int
-	DownloadCount *int `yaml:"download_count"`
-	Created       time.Time
-	Updated       time.Time
-	DownloadURL   *string `yaml:"download_url"`
-	// if DownloadURL is nil, the function should be invoked
-	DownloadFunc func() (io.ReadCloser, error) `yaml:"-"`
-}
-
 // Release represents a release
 type Release struct {
 	TagName         string `yaml:"tag_name"`

services/migrations/gitea_uploader.go

@@ -237,6 +237,40 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
 	return nil
 }
 
+func (g *GiteaLocalUploader) uploadAttachment(asset *base.Asset) (*repo_model.Attachment, error) {
+	attach := repo_model.Attachment{
+		UUID:          gouuid.New().String(),
+		Name:          asset.Name,
+		DownloadCount: int64(*asset.DownloadCount),
+		Size:          int64(*asset.Size),
+		CreatedUnix:   timeutil.TimeStamp(asset.Created.Unix()),
+	}
+
+	// asset.DownloadURL maybe a local file
+	var rc io.ReadCloser
+	var err error
+	if asset.DownloadFunc != nil {
+		rc, err = asset.DownloadFunc()
+		if err != nil {
+			return nil, err
+		}
+	} else if asset.DownloadURL != nil {
+		rc, err = uri.Open(*asset.DownloadURL)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if rc == nil {
+		return nil, nil
+	}
+	defer rc.Close()
+	_, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size))
+	if err != nil {
+		return nil, err
+	}
+	return &attach, nil
+}
+
 // CreateReleases creates releases
 func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 	rels := make([]*models.Release, 0, len(releases))
@@ -289,42 +323,13 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 					asset.Created = release.Created
 				}
 			}
-			attach := repo_model.Attachment{
-				UUID:          gouuid.New().String(),
-				Name:          asset.Name,
-				DownloadCount: int64(*asset.DownloadCount),
-				Size:          int64(*asset.Size),
-				CreatedUnix:   timeutil.TimeStamp(asset.Created.Unix()),
-			}
-
-			// download attachment
-			err := func() error {
-				// asset.DownloadURL maybe a local file
-				var rc io.ReadCloser
-				var err error
-				if asset.DownloadFunc != nil {
-					rc, err = asset.DownloadFunc()
-					if err != nil {
-						return err
-					}
-				} else if asset.DownloadURL != nil {
-					rc, err = uri.Open(*asset.DownloadURL)
-					if err != nil {
-						return err
-					}
-				}
-				if rc == nil {
-					return nil
-				}
-				_, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size))
-				rc.Close()
-				return err
-			}()
+			attach, err := g.uploadAttachment(asset)
 			if err != nil {
 				return err
 			}
-
-			rel.Attachments = append(rel.Attachments, &attach)
+			if attach != nil {
+				rel.Attachments = append(rel.Attachments, attach)
+			}
 		}
 
 		rels = append(rels, &rel)
@@ -406,6 +411,27 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
 			}
 			is.Reactions = append(is.Reactions, &res)
 		}
+
+		for _, asset := range issue.Assets {
+			if asset.Created.IsZero() {
+				if !asset.Updated.IsZero() {
+					asset.Created = asset.Updated
+				} else {
+					asset.Created = issue.Updated
+				}
+			}
+			attach, err := g.uploadAttachment(asset)
+			if err != nil {
+				return err
+			}
+			if attach != nil {
+				is.Attachments = append(is.Attachments, attach)
+				if asset.OriginalURL != "" {
+					is.Content = strings.ReplaceAll(is.Content, asset.OriginalURL, fmt.Sprintf("/attachments/%s", attach.UUID))
+				}
+			}
+		}
+
 		iss = append(iss, &is)
 	}
 
@@ -544,6 +570,26 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
 			cm.Reactions = append(cm.Reactions, &res)
 		}
 
+		for _, asset := range comment.Assets {
+			if asset.Created.IsZero() {
+				if !asset.Updated.IsZero() {
+					asset.Created = asset.Updated
+				} else {
+					asset.Created = comment.Updated
+				}
+			}
+			attach, err := g.uploadAttachment(asset)
+			if err != nil {
+				return err
+			}
+			if attach != nil {
+				cm.Attachments = append(cm.Attachments, attach)
+				if asset.OriginalURL != "" {
+					cm.Content = strings.ReplaceAll(cm.Content, asset.OriginalURL, fmt.Sprintf("/attachments/%s", attach.UUID))
+				}
+			}
+		}
+
 		cms = append(cms, &cm)
 	}
 
@@ -566,6 +612,26 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error
 			return err
 		}
 
+		for _, asset := range pr.Assets {
+			if asset.Created.IsZero() {
+				if !asset.Updated.IsZero() {
+					asset.Created = asset.Updated
+				} else {
+					asset.Created = pr.Updated
+				}
+			}
+			attach, err := g.uploadAttachment(asset)
+			if err != nil {
+				return err
+			}
+			if attach != nil {
+				gpr.Issue.Attachments = append(gpr.Issue.Attachments, attach)
+				if asset.OriginalURL != "" {
+					gpr.Issue.Content = strings.ReplaceAll(gpr.Issue.Content, asset.OriginalURL, fmt.Sprintf("/attachments/%s", attach.UUID))
+				}
+			}
+		}
+
 		gprs = append(gprs, gpr)
 	}
 	if err := models.InsertPullRequests(gprs...); err != nil {

services/migrations/github_exported_data.go

@@ -131,6 +131,7 @@ func (g *githubUser) Email() string {
   },
 */
 type githubAttachment struct {
+	URL              string `json:"url"`
 	Issue            string
 	IssueComment     string `json:"issue_comment"`
 	User             string
@@ -148,6 +149,10 @@ func (g *githubAttachment) IsIssue() bool {
 	return len(g.Issue) > 0
 }
 
+func (g *githubAttachment) IsComment() bool {
+	return len(g.IssueComment) > 0
+}
+
 func (g *githubAttachment) IssueID() int64 {
 	if g.IsIssue() {
 		return parseGitHubResID(g.Issue)
@@ -170,7 +175,9 @@ func (r *GithubExportedDataRestorer) convertAttachments(ls []githubAttachment) [
 			ContentType: &l.AssetContentType,
 			Size:        &size,
 			Created:     l.CreatedAt,
+			Updated:     l.CreatedAt,
 			DownloadURL: &assetURL,
+			OriginalURL: l.URL,
 		})
 	}
 	return res
@@ -309,11 +316,16 @@ func (r *GithubExportedDataRestorer) CleanUp() {
 	}
 }
 
-// replaceComment replace #id to new form
+var (
+	regMatchAttachment = regexp.MustCompile("https://user-images.githubusercontent.com/([0-9]+)/(.*)")
+)
+
+// replaceGithubLinks replace #id to new form
 // i.e.
 // 1) https://github.com/userstyles-world/userstyles.world/commit/b70d545a1cbb5c92ca20f442f59de5d955600408 -> b70d545a1cbb5c92ca20f442f59de5d955600408
 // 2) https://github.com/go-gitea/gitea/issue/1 -> #1
 // 3) https://github.com/go-gitea/gitea/pull/2 -> #2
+// 4) https://user-images.githubusercontent.com/1824502/146297011-8c01ea20-276e-421b-98a9-e39fe76ac046.png -> attachment link
 func (r *GithubExportedDataRestorer) replaceGithubLinks(content string) string {
 	c := r.regMatchIssue.ReplaceAllString(content, "#$2")
 	c = r.regMatchCommit.ReplaceAllString(c, "$1")
@@ -759,6 +771,7 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue
 }
 
 type githubComment struct {
+	URL         string `json:"url"`
 	Issue       string
 	PullRequest string `json:"pull_request"`
 	User        string
@@ -1014,6 +1027,7 @@ func (r *GithubExportedDataRestorer) GetComments(opts base.GetCommentOptions) ([
 				Updated:     c.CreatedAt, // FIXME:
 				Content:     r.replaceGithubLinks(c.Body),
 				Reactions:   r.getReactions(c.Reactions),
+				Assets:      r.convertAttachments(r.issueAttachments[c.URL]),
 			})
 		}
 		return nil
@@ -1152,6 +1166,7 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base
 				PosterEmail: email,
 				Context:     base.BasicIssueContext(pr.Index()),
 				Reactions:   r.getReactions(pr.Reactions),
+				Assets:      r.convertAttachments(r.issueAttachments[pr.URL]),
 				Created:     pr.CreatedAt,
 				Closed:      pr.ClosedAt,
 				Labels:      r.getLabels(pr.Labels),