Skip to content

Dump github/gitlab/gitea repository data to a local directory and restore to gitea #12244

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 47 commits into from
Dec 27, 2020
Merged
Changes from 1 commit
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
7798250
Dump github/gitlab repository data to a local directory
lunny Jul 14, 2020
9fa7df4
Fix lint
lunny Jul 14, 2020
13d4b3a
Adjust directory structure
lunny Jul 18, 2020
8c3f3db
Allow migration special units
lunny Jul 18, 2020
9e7915c
Allow migration ignore release assets
lunny Jul 18, 2020
ed4e376
Fix lint
lunny Jul 19, 2020
c49020b
Add restore repository
lunny Aug 22, 2020
c13e745
stage the changes
lunny Sep 1, 2020
aa5ffb3
Merge
lunny Sep 11, 2020
92fbde7
Fix lint
lunny Sep 11, 2020
3999010
Update the interface
lunny Oct 18, 2020
6d633ef
Add some restore methods
lunny Oct 20, 2020
7006ba2
Finish restore
lunny Oct 21, 2020
d1746bc
Add comments
lunny Oct 21, 2020
08e7770
Fix restore
lunny Oct 21, 2020
3c7b2d2
Add a token flag
lunny Oct 21, 2020
770aac9
Fix bug
lunny Oct 21, 2020
d367872
Fix test
lunny Oct 21, 2020
bbc289c
Fix test
lunny Oct 22, 2020
4f43c44
Fix bug
lunny Oct 23, 2020
0b7786f
Fix bug
lunny Oct 23, 2020
7cfd9d8
Fix lint
lunny Oct 23, 2020
c43966b
Fix restore
lunny Oct 23, 2020
a8bdf48
refactor downloader
lunny Oct 23, 2020
20cbc64
fmt
lunny Oct 23, 2020
79f0a4a
Fix bug isEnd detection on getIssues
lunny Oct 24, 2020
ad60586
Refactor maxPerPage
lunny Oct 24, 2020
d33175d
Remove unused codes
lunny Oct 24, 2020
04bebd6
Remove unused codes
lunny Oct 24, 2020
af0be0b
Fix bug
lunny Oct 27, 2020
e615c19
Fix restore
lunny Oct 29, 2020
ab53820
Fix dump
lunny Nov 3, 2020
96ca8e4
Uploader should not depend downloader
lunny Nov 23, 2020
3075168
use release attachment name but not id
lunny Dec 21, 2020
97cc82c
Fix restore bug
lunny Dec 22, 2020
6c5044d
Fix lint
lunny Dec 22, 2020
c09c7ed
Fix restore bug
lunny Dec 26, 2020
8f4e741
Add a method of DownloadFunc for base.Release to make uploader not de…
lunny Dec 26, 2020
2124a61
fix Release yml marshal
lunny Dec 26, 2020
2e90a9f
Fix trace information
lunny Dec 26, 2020
2b0fe43
Fix bug when dump & restore
lunny Dec 26, 2020
f91ba44
Save relative path on yml file
lunny Dec 26, 2020
eb5da17
Fix bug
lunny Dec 26, 2020
3c5b2e1
Use relative path
lunny Dec 26, 2020
80b5c9a
Update docs
lunny Dec 26, 2020
ceefd98
Use git service string but not int
lunny Dec 27, 2020
695d23e
Recognize clone addr to service type
lunny Dec 27, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix dump
  • Loading branch information
lunny committed Dec 27, 2020
commit ab53820eacc06bb5cd54d7c78a60ecbdc48aef52
58 changes: 36 additions & 22 deletions modules/migrations/dump.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type RepositoryDumper struct {
baseDir string
repoOwner string
repoName string
opts base.MigrateOptions
milestoneFile *os.File
labelFile *os.File
releaseFile *os.File
Expand All @@ -47,17 +48,17 @@ type RepositoryDumper struct {
}

// NewRepositoryDumper creates an gitea Uploader
func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, migrateReleaseAssets bool) *RepositoryDumper {
func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, opts base.MigrateOptions) *RepositoryDumper {
baseDir = filepath.Join(baseDir, repoOwner, repoName)
return &RepositoryDumper{
ctx: ctx,
baseDir: baseDir,
repoOwner: repoOwner,
repoName: repoName,
prHeadCache: make(map[string]struct{}),
commentFiles: make(map[int64]*os.File),
reviewFiles: make(map[int64]*os.File),
migrateReleaseAssets: migrateReleaseAssets,
ctx: ctx,
opts: opts,
baseDir: baseDir,
repoOwner: repoOwner,
repoName: repoName,
prHeadCache: make(map[string]struct{}),
commentFiles: make(map[int64]*os.File),
reviewFiles: make(map[int64]*os.File),
}
}

Expand Down Expand Up @@ -106,6 +107,22 @@ func (g *RepositoryDumper) reviewDir() string {
return filepath.Join(g.baseDir, "reviews")
}

func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) {
if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword)
if len(g.opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", g.opts.AuthToken)
}
remoteAddr = u.String()
}

return remoteAddr, nil
}

// CreateRepo creates a repository
func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil {
Expand Down Expand Up @@ -150,17 +167,9 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp

migrateTimeout := 2 * time.Hour

var remoteAddr = repo.CloneURL
if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
u, err := url.Parse(repo.CloneURL)
if err != nil {
return err
}
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
if len(opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", opts.AuthToken)
}
remoteAddr = u.String()
remoteAddr, err := g.setURLToken(repo.CloneURL)
if err != nil {
return err
}

err = git.Clone(remoteAddr, repoPath, git.CloneRepoOptions{
Expand Down Expand Up @@ -434,7 +443,11 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
for _, pr := range prs {
// download patch file
err := func() error {
resp, err := http.Get(pr.PatchURL)
u, err := g.setURLToken(pr.PatchURL)
if err != nil {
return err
}
resp, err := http.Get(u)
if err != nil {
return err
}
Expand Down Expand Up @@ -476,6 +489,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
_, ok := g.prHeadCache[remote]
if !ok {
// git remote add
// TODO: how to handle private CloneURL?
err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
if err != nil {
log.Error("AddRemote failed: %s", err)
Expand Down Expand Up @@ -559,7 +573,7 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi
if err != nil {
return err
}
var uploader = NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts.ReleaseAssets)
var uploader = NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts)

if err := migrateRepository(downloader, uploader, opts); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
Expand Down