Move database settings from models to setting (#7806)

* move database settings from models to setting

* update docs

* fix checkout pr

* fix tests

* fix lint

* remove unsupported tidb options

* correct wrong variable name

* remove tidb totally

Author: lunny

cmd/cmd.go

@@ -38,7 +38,7 @@ func initDB() error {
 
 func initDBDisableConsole(disableConsole bool) error {
 	setting.NewContext()
-	models.LoadConfigs()
+	setting.InitDBConfig()
 
 	setting.NewXORMLogService(disableConsole)
 	if err := models.SetEngine(); err != nil {

cmd/convert.go

@@ -31,9 +31,9 @@ func runConvert(ctx *cli.Context) error {
 	log.Trace("AppWorkPath: %s", setting.AppWorkPath)
 	log.Trace("Custom path: %s", setting.CustomPath)
 	log.Trace("Log path: %s", setting.LogRootPath)
-	models.LoadConfigs()
+	setting.InitDBConfig()
 
-	if models.DbCfg.Type != "mysql" {
+	if !setting.Database.UseMySQL {
 		fmt.Println("This command can only be used with a MySQL database")
 		return nil
 	}

cmd/dump.go

@@ -58,7 +58,6 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
 func runDump(ctx *cli.Context) error {
 	setting.NewContext()
 	setting.NewServices() // cannot access session settings otherwise
-	models.LoadConfigs()
 
 	err := models.SetEngine()
 	if err != nil {
@@ -104,8 +103,8 @@ func runDump(ctx *cli.Context) error {
 	}
 
 	targetDBType := ctx.String("database")
-	if len(targetDBType) > 0 && targetDBType != models.DbCfg.Type {
-		log.Printf("Dumping database %s => %s...", models.DbCfg.Type, targetDBType)
+	if len(targetDBType) > 0 && targetDBType != setting.Database.Type {
+		log.Printf("Dumping database %s => %s...", setting.Database.Type, targetDBType)
 	} else {
 		log.Printf("Dumping database...")
 	}

cmd/migrate.go

@@ -30,7 +30,7 @@ func runMigrate(ctx *cli.Context) error {
 	log.Trace("AppWorkPath: %s", setting.AppWorkPath)
 	log.Trace("Custom path: %s", setting.CustomPath)
 	log.Trace("Log path: %s", setting.LogRootPath)
-	models.LoadConfigs()
+	setting.InitDBConfig()
 
 	if err := models.NewEngine(migrations.Migrate); err != nil {
 		log.Fatal("Failed to initialize ORM engine: %v", err)

contrib/pr/checkout.go

@@ -79,16 +79,16 @@ func runPR() {
 	setting.CheckLFSVersion()
 	//models.LoadConfigs()
 	/*
-		models.DbCfg.Type = "sqlite3"
-		models.DbCfg.Path = ":memory:"
-		models.DbCfg.Timeout = 500
+		setting.Database.Type = "sqlite3"
+		setting.Database.Path = ":memory:"
+		setting.Database.Timeout = 500
 	*/
 	db := setting.Cfg.Section("database")
 	db.NewKey("DB_TYPE", "sqlite3")
 	db.NewKey("PATH", ":memory:")
-	setting.LogSQL = true
-	models.LoadConfigs()
+
 	routers.NewServices()
+	setting.Database.LogSQL = true
 	//x, err = xorm.NewEngine("sqlite3", "file::memory:?cache=shared")
 
 	var helper testfixtures.Helper = &testfixtures.SQLite{}

custom/conf/app.ini.sample

@@ -275,6 +275,10 @@ LOG_SQL = true
 DB_RETRIES = 10
 ; Backoff time per DB retry (time.Duration)
 DB_RETRY_BACKOFF = 3s
+; Max idle database connections on connnection pool, default is 0
+MAX_IDLE_CONNS = 0
+; Database connection max life time, default is 3s
+CONN_MAX_LIFETIME = 3s
 
 [indexer]
 ; Issue indexer type, currently support: bleve or db, default is bleve

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -166,6 +166,8 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `LOG_SQL`: **true**: Log the executed SQL.
 - `DB_RETRIES`: **10**: How many ORM init / DB connect attempts allowed.
 - `DB_RETRY_BACKOFF`: **3s**: time.Duration to wait before trying another ORM init / DB connect attempt, if failure occured.
+- `MAX_IDLE_CONNS` **0**: Max idle database connections on connnection pool, default is 0
+- `CONN_MAX_LIFETIME` **3s**: Database connection max lifetime
 
 ## Indexer (`indexer`)
 

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -82,6 +82,8 @@ menu:
 - `CHARSET`: **utf8**: 仅当数据库为 MySQL 时有效, 可以为 "utf8" 或 "utf8mb4"。注意：如果使用 "utf8mb4"，你的 MySQL InnoDB 版本必须在 5.6 以上。
 - `PATH`: Tidb 或者 SQLite3 数据文件存放路径。
 - `LOG_SQL`: **true**: 显示生成的SQL，默认为真。
+- `MAX_IDLE_CONNS` **0**: 最大空闲数据库连接
+- `CONN_MAX_LIFETIME` **3s**: 数据库连接最大存活时间
 
 ## Indexer (`indexer`)
 

integrations/integration_test.go

@@ -59,13 +59,13 @@ func TestMain(m *testing.M) {
 	routes.RegisterRoutes(mac)
 
 	var helper testfixtures.Helper
-	if setting.UseMySQL {
+	if setting.Database.UseMySQL {
 		helper = &testfixtures.MySQL{}
-	} else if setting.UsePostgreSQL {
+	} else if setting.Database.UsePostgreSQL {
 		helper = &testfixtures.PostgreSQL{}
-	} else if setting.UseSQLite3 {
+	} else if setting.Database.UseSQLite3 {
 		helper = &testfixtures.SQLite{}
-	} else if setting.UseMSSQL {
+	} else if setting.Database.UseMSSQL {
 		helper = &testfixtures.SQLServer{}
 	} else {
 		fmt.Println("Unsupported RDBMS for integration tests")
@@ -121,27 +121,27 @@ func initIntegrationTest() {
 	setting.SetCustomPathAndConf("", "", "")
 	setting.NewContext()
 	setting.CheckLFSVersion()
-	models.LoadConfigs()
+	setting.InitDBConfig()
 
 	switch {
-	case setting.UseMySQL:
+	case setting.Database.UseMySQL:
 		db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/",
-			models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host))
+			setting.Database.User, setting.Database.Passwd, setting.Database.Host))
 		defer db.Close()
 		if err != nil {
 			log.Fatalf("sql.Open: %v", err)
 		}
 		if _, err = db.Exec("CREATE DATABASE IF NOT EXISTS testgitea"); err != nil {
 			log.Fatalf("db.Exec: %v", err)
 		}
-	case setting.UsePostgreSQL:
+	case setting.Database.UsePostgreSQL:
 		db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
-			models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.SSLMode))
+			setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
 		defer db.Close()
 		if err != nil {
 			log.Fatalf("sql.Open: %v", err)
 		}
-		rows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", models.DbCfg.Name))
+		rows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
 		if err != nil {
 			log.Fatalf("db.Query: %v", err)
 		}
@@ -153,10 +153,10 @@ func initIntegrationTest() {
 		if _, err = db.Exec("CREATE DATABASE testgitea"); err != nil {
 			log.Fatalf("db.Exec: %v", err)
 		}
-	case setting.UseMSSQL:
-		host, port := models.ParseMSSQLHostPort(models.DbCfg.Host)
+	case setting.Database.UseMSSQL:
+		host, port := setting.ParseMSSQLHostPort(setting.Database.Host)
 		db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
-			host, port, "master", models.DbCfg.User, models.DbCfg.Passwd))
+			host, port, "master", setting.Database.User, setting.Database.Passwd))
 		if err != nil {
 			log.Fatalf("sql.Open: %v", err)
 		}

integrations/lfs_getobject_test.go

@@ -39,7 +39,7 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string
 	assert.NoError(t, err)
 	var lfsMetaObject *models.LFSMetaObject
 
-	if setting.UsePostgreSQL {
+	if setting.Database.UsePostgreSQL {
 		lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID}
 	} else {
 		lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID}

integrations/migration-test/migration_test.go

@@ -54,7 +54,7 @@ func initMigrationTest(t *testing.T) {
 
 	setting.NewContext()
 	setting.CheckLFSVersion()
-	models.LoadConfigs()
+	setting.InitDBConfig()
 	setting.NewLogServices(true)
 }
 
@@ -64,7 +64,7 @@ func availableVersions() ([]string, error) {
 		return nil, err
 	}
 	defer migrationsDir.Close()
-	versionRE, err := regexp.Compile("gitea-v(?P<version>.+)\\." + regexp.QuoteMeta(models.DbCfg.Type) + "\\.sql.gz")
+	versionRE, err := regexp.Compile("gitea-v(?P<version>.+)\\." + regexp.QuoteMeta(setting.Database.Type) + "\\.sql.gz")
 	if err != nil {
 		return nil, err
 	}
@@ -85,7 +85,7 @@ func availableVersions() ([]string, error) {
 }
 
 func readSQLFromFile(version string) (string, error) {
-	filename := fmt.Sprintf("integrations/migration-test/gitea-v%s.%s.sql.gz", version, models.DbCfg.Type)
+	filename := fmt.Sprintf("integrations/migration-test/gitea-v%s.%s.sql.gz", version, setting.Database.Type)
 
 	if _, err := os.Stat(filename); os.IsNotExist(err) {
 		return "", nil
@@ -114,71 +114,71 @@ func restoreOldDB(t *testing.T, version string) bool {
 	data, err := readSQLFromFile(version)
 	assert.NoError(t, err)
 	if len(data) == 0 {
-		integrations.Printf("No db found to restore for %s version: %s\n", models.DbCfg.Type, version)
+		integrations.Printf("No db found to restore for %s version: %s\n", setting.Database.Type, version)
 		return false
 	}
 
 	switch {
-	case setting.UseSQLite3:
-		os.Remove(models.DbCfg.Path)
-		err := os.MkdirAll(path.Dir(models.DbCfg.Path), os.ModePerm)
+	case setting.Database.UseSQLite3:
+		os.Remove(setting.Database.Path)
+		err := os.MkdirAll(path.Dir(setting.Database.Path), os.ModePerm)
 		assert.NoError(t, err)
 
-		db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", models.DbCfg.Path, models.DbCfg.Timeout))
+		db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", setting.Database.Path, setting.Database.Timeout))
 		assert.NoError(t, err)
 		defer db.Close()
 
 		_, err = db.Exec(data)
 		assert.NoError(t, err)
 		db.Close()
 
-	case setting.UseMySQL:
+	case setting.Database.UseMySQL:
 		db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/",
-			models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host))
+			setting.Database.User, setting.Database.Passwd, setting.Database.Host))
 		assert.NoError(t, err)
 		defer db.Close()
 
-		_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", models.DbCfg.Name))
+		_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name))
 		assert.NoError(t, err)
 
-		_, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", models.DbCfg.Name))
+		_, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name))
 		assert.NoError(t, err)
 
 		db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?multiStatements=true",
-			models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.Name))
+			setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name))
 		assert.NoError(t, err)
 		defer db.Close()
 
 		_, err = db.Exec(data)
 		assert.NoError(t, err)
 		db.Close()
 
-	case setting.UsePostgreSQL:
+	case setting.Database.UsePostgreSQL:
 		db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
-			models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.SSLMode))
+			setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
 		assert.NoError(t, err)
 		defer db.Close()
 
-		_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", models.DbCfg.Name))
+		_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name))
 		assert.NoError(t, err)
 
-		_, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", models.DbCfg.Name))
+		_, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name))
 		assert.NoError(t, err)
 		db.Close()
 
 		db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
-			models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.Name, models.DbCfg.SSLMode))
+			setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
 		assert.NoError(t, err)
 		defer db.Close()
 
 		_, err = db.Exec(data)
 		assert.NoError(t, err)
 		db.Close()
 
-	case setting.UseMSSQL:
-		host, port := models.ParseMSSQLHostPort(models.DbCfg.Host)
+	case setting.Database.UseMSSQL:
+		host, port := setting.ParseMSSQLHostPort(setting.Database.Host)
 		db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
-			host, port, "master", models.DbCfg.User, models.DbCfg.Passwd))
+			host, port, "master", setting.Database.User, setting.Database.Passwd))
 		assert.NoError(t, err)
 		defer db.Close()
 
@@ -191,7 +191,7 @@ func restoreOldDB(t *testing.T, version string) bool {
 				dbname := statement[5 : len(statement)-1]
 				db.Close()
 				db, err = sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
-					host, port, dbname, models.DbCfg.User, models.DbCfg.Passwd))
+					host, port, dbname, setting.Database.User, setting.Database.Passwd))
 				assert.NoError(t, err)
 				defer db.Close()
 			}
@@ -210,7 +210,7 @@ func wrappedMigrate(x *xorm.Engine) error {
 
 func doMigrationTest(t *testing.T, version string) {
 	integrations.PrintCurrentTest(t)
-	integrations.Printf("Performing migration test for %s version: %s\n", models.DbCfg.Type, version)
+	integrations.Printf("Performing migration test for %s version: %s\n", setting.Database.Type, version)
 	if !restoreOldDB(t, version) {
 		return
 	}
@@ -227,7 +227,7 @@ func doMigrationTest(t *testing.T, version string) {
 func TestMigrations(t *testing.T) {
 	initMigrationTest(t)
 
-	dialect := models.DbCfg.Type
+	dialect := setting.Database.Type
 	versions, err := availableVersions()
 	assert.NoError(t, err)
 

models/convert.go

@@ -4,11 +4,15 @@
 
 package models
 
-import "fmt"
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/setting"
+)
 
 // ConvertUtf8ToUtf8mb4 converts database and tables from utf8 to utf8mb4 if it's mysql
 func ConvertUtf8ToUtf8mb4() error {
-	_, err := x.Exec(fmt.Sprintf("ALTER DATABASE `%s` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci", DbCfg.Name))
+	_, err := x.Exec(fmt.Sprintf("ALTER DATABASE `%s` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci", setting.Database.Name))
 	if err != nil {
 		return err
 	}

models/migrations/migrations.go

@@ -296,7 +296,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
 	// TODO: This will not work if there are foreign keys
 
 	switch {
-	case setting.UseSQLite3:
+	case setting.Database.UseSQLite3:
 		// First drop the indexes on the columns
 		res, errIndex := sess.Query(fmt.Sprintf("PRAGMA index_list(`%s`)", tableName))
 		if errIndex != nil {
@@ -372,7 +372,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
 			return err
 		}
 
-	case setting.UsePostgreSQL:
+	case setting.Database.UsePostgreSQL:
 		cols := ""
 		for _, col := range columnNames {
 			if cols != "" {
@@ -383,7 +383,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
 		if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil {
 			return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err)
 		}
-	case setting.UseMySQL, setting.UseTiDB:
+	case setting.Database.UseMySQL:
 		// Drop indexes on columns first
 		sql := fmt.Sprintf("SHOW INDEX FROM %s WHERE column_name IN ('%s')", tableName, strings.Join(columnNames, "','"))
 		res, err := sess.Query(sql)
@@ -409,7 +409,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
 		if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil {
 			return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err)
 		}
-	case setting.UseMSSQL:
+	case setting.Database.UseMSSQL:
 		cols := ""
 		for _, col := range columnNames {
 			if cols != "" {

models/migrations/v19.go

@@ -42,7 +42,7 @@ func generateAndMigrateGitHooks(x *xorm.Engine) (err error) {
 		}
 	)
 
-	return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
+	return x.Where("id > 0").BufferSize(setting.Database.IterateBufferSize).Iterate(new(Repository),
 		func(idx int, bean interface{}) error {
 			repo := bean.(*Repository)
 			user := new(User)

models/migrations/v22.go

@@ -42,7 +42,7 @@ func generateAndMigrateWikiGitHooks(x *xorm.Engine) (err error) {
 		}
 	)
 
-	return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
+	return x.Where("id > 0").BufferSize(setting.Database.IterateBufferSize).Iterate(new(Repository),
 		func(idx int, bean interface{}) error {
 			repo := bean.(*Repository)
 			user := new(User)

models/migrations/v26.go

@@ -36,7 +36,7 @@ func generateAndMigrateGitHookChains(x *xorm.Engine) (err error) {
 		hookTpl   = fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType)
 	)
 
-	return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository),
+	return x.Where("id > 0").BufferSize(setting.Database.IterateBufferSize).Iterate(new(Repository),
 		func(idx int, bean interface{}) error {
 			repo := bean.(*Repository)
 			user := new(User)