Skip to content

Commit 9a86744

Browse files
committed
fix(db): avoid redundant schema migrations on restart
Upgrade the SQLite GORM driver to correctly distinguish unique indexes from constraints. Normalize equivalent MySQL decimal defaults and PostgreSQL CHAR metadata during migration comparison while preserving real schema changes. Validation: 27 fresh-start and rc.26 upgrade scenarios using SQLite 3.50.4, MySQL 5.7.44, PostgreSQL 9.6.24, and ClickHouse 25.8.33.6; 15 upgraded databases passed uniqueness checks. Relational databases issue no DDL on unchanged restarts. Existing ClickHouse TTL synchronization remains unchanged.
1 parent d8cb177 commit 9a86744

7 files changed

Lines changed: 260 additions & 5 deletions

File tree

‎AGENTS.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,8 @@ Do NOT directly import or call `encoding/json` in business code. `json.RawMessag
135135
- Avoid hand-written assertion helpers unless they encode a reusable project-specific invariant.
136136
- When cleaning tests, preserve meaningful regression coverage. If a deleted test covered a real contract indirectly, replace it with a smaller test that asserts that contract directly.
137137

138+
**Documentation files:** Do NOT add new files under `docs/` or any of its subdirectories unless the user explicitly requests it.
139+
138140
### Frontend Rules
139141

140142
- Use `bun` as the preferred package manager and script runner for the frontend (`web/`):

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ require (
1818
github.com/gin-contrib/gzip v0.0.6
1919
github.com/gin-contrib/static v0.0.1
2020
github.com/gin-gonic/gin v1.9.1
21-
github.com/glebarez/sqlite v1.9.0
21+
github.com/glebarez/sqlite v1.11.0
2222
github.com/go-audio/aiff v1.1.0
2323
github.com/go-audio/wav v1.1.0
2424
github.com/go-playground/validator/v10 v10.20.0

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1118,6 +1118,8 @@ github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9g
11181118
github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
11191119
github.com/glebarez/sqlite v1.9.0 h1:Aj6bPA12ZEx5GbSF6XADmCkYXlljPNUY+Zf1EQxynXs=
11201120
github.com/glebarez/sqlite v1.9.0/go.mod h1:YBYCoyupOao60lzp1MVBLEjZfgkq0tdB1voAQ09K9zw=
1121+
github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
1122+
github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
11211123
github.com/go-audio/aiff v1.1.0 h1:m2LYgu/2BarpF2yZnFPWtY3Tp41k0A4y51gDRZZsEuU=
11221124
github.com/go-audio/aiff v1.1.0/go.mod h1:sDik1muYvhPiccClfri0fv6U2fyH/dy4VRWmUz0cz9Q=
11231125
github.com/go-audio/audio v1.0.0 h1:zS9vebldgbQqktK4H0lUqWrG8P0NxCJVqcj7ZpNnwd4=

‎model/main.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -153,10 +153,10 @@ func chooseDB(envName string, isLog bool) (*gorm.DB, common.DatabaseType, error)
153153
common.SysLog("using PostgreSQL as database")
154154
// 同时关闭 pgx 隐式与 GORM 显式预处理语句:命名 prepared statement 与
155155
// 事务池代理(PgBouncer/Neon/Supabase)不兼容,会触发 FATAL 08P01/42P05。
156-
db, err := gorm.Open(postgres.New(postgres.Config{
156+
db, err := gorm.Open(postgresMigrationDialector{postgres.Dialector{Config: &postgres.Config{
157157
DSN: dsn,
158158
PreferSimpleProtocol: true,
159-
}), newGormConfig(false))
159+
}}}, newGormConfig(false))
160160
return db, common.DatabaseTypePostgreSQL, err
161161
}
162162
if strings.HasPrefix(dsn, "local") {
@@ -174,7 +174,7 @@ func chooseDB(envName string, isLog bool) (*gorm.DB, common.DatabaseType, error)
174174
dsn += "?parseTime=true"
175175
}
176176
}
177-
db, err := gorm.Open(mysql.Open(dsn), newGormConfig(true))
177+
db, err := gorm.Open(mysqlMigrationDialector{mysql.Dialector{Config: &mysql.Config{DSN: dsn}}}, newGormConfig(true))
178178
return db, common.DatabaseTypeMySQL, err
179179
}
180180
// Use SQLite

‎model/migration_dialector.go‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
package model
2+
3+
import (
4+
"strings"
5+
6+
"github.com/shopspring/decimal"
7+
"gorm.io/driver/mysql"
8+
"gorm.io/driver/postgres"
9+
"gorm.io/gorm"
10+
"gorm.io/gorm/schema"
11+
)
12+
13+
// Embed the concrete dialectors to retain their transaction/savepoint and other
14+
// optional GORM interfaces. Only schema comparison needs normalization.
15+
type mysqlMigrationDialector struct{ mysql.Dialector }
16+
17+
func (d mysqlMigrationDialector) Migrator(db *gorm.DB) gorm.Migrator {
18+
return mysqlSchemaMigrator{d.Dialector.Migrator(db).(mysql.Migrator)}
19+
}
20+
21+
type mysqlSchemaMigrator struct{ mysql.Migrator }
22+
23+
func (m mysqlSchemaMigrator) MigrateColumn(value any, field *schema.Field, column gorm.ColumnType) error {
24+
if !field.HasDefaultValue || !strings.EqualFold(column.DatabaseTypeName(), "decimal") {
25+
return m.Migrator.MigrateColumn(value, field, column)
26+
}
27+
stored, ok := column.DefaultValue()
28+
if !ok {
29+
return m.Migrator.MigrateColumn(value, field, column)
30+
}
31+
storedNumber, storedErr := decimal.NewFromString(stored)
32+
modelNumber, modelErr := decimal.NewFromString(field.DefaultValue)
33+
if storedErr != nil || modelErr != nil || !storedNumber.Equal(modelNumber) {
34+
return m.Migrator.MigrateColumn(value, field, column)
35+
}
36+
// MySQL pads decimal defaults (0 -> 0.000000). Skip only the equivalent
37+
// default comparison, retaining type/size/null checks. AlterColumn still
38+
// reads the original model, including its default.
39+
comparisonField := *field
40+
comparisonField.HasDefaultValue = false
41+
comparisonField.DefaultValue = ""
42+
comparisonField.DefaultValueInterface = nil
43+
return m.Migrator.MigrateColumn(value, &comparisonField, columnWithoutDefault{column})
44+
}
45+
46+
type migrationColumnType interface{ gorm.ColumnType }
47+
48+
type columnWithoutDefault struct{ migrationColumnType }
49+
50+
func (columnWithoutDefault) DefaultValue() (string, bool) { return "", false }
51+
52+
type postgresMigrationDialector struct{ postgres.Dialector }
53+
54+
func (d postgresMigrationDialector) Migrator(db *gorm.DB) gorm.Migrator {
55+
return postgresSchemaMigrator{d.Dialector.Migrator(db).(postgres.Migrator)}
56+
}
57+
58+
type postgresSchemaMigrator struct{ postgres.Migrator }
59+
60+
func (m postgresSchemaMigrator) MigrateColumn(value any, field *schema.Field, column gorm.ColumnType) error {
61+
if column.DatabaseTypeName() == "bpchar" && strings.HasPrefix(strings.ToLower(string(field.DataType)), "char(") {
62+
// PostgreSQL reports CHAR(n) as bpchar. Normalize the name, retaining
63+
// Length() so a real CHAR length change still triggers migration.
64+
column = charColumnType{column}
65+
}
66+
return m.Migrator.MigrateColumn(value, field, column)
67+
}
68+
69+
type charColumnType struct{ migrationColumnType }
70+
71+
func (charColumnType) DatabaseTypeName() string { return "char" }

‎model/migration_dialector_test.go‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
package model
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"strings"
7+
"testing"
8+
9+
"github.com/QuantumNous/new-api/common"
10+
"github.com/stretchr/testify/assert"
11+
"github.com/stretchr/testify/require"
12+
"gorm.io/gorm"
13+
)
14+
15+
type MigrationIdentityFields struct {
16+
ID int `gorm:"primaryKey"`
17+
Name string `gorm:"size:64;unique"`
18+
Reference string `gorm:"size:64;uniqueIndex"`
19+
Provider string `gorm:"size:32;uniqueIndex:,composite:provider_subject"`
20+
Subject string `gorm:"size:64;uniqueIndex:,composite:provider_subject"`
21+
}
22+
23+
type migrationIdentityV1 struct {
24+
MigrationIdentityFields
25+
Digest string `gorm:"type:char(32)"`
26+
}
27+
28+
type migrationIdentityV2 struct {
29+
MigrationIdentityFields
30+
Digest string `gorm:"type:char(64)"`
31+
Note string `gorm:"size:128"`
32+
}
33+
34+
type migrationConstraintV1 struct {
35+
ID int `gorm:"primaryKey"`
36+
Name string `gorm:"size:64"`
37+
}
38+
39+
type migrationConstraintV2 struct {
40+
ID int `gorm:"primaryKey"`
41+
Name string `gorm:"size:64;unique"`
42+
}
43+
44+
type migrationDecimalV1 struct {
45+
ID int `gorm:"primaryKey"`
46+
Price float64 `gorm:"type:decimal(10,6);default:0"`
47+
}
48+
49+
type migrationDecimalV2 struct {
50+
ID int `gorm:"primaryKey"`
51+
Price float64 `gorm:"type:decimal(12,6);not null;default:0"`
52+
}
53+
54+
type migrationDecimalV3 struct {
55+
ID int `gorm:"primaryKey"`
56+
Price float64 `gorm:"type:decimal(12,6);not null;default:1.25"`
57+
}
58+
59+
func TestMigrationSchemaStability(t *testing.T) {
60+
for _, dialect := range []string{"sqlite", "mysql", "postgres"} {
61+
t.Run(dialect, func(t *testing.T) {
62+
var dsn string
63+
switch dialect {
64+
case "sqlite":
65+
dsn = "local"
66+
previousPath := common.SQLitePath
67+
common.SQLitePath = filepath.Join(t.TempDir(), "migration.db")
68+
t.Cleanup(func() { common.SQLitePath = previousPath })
69+
case "mysql":
70+
dsn = os.Getenv("TEST_MYSQL_DSN")
71+
case "postgres":
72+
dsn = os.Getenv("TEST_POSTGRES_DSN")
73+
}
74+
if dsn == "" {
75+
t.Skip("test database DSN is not configured")
76+
}
77+
t.Setenv("MIGRATION_TEST_DSN", dsn)
78+
db, _, err := chooseDB("MIGRATION_TEST_DSN", false)
79+
require.NoError(t, err)
80+
sqlDB, err := db.DB()
81+
require.NoError(t, err)
82+
t.Cleanup(func() { _ = sqlDB.Close() })
83+
recorder := &migrationSQLRecorder{}
84+
db = db.Session(&gorm.Session{Logger: recorder})
85+
86+
t.Run("identity_and_indexes", func(t *testing.T) {
87+
const table = "migration_identity_test"
88+
t.Cleanup(func() { _ = db.Migrator().DropTable(table) })
89+
require.NoError(t, db.Table(table).AutoMigrate(&migrationIdentityV1{}))
90+
row := migrationIdentityV1{
91+
MigrationIdentityFields: MigrationIdentityFields{ID: 1, Name: "root", Reference: "token-reference", Provider: "oidc", Subject: "subject"},
92+
Digest: "old-digest",
93+
}
94+
require.NoError(t, db.Table(table).Create(&row).Error)
95+
recorder.reset()
96+
require.NoError(t, db.Table(table).AutoMigrate(&migrationIdentityV1{}))
97+
assert.Empty(t, recorder.schemaMutations())
98+
99+
require.NoError(t, db.Table(table).AutoMigrate(&migrationIdentityV2{}))
100+
columns, err := db.Table(table).Migrator().ColumnTypes(&migrationIdentityV2{})
101+
require.NoError(t, err)
102+
for _, column := range columns {
103+
if column.Name() == "digest" {
104+
length, ok := column.Length()
105+
require.True(t, ok)
106+
assert.EqualValues(t, 64, length)
107+
}
108+
}
109+
assert.True(t, db.Table(table).Migrator().HasColumn(&migrationIdentityV2{}, "note"))
110+
recorder.reset()
111+
require.NoError(t, db.Table(table).AutoMigrate(&migrationIdentityV2{}))
112+
assert.Empty(t, recorder.schemaMutations())
113+
var saved migrationIdentityV2
114+
require.NoError(t, db.Table(table).First(&saved, 1).Error)
115+
assert.Equal(t, row.MigrationIdentityFields, saved.MigrationIdentityFields)
116+
expectedDigest := row.Digest
117+
if dialect == "postgres" {
118+
expectedDigest += strings.Repeat(" ", 64-len(row.Digest))
119+
}
120+
assert.Equal(t, expectedDigest, saved.Digest)
121+
for _, duplicate := range []migrationIdentityV2{
122+
{MigrationIdentityFields: MigrationIdentityFields{Name: "root", Reference: "other-1", Provider: "other", Subject: "1"}},
123+
{MigrationIdentityFields: MigrationIdentityFields{Name: "other-2", Reference: "token-reference", Provider: "other", Subject: "2"}},
124+
{MigrationIdentityFields: MigrationIdentityFields{Name: "other-3", Reference: "other-3", Provider: "oidc", Subject: "subject"}},
125+
} {
126+
assert.Error(t, db.Table(table).Create(&duplicate).Error)
127+
}
128+
})
129+
130+
t.Run("unique_constraint_changes", func(t *testing.T) {
131+
const table = "migration_constraint_test"
132+
t.Cleanup(func() { _ = db.Migrator().DropTable(table) })
133+
require.NoError(t, db.Table(table).AutoMigrate(&migrationConstraintV1{}))
134+
require.NoError(t, db.Table(table).Create(&migrationConstraintV1{Name: "existing"}).Error)
135+
require.NoError(t, db.Table(table).AutoMigrate(&migrationConstraintV2{}))
136+
assert.Error(t, db.Table(table).Create(&migrationConstraintV2{Name: "existing"}).Error)
137+
recorder.reset()
138+
require.NoError(t, db.Table(table).AutoMigrate(&migrationConstraintV2{}))
139+
assert.Empty(t, recorder.schemaMutations())
140+
require.NoError(t, db.Table(table).AutoMigrate(&migrationConstraintV1{}))
141+
require.NoError(t, db.Table(table).Create(&migrationConstraintV1{Name: "existing"}).Error)
142+
})
143+
144+
if dialect == "mysql" {
145+
t.Run("decimal_default_and_real_changes", func(t *testing.T) {
146+
const table = "migration_decimal_test"
147+
t.Cleanup(func() { _ = db.Migrator().DropTable(table) })
148+
require.NoError(t, db.Table(table).AutoMigrate(&migrationDecimalV1{}))
149+
require.NoError(t, db.Table(table).Create(&migrationDecimalV1{ID: 1, Price: 12.345678}).Error)
150+
for _, target := range []any{&migrationDecimalV1{}, &migrationDecimalV2{}, &migrationDecimalV3{}} {
151+
require.NoError(t, db.Table(table).AutoMigrate(target))
152+
recorder.reset()
153+
require.NoError(t, db.Table(table).AutoMigrate(target))
154+
assert.Empty(t, recorder.schemaMutations())
155+
}
156+
columns, err := db.Table(table).Migrator().ColumnTypes(&migrationDecimalV3{})
157+
require.NoError(t, err)
158+
for _, column := range columns {
159+
if column.Name() == "price" {
160+
precision, scale, ok := column.DecimalSize()
161+
require.True(t, ok)
162+
assert.EqualValues(t, 12, precision)
163+
assert.EqualValues(t, 6, scale)
164+
nullable, ok := column.Nullable()
165+
require.True(t, ok)
166+
assert.False(t, nullable)
167+
}
168+
}
169+
require.NoError(t, db.Table(table).Create(&map[string]any{"id": 2}).Error)
170+
var prices []float64
171+
require.NoError(t, db.Table(table).Order("id").Pluck("price", &prices).Error)
172+
assert.Equal(t, []float64{12.345678, 1.25}, prices)
173+
})
174+
}
175+
})
176+
}
177+
}

‎model/user_session_migration_test.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,10 @@ func (recorder *migrationSQLRecorder) schemaMutations() []string {
6060
if strings.HasPrefix(normalized, "ALTER TABLE") ||
6161
strings.HasPrefix(normalized, "CREATE TABLE") ||
6262
strings.HasPrefix(normalized, "DROP TABLE") ||
63-
strings.HasPrefix(normalized, "RENAME TABLE") {
63+
strings.HasPrefix(normalized, "RENAME TABLE") ||
64+
strings.HasPrefix(normalized, "CREATE INDEX") ||
65+
strings.HasPrefix(normalized, "CREATE UNIQUE INDEX") ||
66+
strings.HasPrefix(normalized, "DROP INDEX") {
6467
mutations = append(mutations, statement)
6568
}
6669
}

0 commit comments

Comments
 (0)