From 70c0ea2241851831e23386ab9228ea3f252687e3 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Sun, 11 Oct 2026 06:10:30 +0800 Subject: [PATCH] fix(billing): commit asynchronous task quota adjustments atomically Keep wallet/subscription, token, usage counters and task quota in one transaction. Add isolated synthetic regression fixtures and a scoped network-isolated test check. Do not change price selection, rounding, top-up conversion or discount policy. --- .../workflows/task-settlement-regression.yml | 71 +++++++ apps/api-go/model/task_quota_settlement.go | 109 ++++++++++ apps/api-go/service/task_billing.go | 43 ++-- .../service/task_quota_settlement_test.go | 186 ++++++++++++++++++ 4 files changed, 378 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/task-settlement-regression.yml create mode 100644 apps/api-go/model/task_quota_settlement.go create mode 100644 apps/api-go/service/task_quota_settlement_test.go diff --git a/.github/workflows/task-settlement-regression.yml b/.github/workflows/task-settlement-regression.yml new file mode 100644 index 000000000..8b7c18384 --- /dev/null +++ b/.github/workflows/task-settlement-regression.yml @@ -0,0 +1,71 @@ +name: Isolated task settlement regression + +on: + workflow_dispatch: + pull_request: + paths: + - 'apps/api-go/model/task_quota_settlement.go' + - 'apps/api-go/service/task_billing.go' + - 'apps/api-go/service/task_quota_settlement_test.go' + - '.github/workflows/task-settlement-regression.yml' + +permissions: + contents: read + +jobs: + isolated-regression: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 + with: + go-version-file: apps/api-go/go.mod + cache-dependency-path: apps/api-go/go.sum + - name: Record source and compile without executing application code + working-directory: apps/api-go + shell: bash + run: | + set -euo pipefail + evidence="$RUNNER_TEMP/task-settlement-evidence" + mkdir -p "$evidence" + git rev-parse HEAD > "$evidence/revision.txt" + go version > "$evidence/go-version.txt" + sha256sum model/task_quota_settlement.go service/task_billing.go service/task_quota_settlement_test.go > "$evidence/source-sha256.txt" + test -z "$(gofmt -l model/task_quota_settlement.go service/task_billing.go service/task_quota_settlement_test.go)" + go test -p 2 -race -c -o "$RUNNER_TEMP/task-settlement.test" ./service 2>&1 | tee "$evidence/compile.log" + go test -p 2 -c -o "$RUNNER_TEMP/payment-units.test" ./pkg/paymentpricing 2>&1 | tee "$evidence/payment-compile.log" + - name: Run tests with a new network namespace and clean environment + shell: bash + run: | + set -uo pipefail + evidence="$RUNNER_TEMP/task-settlement-evidence" + status=0 + for suite in task-settlement payment-units; do + pattern='.' + if [[ "$suite" == task-settlement ]]; then pattern='^TestTaskQuotaSettlement'; fi + sudo unshare --net -- bash -c ' + set -euo pipefail + ip link set lo up + work=$(mktemp -d) + trap '\''rm -rf "$work"'\'' EXIT + cd "$work" + env -i PATH=/usr/bin:/bin HOME="$work" TMPDIR="$work" \ + "$1" -test.run "$2" -test.count=1 -test.v -test.timeout=120s + ' isolated-test "$RUNNER_TEMP/$suite.test" "$pattern" 2>&1 | tee "$evidence/$suite.log" + code=${PIPESTATUS[0]} + printf '%s\n' "$code" > "$evidence/$suite.exit-code.txt" + if [[ "$code" != 0 ]]; then status=1; fi + done + (cd "$evidence" && sha256sum *.log > logs-sha256.txt) + exit "$status" + - name: Upload source-matched test evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: task-settlement-evidence + path: ${{ runner.temp }}/task-settlement-evidence + retention-days: 7 + if-no-files-found: error diff --git a/apps/api-go/model/task_quota_settlement.go b/apps/api-go/model/task_quota_settlement.go new file mode 100644 index 000000000..5cd0ac539 --- /dev/null +++ b/apps/api-go/model/task_quota_settlement.go @@ -0,0 +1,109 @@ +package model + +import ( + "errors" + + "github.com/LIghtJUNction/api.lmm.best/common" + "gorm.io/gorm" +) + +var ErrTaskQuotaSettlementConflict = errors.New("task quota settlement conflicts with persisted state") + +// ApplyTaskQuotaSettlement commits the funding, token, usage counters and task +// marker together. expectedQuota prevents a stale result from replacing a newer +// settlement. A retry for an already committed target has no financial effect. +// This function does not choose a price, change rounding, or authorize a refund. +func ApplyTaskQuotaSettlement(id int64, expectedQuota, actualQuota int) (int, bool, error) { + if DB == nil || id <= 0 || expectedQuota < 0 || expectedQuota > common.MaxQuota || + actualQuota <= 0 || actualQuota > common.MaxQuota { + return 0, false, gorm.ErrInvalidData + } + var previousQuota, userID int + var tokenKey string + var applied bool + err := DB.Transaction(func(tx *gorm.DB) error { + var task Task + if err := lockForUpdate(tx).Where("id = ?", id).First(&task).Error; err != nil { + return err + } + previousQuota = task.Quota + if task.UserId <= 0 || task.Quota < 0 || task.Quota > common.MaxQuota || + task.RefundStatus != "" || task.Status == TaskStatusFailure { + return ErrTaskQuotaSettlementConflict + } + if task.Quota == actualQuota { + return nil + } + if task.Quota != expectedQuota { + return ErrTaskQuotaSettlementConflict + } + delta := actualQuota - task.Quota + userID = task.UserId + // Keep this compare-and-swap in the same transaction as every balance. + // It is also required on databases where SELECT FOR UPDATE is omitted. + result := tx.Model(&Task{}). + Where("id = ? AND quota = ? AND (refund_status = '' OR refund_status IS NULL)", id, expectedQuota). + Updates(map[string]interface{}{"quota": actualQuota, "updated_at": common.GetTimestamp()}) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return ErrTaskQuotaSettlementConflict + } + + if task.PrivateData.BillingSource == "subscription" && task.PrivateData.SubscriptionId > 0 { + if err := postConsumeUserSubscriptionDeltaTx(tx, task.PrivateData.SubscriptionId, int64(delta)); err != nil { + return err + } + } else if err := ApplyWalletQuotaDelta(tx, userID, -delta); err != nil { + return err + } + + if task.PrivateData.TokenId > 0 { + var token Token + err := lockForUpdate(tx).Where("id = ?", task.PrivateData.TokenId).First(&token).Error + if err == nil { + if token.UserId != userID { + return ErrTaskQuotaSettlementConflict + } + tokenKey = token.Key + if err := tx.Model(&Token{}).Where("id = ? AND user_id = ?", token.Id, userID). + Updates(map[string]interface{}{ + "remain_quota": gorm.Expr("remain_quota - ?", delta), + "used_quota": gorm.Expr("used_quota + ?", delta), + "accessed_time": common.GetTimestamp(), + }).Error; err != nil { + return err + } + } else if !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + // A deleted token must not prevent settlement of its completed task. + } + if err := tx.Model(&User{}).Where("id = ?", userID). + UpdateColumn("used_quota", boundedQuotaCounterExpr("used_quota", delta)).Error; err != nil { + return err + } + if task.ChannelId > 0 { + if err := tx.Model(&Channel{}).Where("id = ?", task.ChannelId). + UpdateColumn("used_quota", boundedQuotaCounterExpr("used_quota", delta)).Error; err != nil { + return err + } + } + applied = true + return nil + }) + if err != nil || !applied { + return previousQuota, false, err + } + // Cache failures cannot turn a committed settlement into a second debit. + if err := invalidateUserCache(userID); err != nil { + common.SysLog("invalidate settled task user cache: " + err.Error()) + } + if tokenKey != "" { + if err := invalidateTokenCacheForMutation(tokenKey); err != nil { + common.SysLog("invalidate settled task token cache: " + err.Error()) + } + } + return previousQuota, true, nil +} diff --git a/apps/api-go/service/task_billing.go b/apps/api-go/service/task_billing.go index de0b15328..c19a5ddf5 100644 --- a/apps/api-go/service/task_billing.go +++ b/apps/api-go/service/task_billing.go @@ -228,43 +228,24 @@ func RefundTaskQuota(ctx context.Context, task *model.Task, reason string) bool // reason 用于日志记录(例如 "token重算" 或 "adaptor调整")。 // clamps 可选:若计算 actualQuota 时发生额度饱和,将其记入日志 admin_info(仅管理员可见)。 func RecalculateTaskQuota(ctx context.Context, task *model.Task, actualQuota int, reason string, clamps ...*common.QuotaClamp) { - if actualQuota <= 0 { + if task == nil || actualQuota <= 0 { return } - preConsumedQuota := task.Quota - quotaDelta := actualQuota - preConsumedQuota - - if quotaDelta == 0 { - logger.LogInfo(ctx, fmt.Sprintf("任务 %s 预扣费准确(%s,%s)", - task.TaskID, logger.LogQuota(actualQuota), reason)) - return - } - - logger.LogInfo(ctx, fmt.Sprintf("任务 %s 差额结算:delta=%s(实际:%s,预扣:%s,%s)", - task.TaskID, - logger.LogQuota(quotaDelta), - logger.LogQuota(actualQuota), - logger.LogQuota(preConsumedQuota), - reason, - )) - - // 调整资金来源 - if err := taskAdjustFunding(task, quotaDelta); err != nil { - logger.LogError(ctx, fmt.Sprintf("差额结算资金调整失败 task %s: %s", task.TaskID, err.Error())) + preConsumedQuota, applied, err := model.ApplyTaskQuotaSettlement(task.ID, task.Quota, actualQuota) + if err != nil { + logger.LogError(ctx, fmt.Sprintf("任务差额结算未提交 task %s: %s", task.TaskID, err.Error())) return } - - // 调整令牌额度 - taskAdjustTokenQuota(ctx, task, quotaDelta) - + // Refresh stale caller snapshots only after the database has committed. task.Quota = actualQuota - if err := task.UpdateQuota(); err != nil { - logger.LogError(ctx, fmt.Sprintf("差额结算回写 quota 失败 task %s: %s", task.TaskID, err.Error())) + if !applied { + return } - - // 提交阶段已经累计过一次请求;结算阶段只调整最终用量。 - model.UpdateUserUsedQuota(task.UserId, quotaDelta) - model.UpdateChannelUsedQuota(task.ChannelId, quotaDelta) + quotaDelta := actualQuota - preConsumedQuota + logger.LogInfo(ctx, fmt.Sprintf("任务 %s 差额结算:delta=%s(实际:%s,预扣:%s,%s)", + task.TaskID, logger.LogQuota(quotaDelta), logger.LogQuota(actualQuota), + logger.LogQuota(preConsumedQuota), reason)) + // LOG_DB can be separate. Logs are not the balance idempotency marker. var logType int var logQuota int diff --git a/apps/api-go/service/task_quota_settlement_test.go b/apps/api-go/service/task_quota_settlement_test.go new file mode 100644 index 000000000..49e943202 --- /dev/null +++ b/apps/api-go/service/task_quota_settlement_test.go @@ -0,0 +1,186 @@ +package service + +import ( + "context" + "errors" + "path/filepath" + "sync" + "testing" + + "github.com/LIghtJUNction/api.lmm.best/common" + "github.com/LIghtJUNction/api.lmm.best/model" + "github.com/glebarez/sqlite" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +// Only a new temporary SQLite database is used. These tests never read a DSN, +// send a payment, or use Redis. They do not prove PostgreSQL lock behaviour. +func taskSettlementFixture(t *testing.T, subscription bool) *model.Task { + t.Helper() + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "settlement.sqlite")), &gorm.Config{}) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + sqlDB.SetMaxOpenConns(1) + t.Cleanup(func() { require.NoError(t, sqlDB.Close()) }) + require.NoError(t, db.AutoMigrate(&model.Task{}, &model.User{}, &model.Token{}, &model.Channel{}, &model.Log{}, &model.UserSubscription{})) + oldDB, oldLog := model.DB, model.LOG_DB + oldRedis, oldMemory := common.RedisEnabled, common.MemoryCacheEnabled + oldBatch, oldExport := common.BatchUpdateEnabled, common.DataExportEnabled + model.DB, model.LOG_DB = db, db + common.RedisEnabled, common.MemoryCacheEnabled = false, false + common.BatchUpdateEnabled, common.DataExportEnabled = false, false + t.Cleanup(func() { + model.DB, model.LOG_DB = oldDB, oldLog + common.RedisEnabled, common.MemoryCacheEnabled = oldRedis, oldMemory + common.BatchUpdateEnabled, common.DataExportEnabled = oldBatch, oldExport + }) + wallet, source, subscriptionID := 900, BillingSourceWallet, 0 + if subscription { + wallet, source, subscriptionID = 1000, BillingSourceSubscription, 9201 + } + seedUser(t, 9201, wallet) + if subscription { + seedSubscription(t, subscriptionID, 9201, 1000, 100) + } + seedToken(t, 9201, 9201, "synthetic-settlement-token", 900) + seedChannel(t, 9201) + seedChargedAccounting(t, 9201, 9201, 9201, 100, 1) + task := makeTask(9201, 9201, 100, 9201, source, subscriptionID) + task.TaskID = "synthetic-settlement-task" + require.NoError(t, task.Insert()) + return task +} + +func assertTaskSettlement(t *testing.T, task *model.Task, final int, subscription bool) { + t.Helper() + var user model.User + var token model.Token + var channel model.Channel + var stored model.Task + require.NoError(t, model.DB.First(&user, task.UserId).Error) + require.NoError(t, model.DB.First(&token, task.PrivateData.TokenId).Error) + require.NoError(t, model.DB.First(&channel, task.ChannelId).Error) + require.NoError(t, model.DB.First(&stored, task.ID).Error) + wallet := 1000 - final + if subscription { + wallet = 1000 + var sub model.UserSubscription + require.NoError(t, model.DB.First(&sub, task.PrivateData.SubscriptionId).Error) + require.EqualValues(t, final, sub.AmountUsed) + } + require.EqualValues(t, wallet, user.Quota) + require.EqualValues(t, final, user.UsedQuota) + require.EqualValues(t, 1, user.RequestCount) + require.EqualValues(t, 1000-final, token.RemainQuota) + require.EqualValues(t, final, token.UsedQuota) + require.EqualValues(t, final, channel.UsedQuota) + require.EqualValues(t, final, stored.Quota) + t.Logf("funding=%s wallet=%d token_remain=%d task=%d user_used=%d channel_used=%d", + task.PrivateData.BillingSource, user.Quota, token.RemainQuota, stored.Quota, user.UsedQuota, channel.UsedQuota) +} + +func TestTaskQuotaSettlementStaleReplay(t *testing.T) { + for _, subscription := range []bool{false, true} { + for _, final := range []int{75, 150} { + t.Run(testTaskSettlementName(subscription, final), func(t *testing.T) { + task := taskSettlementFixture(t, subscription) + stale := *task + RecalculateTaskQuota(context.Background(), task, final, "synthetic final usage") + RecalculateTaskQuota(context.Background(), &stale, final, "same final usage") + assertTaskSettlement(t, task, final, subscription) + require.Equal(t, final, stale.Quota) + }) + } + } +} + +func testTaskSettlementName(subscription bool, final int) string { + name := "wallet" + if subscription { + name = "subscription" + } + if final < 100 { + return name + "_return_difference" + } + return name + "_charge_difference" +} + +func TestTaskQuotaSettlementRollbackAndRetry(t *testing.T) { + for _, subscription := range []bool{false, true} { + for _, table := range []string{"tasks", "tokens", "channels"} { + t.Run(testTaskSettlementName(subscription, 150)+"/"+table, func(t *testing.T) { + task := taskSettlementFixture(t, subscription) + injected := errors.New("synthetic database write failure") + callback := "test:task_settlement_failure" + require.NoError(t, model.DB.Callback().Update().Before("gorm:update").Register(callback, func(tx *gorm.DB) { + if tx.Statement.Table == table { + tx.AddError(injected) + } + })) + t.Cleanup(func() { _ = model.DB.Callback().Update().Remove(callback) }) + _, applied, err := model.ApplyTaskQuotaSettlement(task.ID, 100, 150) + require.ErrorIs(t, err, injected) + require.False(t, applied) + assertTaskSettlement(t, task, 100, subscription) + require.NoError(t, model.DB.Callback().Update().Remove(callback)) + RecalculateTaskQuota(context.Background(), task, 150, "retry after rollback") + assertTaskSettlement(t, task, 150, subscription) + }) + } + } +} + +func TestTaskQuotaSettlementConcurrentReplay(t *testing.T) { + task := taskSettlementFixture(t, false) + const workers = 8 + results := make(chan bool, workers) + errs := make(chan error, workers) + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, applied, err := model.ApplyTaskQuotaSettlement(task.ID, 100, 150) + results <- applied + errs <- err + }() + } + wg.Wait() + close(results) + close(errs) + appliedCount := 0 + for applied := range results { + if applied { + appliedCount++ + } + } + for err := range errs { + require.NoError(t, err) + } + require.Equal(t, 1, appliedCount) + assertTaskSettlement(t, task, 150, false) +} + +func TestTaskQuotaSettlementRejectsConflictingState(t *testing.T) { + task := taskSettlementFixture(t, false) + _, applied, err := model.ApplyTaskQuotaSettlement(task.ID, 100, 150) + require.NoError(t, err) + require.True(t, applied) + _, applied, err = model.ApplyTaskQuotaSettlement(task.ID, 100, 200) + require.ErrorIs(t, err, model.ErrTaskQuotaSettlementConflict) + require.False(t, applied) + assertTaskSettlement(t, task, 150, false) +} + +func TestTaskQuotaSettlementDoesNotRacePreparedRefund(t *testing.T) { + task := taskSettlementFixture(t, false) + _, pending, err := model.PrepareTaskRefundIntent(task.ID, 100) + require.NoError(t, err) + require.True(t, pending) + _, applied, err := model.ApplyTaskQuotaSettlement(task.ID, 100, 150) + require.ErrorIs(t, err, model.ErrTaskQuotaSettlementConflict) + require.False(t, applied) + assertTaskSettlement(t, task, 100, false) +}