diff --git a/Makefile b/Makefile index 34f646f37c644..743e97b4ca7b4 100644 --- a/Makefile +++ b/Makefile @@ -556,7 +556,7 @@ mock_import: mockgen tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/lightning/backend Backend,EngineWriter,TargetInfoGetter > br/pkg/mock/backend.go tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/lightning/common ChunkFlushStatus > br/pkg/mock/common.go tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/lightning/backend/encode Encoder,EncodingBuilder,Rows,Row > br/pkg/mock/encode.go - tools/bin/mockgen -package mocklocal github.com/pingcap/tidb/pkg/lightning/backend/local DiskUsage,TiKVModeSwitcher,StoreHelper > br/pkg/mock/mocklocal/local.go + tools/bin/mockgen -package mocklocal github.com/pingcap/tidb/pkg/ingestor/ingestctrl DiskUsage,TiKVModeSwitcher,StoreHelper > br/pkg/mock/mocklocal/local.go tools/bin/mockgen -package mock github.com/pingcap/tidb/br/pkg/utils TaskRegister > br/pkg/mock/task_register.go tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/dxf/framework/taskexecutor TaskTable,TaskExecutor,Extension > pkg/dxf/framework/mock/task_executor_mock.go tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/dxf/framework/scheduler Scheduler,CleanUpRoutine,TaskManager > pkg/dxf/framework/mock/scheduler_mock.go diff --git a/br/pkg/mock/mocklocal/local.go b/br/pkg/mock/mocklocal/local.go index b6c70a530b912..0d0a0591caa7e 100644 --- a/br/pkg/mock/mocklocal/local.go +++ b/br/pkg/mock/mocklocal/local.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/pingcap/tidb/pkg/lightning/backend/local (interfaces: DiskUsage,TiKVModeSwitcher,StoreHelper) +// Source: github.com/pingcap/tidb/pkg/ingestor/ingestctrl (interfaces: DiskUsage,TiKVModeSwitcher,StoreHelper) // // Generated by this command: // -// mockgen -package mocklocal github.com/pingcap/tidb/pkg/lightning/backend/local DiskUsage,TiKVModeSwitcher,StoreHelper +// mockgen -package mocklocal github.com/pingcap/tidb/pkg/ingestor/ingestctrl DiskUsage,TiKVModeSwitcher,StoreHelper // // Package mocklocal is a generated GoMock package. package mocklocal diff --git a/build/nogo_config.json b/build/nogo_config.json index f3f2fcb89c5d2..468d33e6f3de7 100644 --- a/build/nogo_config.json +++ b/build/nogo_config.json @@ -423,7 +423,7 @@ "pkg/lightning/mydump/": "ignore mydump code", "pkg/lightning/checkpoints/checkpoints.go": "ignore checkpoints code", "pkg/lightning/checkpoints/glue_checkpoint.go": "ignore glue_checkpoint code", - "pkg/lightning/backend/local/": "ignore local code", + "pkg/ingestor/ingestctrl/": "ignore local code", "pkg/lightning/backend/tidb/tidb.go": "ignore tidb code", "pkg/lightning/backend/tidb/tidb_test.go": "ignore tidb code", "pkg/lightning/checkpoints/checkpoints_test.go": "ignore checkpoints code", @@ -657,7 +657,7 @@ "pkg/util/chunk": "ignore util/chunk", "pkg/lightning/mydump/": "more than 50", "lightning/pkg/importer/": "more than 50", - "pkg/lightning/backend/local/": "more than 50", + "pkg/ingestor/ingestctrl/": "more than 50", "br/pkg/restore/": "more than 50", "br/pkg/storage/": "more than 50", "pkg/ddl/tests/partition/": "more than 50" @@ -1065,8 +1065,8 @@ "br/pkg/streamhelper/advancer_cliext.go": "github.com/golang/protobuf deprecated", "pkg/lightning/checkpoints/checkpoints.go": "cfg.TikvImporter.Addr is deprecated", "pkg/lightning/checkpoints/glue_checkpoint.go": "cfg.TikvImporter.Addr is deprecated", - "pkg/lightning/backend/local/local.go": "grpc Compressor/Decompressor is deprecated", - "pkg/lightning/backend/local/compress.go": "grpc Compressor/Decompressor is deprecated", + "pkg/ingestor/ingestctrl/local.go": "grpc Compressor/Decompressor is deprecated", + "pkg/ingestor/ingestctrl/compress.go": "grpc Compressor/Decompressor is deprecated", "pkg/domain/infosync/resource_manager_client.go": "github.com/golang/protobuf deprecated" }, "only_files": { diff --git a/lightning/pkg/importer/BUILD.bazel b/lightning/pkg/importer/BUILD.bazel index 6187c50c6a549..cfb58c73d397a 100644 --- a/lightning/pkg/importer/BUILD.bazel +++ b/lightning/pkg/importer/BUILD.bazel @@ -33,12 +33,12 @@ go_library( "//pkg/config", "//pkg/ddl", "//pkg/errno", + "//pkg/ingestor/ingestctrl", "//pkg/keyspace", "//pkg/kv", "//pkg/lightning/backend", "//pkg/lightning/backend/encode", "//pkg/lightning/backend/kv", - "//pkg/lightning/backend/local", "//pkg/lightning/backend/tidb", "//pkg/lightning/common", "//pkg/lightning/config", @@ -139,11 +139,11 @@ go_test( "//pkg/config/kerneltype", "//pkg/ddl", "//pkg/errno", + "//pkg/ingestor/ingestctrl", "//pkg/kv", "//pkg/lightning/backend", "//pkg/lightning/backend/encode", "//pkg/lightning/backend/kv", - "//pkg/lightning/backend/local", "//pkg/lightning/backend/tidb", "//pkg/lightning/common", "//pkg/lightning/config", diff --git a/lightning/pkg/importer/checksum_helper.go b/lightning/pkg/importer/checksum_helper.go index 5ff3c6f1ae07a..1311795b6b2e0 100644 --- a/lightning/pkg/importer/checksum_helper.go +++ b/lightning/pkg/importer/checksum_helper.go @@ -19,8 +19,8 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/br/pkg/pdutil" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/config" "github.com/pingcap/tidb/pkg/lightning/importdef" @@ -31,7 +31,7 @@ import ( ) // NewChecksumManager creates a new checksum manager. -func NewChecksumManager(ctx context.Context, rc *Controller, store kv.Storage) (local.ChecksumManager, error) { +func NewChecksumManager(ctx context.Context, rc *Controller, store kv.Storage) (ingestctrl.ChecksumManager, error) { // if we don't need checksum, just return nil if rc.cfg.TikvImporter.Backend == config.BackendTiDB || rc.cfg.PostRestore.Checksum == config.OpLevelOff { return nil, nil @@ -43,20 +43,20 @@ func NewChecksumManager(ctx context.Context, rc *Controller, store kv.Storage) ( } // for v4.0.0 or upper, we can use the gc ttl api - var manager local.ChecksumManager + var manager ingestctrl.ChecksumManager if pdVersion.Major >= 4 && !rc.cfg.PostRestore.ChecksumViaSQL { backoffWeight, err := common.GetBackoffWeightFromDB(ctx, rc.db) // only set backoff weight when it's smaller than default value - if err == nil && backoffWeight >= local.DefaultBackoffWeight { + if err == nil && backoffWeight >= ingestctrl.DefaultBackoffWeight { logutil.Logger(ctx).Info("get tidb_backoff_weight", zap.Int("backoff_weight", backoffWeight)) } else { - logutil.Logger(ctx).Info("set tidb_backoff_weight to default", zap.Int("backoff_weight", local.DefaultBackoffWeight)) - backoffWeight = local.DefaultBackoffWeight + logutil.Logger(ctx).Info("set tidb_backoff_weight to default", zap.Int("backoff_weight", ingestctrl.DefaultBackoffWeight)) + backoffWeight = ingestctrl.DefaultBackoffWeight } - manager = local.NewTiKVChecksumManager(store.GetClient(), rc.pdCli, uint(rc.cfg.TiDB.DistSQLScanConcurrency), backoffWeight, rc.resourceGroupName, rc.taskType) + manager = ingestctrl.NewTiKVChecksumManager(store.GetClient(), rc.pdCli, uint(rc.cfg.TiDB.DistSQLScanConcurrency), backoffWeight, rc.resourceGroupName, rc.taskType) } else { - manager = local.NewTiDBChecksumExecutor(rc.db) + manager = ingestctrl.NewTiDBChecksumExecutor(rc.db) } return manager, nil @@ -64,9 +64,9 @@ func NewChecksumManager(ctx context.Context, rc *Controller, store kv.Storage) ( // DoChecksum do checksum for tables. // table should be in ., format. e.g. foo.bar -func DoChecksum(ctx context.Context, table *importdef.TableInfo) (*local.RemoteChecksum, error) { +func DoChecksum(ctx context.Context, table *importdef.TableInfo) (*ingestctrl.RemoteChecksum, error) { var err error - manager, ok := ctx.Value(&checksumManagerKey).(local.ChecksumManager) + manager, ok := ctx.Value(&checksumManagerKey).(ingestctrl.ChecksumManager) if !ok { return nil, errors.New("No gcLifeTimeManager found in context, check context initialization") } diff --git a/lightning/pkg/importer/get_pre_info.go b/lightning/pkg/importer/get_pre_info.go index 3ad4ca7b720a1..5745a2ef6dc30 100644 --- a/lightning/pkg/importer/get_pre_info.go +++ b/lightning/pkg/importer/get_pre_info.go @@ -30,10 +30,10 @@ import ( ropts "github.com/pingcap/tidb/lightning/pkg/importer/opts" "github.com/pingcap/tidb/pkg/ddl" "github.com/pingcap/tidb/pkg/errno" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/lightning/backend" "github.com/pingcap/tidb/pkg/lightning/backend/encode" "github.com/pingcap/tidb/pkg/lightning/backend/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/backend/tidb" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/config" @@ -145,7 +145,7 @@ func NewTargetInfoGetterImpl( case config.BackendTiDB: backendTargetInfoGetter = tidb.NewTargetInfoGetter(targetDB) case config.BackendLocal: - backendTargetInfoGetter = local.NewTargetInfoGetter(tls, targetDB, pdHTTPCli) + backendTargetInfoGetter = ingestctrl.NewTargetInfoGetter(tls, targetDB, pdHTTPCli) default: return nil, common.ErrUnknownBackend.GenWithStackByArgs(cfg.TikvImporter.Backend) } @@ -299,7 +299,7 @@ func NewPreImportInfoGetter( case config.BackendTiDB: encBuilder = tidb.NewEncodingBuilder() case config.BackendLocal: - encBuilder = local.NewEncodingBuilder(context.Background()) + encBuilder = ingestctrl.NewEncodingBuilder(context.Background()) default: return nil, common.ErrUnknownBackend.GenWithStackByArgs(cfg.TikvImporter.Backend) } diff --git a/lightning/pkg/importer/import.go b/lightning/pkg/importer/import.go index ad645c50b9098..ac8f5770705d6 100644 --- a/lightning/pkg/importer/import.go +++ b/lightning/pkg/importer/import.go @@ -40,11 +40,11 @@ import ( "github.com/pingcap/tidb/lightning/pkg/errormanager" "github.com/pingcap/tidb/lightning/pkg/progress" tidbconfig "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/keyspace" tidbkv "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/lightning/backend" "github.com/pingcap/tidb/pkg/lightning/backend/encode" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/backend/tidb" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/config" @@ -240,7 +240,7 @@ type Controller struct { preInfoGetter PreImportInfoGetter precheckItemBuilder *PrecheckItemBuilder encBuilder encode.EncodingBuilder - tikvModeSwitcher local.TiKVModeSwitcher + tikvModeSwitcher ingestctrl.TiKVModeSwitcher keyspaceName string resourceGroupName string @@ -369,19 +369,19 @@ func NewImportControllerWithPauser( encodingBuilder = tidb.NewEncodingBuilder() backendObj = tidb.NewTiDBBackend(ctx, db, cfg, errorMgr) case config.BackendLocal: - var rLimit local.RlimT - rLimit, err = local.GetSystemRLimit() + var rLimit ingestctrl.RlimT + rLimit, err = ingestctrl.GetSystemRLimit() if err != nil { return nil, err } - maxOpenFiles := int(rLimit / local.RlimT(cfg.App.TableConcurrency)) + maxOpenFiles := int(rLimit / ingestctrl.RlimT(cfg.App.TableConcurrency)) // check overflow if maxOpenFiles < 0 { maxOpenFiles = math.MaxInt32 } addrs := strings.Split(cfg.TiDB.PdAddr, ",") - pdCli, err = pd.NewClientWithContext(ctx, componentName, addrs, tls.ToPDSecurityOption(), local.PDClientOptions()...) + pdCli, err = pd.NewClientWithContext(ctx, componentName, addrs, tls.ToPDSecurityOption(), ingestctrl.PDClientOptions()...) if err != nil { return nil, errors.Trace(err) } @@ -402,14 +402,14 @@ func NewImportControllerWithPauser( } // simple wraps PD client, so no need to close it separately. - pdCliForTiKV, err := local.NewCodecPDClient(pdCli, p.KeyspaceName) + pdCliForTiKV, err := ingestctrl.NewCodecPDClient(pdCli, p.KeyspaceName) if err != nil { return nil, common.ErrCreatePDClient.Wrap(err).GenWithStackByArgs() } initGlobalConfig(tls.ToTiKVSecurityConfig()) - encodingBuilder = local.NewEncodingBuilder(ctx) + encodingBuilder = ingestctrl.NewEncodingBuilder(ctx) // get resource group name. exec := common.SQLWithRetry{ @@ -445,8 +445,8 @@ func NewImportControllerWithPauser( if isRaftKV2 { raftKV2SwitchModeDuration = cfg.Cron.SwitchMode.Duration } - backendConfig := local.NewBackendConfig(cfg, maxOpenFiles, p.KeyspaceName, p.ResourceGroupName, p.TaskType, raftKV2SwitchModeDuration) - backendObj, err = local.NewBackend(ctx, tls, backendConfig, pdCliForTiKV) + backendConfig := ingestctrl.NewBackendConfig(cfg, maxOpenFiles, p.KeyspaceName, p.ResourceGroupName, p.TaskType, raftKV2SwitchModeDuration) + backendObj, err = ingestctrl.NewBackend(ctx, tls, backendConfig, pdCliForTiKV) if err != nil { return nil, common.NormalizeOrWrapErr(common.ErrUnknown, err) } @@ -479,7 +479,7 @@ func NewImportControllerWithPauser( var wrapper backend.TargetInfoGetter if cfg.TikvImporter.Backend == config.BackendLocal { - wrapper = local.NewTargetInfoGetter(tls, db, pdHTTPCli) + wrapper = ingestctrl.NewTargetInfoGetter(tls, db, pdHTTPCli) } else { wrapper = tidb.NewTargetInfoGetter(db) } @@ -544,7 +544,7 @@ func NewImportControllerWithPauser( preInfoGetter: preInfoGetter, precheckItemBuilder: preCheckBuilder, encBuilder: encodingBuilder, - tikvModeSwitcher: local.NewTiKVModeSwitcher(tls.TLSConfig(), pdHTTPCli, logutil.Logger(ctx)), + tikvModeSwitcher: ingestctrl.NewTiKVModeSwitcher(tls.TLSConfig(), pdHTTPCli, logutil.Logger(ctx)), keyspaceName: p.KeyspaceName, resourceGroupName: p.ResourceGroupName, @@ -760,7 +760,7 @@ func verifyLocalFile(ctx context.Context, cpdb checkpoints.DB, dir string) error for tableName, engineIDs := range targetTables { for _, engineID := range engineIDs { _, eID := backend.MakeUUID(tableName, int64(engineID)) - file := local.Engine{UUID: eID} + file := ingestctrl.Engine{UUID: eID} err := file.Exist(dir) if err != nil { logutil.Logger(ctx).Error("can't find local file", @@ -1705,7 +1705,7 @@ func (rc *Controller) enforceDiskQuota(ctx context.Context) { return } - localBackend := rc.backend.(*local.Backend) + localBackend := rc.backend.(*ingestctrl.Backend) go func() { // locker is assigned when we detect the disk quota is exceeded. // before the disk quota is confirmed exceeded, we keep the diskQuotaLock @@ -1733,7 +1733,7 @@ func (rc *Controller) enforceDiskQuota(ctx context.Context) { } quota := int64(rc.cfg.TikvImporter.DiskQuota) - largeEngines, inProgressLargeEngines, totalDiskSize, totalMemSize := local.CheckDiskQuota(localBackend, quota) + largeEngines, inProgressLargeEngines, totalDiskSize, totalMemSize := ingestctrl.CheckDiskQuota(localBackend, quota) if m, ok := metric.FromContext(ctx); ok { m.LocalStorageUsageBytesGauge.WithLabelValues("disk").Set(float64(totalDiskSize)) m.LocalStorageUsageBytesGauge.WithLabelValues("mem").Set(float64(totalMemSize)) diff --git a/lightning/pkg/importer/meta_manager_test.go b/lightning/pkg/importer/meta_manager_test.go index 774aec021b749..859a0db9c579b 100644 --- a/lightning/pkg/importer/meta_manager_test.go +++ b/lightning/pkg/importer/meta_manager_test.go @@ -24,8 +24,8 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/ddl" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/importdef" "github.com/pingcap/tidb/pkg/lightning/log" @@ -361,7 +361,7 @@ func (s *metaMgrSuite) prepareMockInner(rowsVal [][]driver.Value, nextRowID *int s.mockDB.ExpectExec("\\QUPDATE `test`.`table_meta` SET total_kvs_base = ?, total_bytes_base = ?, checksum_base = ?, status = ? WHERE table_id = ? AND task_id = ?\\E"). WithArgs(checksum.SumKVS(), checksum.SumSize(), checksum.Sum(), metaStatusRestoreStarted.String(), int64(1), int64(1)). WillReturnResult(sqlmock.NewResult(int64(0), int64(1))) - s.checksumMgr.checksum = local.RemoteChecksum{ + s.checksumMgr.checksum = ingestctrl.RemoteChecksum{ TotalBytes: checksum.SumSize(), TotalKVs: checksum.SumKVS(), Checksum: checksum.Sum(), @@ -438,13 +438,13 @@ func TestCheckTasksExclusively(t *testing.T) { } type testChecksumMgr struct { - checksum local.RemoteChecksum + checksum ingestctrl.RemoteChecksum callCnt int } -var _ local.ChecksumManager = (*testChecksumMgr)(nil) +var _ ingestctrl.ChecksumManager = (*testChecksumMgr)(nil) -func (t *testChecksumMgr) Checksum(ctx context.Context, tableInfo *importdef.TableInfo) (*local.RemoteChecksum, error) { +func (t *testChecksumMgr) Checksum(ctx context.Context, tableInfo *importdef.TableInfo) (*ingestctrl.RemoteChecksum, error) { t.callCnt++ return &t.checksum, nil } diff --git a/lightning/pkg/importer/table_import.go b/lightning/pkg/importer/table_import.go index 97ce0044a4b7d..f33edea10f350 100644 --- a/lightning/pkg/importer/table_import.go +++ b/lightning/pkg/importer/table_import.go @@ -33,11 +33,11 @@ import ( "github.com/pingcap/tidb/lightning/pkg/checkpoints" pkgprogress "github.com/pingcap/tidb/lightning/pkg/progress" "github.com/pingcap/tidb/pkg/errno" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" tidbkv "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/lightning/backend" "github.com/pingcap/tidb/pkg/lightning/backend/encode" "github.com/pingcap/tidb/pkg/lightning/backend/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/config" "github.com/pingcap/tidb/pkg/lightning/importdef" @@ -223,8 +223,8 @@ func (tr *TableImporter) importTable( // 2. Do duplicate detection if needed if isLocalBackend(rc.cfg) && rc.cfg.Conflict.PrecheckConflictBeforeImport && rc.cfg.Conflict.Strategy != config.NoneOnDup { _, uuid := backend.MakeUUID(tr.tableName, common.IndexEngineID) - workingDir := filepath.Join(rc.cfg.TikvImporter.SortedKVDir, uuid.String()+local.DupDetectDirSuffix) - resultDir := filepath.Join(rc.cfg.TikvImporter.SortedKVDir, uuid.String()+local.DupResultDirSuffix) + workingDir := filepath.Join(rc.cfg.TikvImporter.SortedKVDir, uuid.String()+ingestctrl.DupDetectDirSuffix) + resultDir := filepath.Join(rc.cfg.TikvImporter.SortedKVDir, uuid.String()+ingestctrl.DupResultDirSuffix) dupIgnoreRows, err := extsort.OpenDiskSorter(resultDir, &extsort.DiskSorterOptions{ Concurrency: rc.cfg.App.RegionConcurrency, @@ -452,7 +452,7 @@ func estimateCompactionThreshold(files []mydump.FileInfo, cp *checkpoints.TableC } totalRawFileSize *= factor - return local.EstimateCompactionThreshold2(totalRawFileSize) + return ingestctrl.EstimateCompactionThreshold2(totalRawFileSize) } func (tr *TableImporter) importEngines(pCtx context.Context, rc *Controller, cp *checkpoints.TableCheckpoint) error { @@ -708,7 +708,7 @@ func (tr *TableImporter) preprocessEngine( if !tr.tableMeta.IsRowOrdered { dataEngineCfg.Local.Compact = true dataEngineCfg.Local.CompactConcurrency = 4 - dataEngineCfg.Local.CompactThreshold = local.CompactionUpperThreshold + dataEngineCfg.Local.CompactThreshold = ingestctrl.CompactionUpperThreshold } dataEngine, err := rc.engineMgr.OpenEngine(ctx, dataEngineCfg, tr.tableName, engineID) if err != nil { @@ -1067,10 +1067,10 @@ func (tr *TableImporter) postProcess( // if we came here, it must be a local backend. // todo: remove this cast after we refactor the backend interface. Physical mode is so different, we shouldn't // try to abstract it with logical mode. - var dupeController *local.DupeController + var dupeController *ingestctrl.DupeController hasDupe := false if rc.cfg.Conflict.Strategy != config.NoneOnDup { - localBackend := rc.backend.(*local.Backend) + localBackend := rc.backend.(*ingestctrl.Backend) var err error dupeController, err = localBackend.GetDupeController(ctx, rc.cfg.TikvImporter.RangeConcurrency*2, rc.errorMgr) if err != nil { @@ -1134,7 +1134,7 @@ func (tr *TableImporter) postProcess( tr.logger.Info("merged local checksum", zap.Object("checksum", &localChecksum)) } - var remoteChecksum *local.RemoteChecksum + var remoteChecksum *ingestctrl.RemoteChecksum remoteChecksum, err = DoChecksum(ctx, tr.tableInfo) failpoint.Inject("checksum-error", func() { tr.logger.Info("failpoint checksum-error injected.") @@ -1361,7 +1361,7 @@ func (tr *TableImporter) importKV( } err := closedEngine.Import(ctx, regionSplitSize, regionSplitKeys) if common.ErrFoundDuplicateKeys.Equal(err) { - err = local.ConvertToErrFoundConflictRecords(err, tr.encTable) + err = ingestctrl.ConvertToErrFoundConflictRecords(err, tr.encTable) } saveCpErr := rc.saveStatusCheckpoint(ctx, tr.tableName, closedEngine.GetID(), err, checkpoints.CheckpointStatusImported) // Don't clean up when save checkpoint failed, because we will verifyLocalFile and import engine again after restart. @@ -1386,7 +1386,7 @@ func (tr *TableImporter) importKV( } // do checksum for each table. -func (tr *TableImporter) compareChecksum(remoteChecksum *local.RemoteChecksum, localChecksum verify.KVChecksum) error { +func (tr *TableImporter) compareChecksum(remoteChecksum *ingestctrl.RemoteChecksum, localChecksum verify.KVChecksum) error { if remoteChecksum.Checksum != localChecksum.Sum() || remoteChecksum.TotalKVs != localChecksum.SumKVS() || remoteChecksum.TotalBytes != localChecksum.SumSize() { diff --git a/lightning/pkg/importer/table_import_test.go b/lightning/pkg/importer/table_import_test.go index 73e89e17fd8ed..f6acce1742db7 100644 --- a/lightning/pkg/importer/table_import_test.go +++ b/lightning/pkg/importer/table_import_test.go @@ -40,10 +40,10 @@ import ( "github.com/pingcap/tidb/lightning/pkg/precheck" "github.com/pingcap/tidb/lightning/pkg/progress" "github.com/pingcap/tidb/pkg/ddl" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/lightning/backend" "github.com/pingcap/tidb/pkg/lightning/backend/encode" "github.com/pingcap/tidb/pkg/lightning/backend/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/backend/tidb" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/config" @@ -768,7 +768,7 @@ func (s *tableRestoreSuite) TestInitializeColumnsGenerated() { func MockDoChecksumCtx(db *sql.DB) context.Context { ctx := context.Background() - manager := local.NewTiDBChecksumExecutor(db) + manager := ingestctrl.NewTiDBChecksumExecutor(db) return context.WithValue(ctx, &checksumManagerKey, manager) } diff --git a/lightning/pkg/server/BUILD.bazel b/lightning/pkg/server/BUILD.bazel index bc120cd3f78f2..93a1191ad5d03 100644 --- a/lightning/pkg/server/BUILD.bazel +++ b/lightning/pkg/server/BUILD.bazel @@ -19,8 +19,8 @@ go_library( "//lightning/pkg/importinto", "//lightning/pkg/progress", "//pkg/expression", + "//pkg/ingestor/ingestctrl", "//pkg/lightning/backend", - "//pkg/lightning/backend/local", "//pkg/lightning/common", "//pkg/lightning/config", "//pkg/lightning/log", diff --git a/lightning/pkg/server/checkpoint_control.go b/lightning/pkg/server/checkpoint_control.go index 642a7d9c8d421..7aa22f3ff2c87 100644 --- a/lightning/pkg/server/checkpoint_control.go +++ b/lightning/pkg/server/checkpoint_control.go @@ -23,8 +23,8 @@ import ( "github.com/pingcap/tidb/lightning/pkg/checkpoints" "github.com/pingcap/tidb/lightning/pkg/importer" "github.com/pingcap/tidb/lightning/pkg/importinto" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/lightning/backend" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/config" "github.com/pingcap/tidb/pkg/lightning/log" @@ -128,7 +128,7 @@ func (c *LegacyCheckpointControl) DestroyError(ctx context.Context, tableName st for engineID := table.MinEngineID; engineID <= table.MaxEngineID; engineID++ { log.L().Info("Closing and cleaning up engine", zap.String("table", table.TableName), zap.Int32("engineID", engineID)) _, eID := backend.MakeUUID(table.TableName, int64(engineID)) - engine := local.Engine{UUID: eID} + engine := ingestctrl.Engine{UUID: eID} err := engine.Cleanup(c.cfg.TikvImporter.SortedKVDir) if err != nil { log.L().Error("Encountered error while cleanup engine", zap.Error(err)) diff --git a/lightning/pkg/server/lightning.go b/lightning/pkg/server/lightning.go index 5dd1d3b552985..27ecf37efabe9 100644 --- a/lightning/pkg/server/lightning.go +++ b/lightning/pkg/server/lightning.go @@ -51,7 +51,7 @@ import ( "github.com/pingcap/tidb/lightning/pkg/importinto" "github.com/pingcap/tidb/lightning/pkg/progress" _ "github.com/pingcap/tidb/pkg/expression" // get rid of `import cycle`: just init expression.RewriteAstExpr,and called at package `backend.kv`. - "github.com/pingcap/tidb/pkg/lightning/backend/local" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/config" "github.com/pingcap/tidb/pkg/lightning/log" @@ -1043,8 +1043,8 @@ func checkSystemRequirement(cfg *config.Config, dbsMeta []*mydump.MDDatabaseMeta maxDBFiles := topNTotalSize / int64(cfg.TikvImporter.LocalWriterMemCacheSize) * 2 // the pebble db and all import routine need upto maxDBFiles fds for read and write. maxOpenDBFiles := maxDBFiles * (1 + int64(cfg.TikvImporter.RangeConcurrency)) - estimateMaxFiles := local.RlimT(cfg.App.RegionConcurrency) + local.RlimT(maxOpenDBFiles) - if err := local.VerifyRLimit(estimateMaxFiles); err != nil { + estimateMaxFiles := ingestctrl.RlimT(cfg.App.RegionConcurrency) + ingestctrl.RlimT(maxOpenDBFiles) + if err := ingestctrl.VerifyRLimit(estimateMaxFiles); err != nil { return err } } diff --git a/lightning/pkg/server/lightning_serial_test.go b/lightning/pkg/server/lightning_serial_test.go index 8a912b8ec7ab0..9ba2747a1202d 100644 --- a/lightning/pkg/server/lightning_serial_test.go +++ b/lightning/pkg/server/lightning_serial_test.go @@ -164,24 +164,24 @@ func TestCheckSystemRequirement(t *testing.T) { }, } - err := failpoint.Enable("github.com/pingcap/tidb/pkg/lightning/backend/local/GetRlimitValue", "return(139415)") + err := failpoint.Enable("github.com/pingcap/tidb/pkg/ingestor/ingestctrl/GetRlimitValue", "return(139415)") require.NoError(t, err) - err = failpoint.Enable("github.com/pingcap/tidb/pkg/lightning/backend/local/SetRlimitError", "return(true)") + err = failpoint.Enable("github.com/pingcap/tidb/pkg/ingestor/ingestctrl/SetRlimitError", "return(true)") require.NoError(t, err) defer func() { - _ = failpoint.Disable("github.com/pingcap/tidb/pkg/lightning/backend/local/SetRlimitError") + _ = failpoint.Disable("github.com/pingcap/tidb/pkg/ingestor/ingestctrl/SetRlimitError") }() // with this dbMetas, the estimated fds will be 139416, so should return error err = checkSystemRequirement(cfg, dbMetas) require.Error(t, err) - err = failpoint.Disable("github.com/pingcap/tidb/pkg/lightning/backend/local/GetRlimitValue") + err = failpoint.Disable("github.com/pingcap/tidb/pkg/ingestor/ingestctrl/GetRlimitValue") require.NoError(t, err) // the min rlimit should be not smaller than the default min value (139416) - err = failpoint.Enable("github.com/pingcap/tidb/pkg/lightning/backend/local/GetRlimitValue", "return(139416)") + err = failpoint.Enable("github.com/pingcap/tidb/pkg/ingestor/ingestctrl/GetRlimitValue", "return(139416)") defer func() { - _ = failpoint.Disable("github.com/pingcap/tidb/pkg/lightning/backend/local/GetRlimitValue") + _ = failpoint.Disable("github.com/pingcap/tidb/pkg/ingestor/ingestctrl/GetRlimitValue") }() require.NoError(t, err) err = checkSystemRequirement(cfg, dbMetas) diff --git a/lightning/tests/lightning_checkpoint_chunks/run.sh b/lightning/tests/lightning_checkpoint_chunks/run.sh index d80cbdfab2d1d..5f7f19a8bfad7 100755 --- a/lightning/tests/lightning_checkpoint_chunks/run.sh +++ b/lightning/tests/lightning_checkpoint_chunks/run.sh @@ -50,7 +50,7 @@ for i in $(seq "$CHUNK_COUNT"); do done PKG="github.com/pingcap/tidb/lightning/pkg" -export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/lightning/backend/local/orphanWriterGoRoutine=return();$PKG/importer/orphanWriterGoRoutine=return();$PKG/server/orphanWriterGoRoutine=return()" +export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/ingestor/ingestctrl/orphanWriterGoRoutine=return();$PKG/importer/orphanWriterGoRoutine=return();$PKG/server/orphanWriterGoRoutine=return()" # test won't panic do_run_lightning config diff --git a/lightning/tests/lightning_concurrent-restore/run.sh b/lightning/tests/lightning_concurrent-restore/run.sh index ddb437478cbf0..bc8ed5fe17d3c 100644 --- a/lightning/tests/lightning_concurrent-restore/run.sh +++ b/lightning/tests/lightning_concurrent-restore/run.sh @@ -31,7 +31,7 @@ run_sql 'select VARIABLE_VALUE from mysql.tidb where VARIABLE_NAME = "tikv_gc_li ORIGINAL_TIKV_GC_LIFE_TIME=$(tail -n 1 "$TEST_DIR/sql_res.$TEST_NAME.txt" | awk '{print $(NF)}') # add a delay after increasing tikv_gc_life_time, in order to increase confilct possibility -export GO_FAILPOINTS='github.com/pingcap/tidb/pkg/lightning/backend/local/IncreaseGCUpdateDuration=sleep(200)' +export GO_FAILPOINTS='github.com/pingcap/tidb/pkg/ingestor/ingestctrl/IncreaseGCUpdateDuration=sleep(200)' # Start importing run_sql 'DROP DATABASE IF EXISTS restore_conc' diff --git a/lightning/tests/lightning_disable_scheduler_by_key_range/run.sh b/lightning/tests/lightning_disable_scheduler_by_key_range/run.sh index 29d418da9c052..f1c5d1b15cc87 100644 --- a/lightning/tests/lightning_disable_scheduler_by_key_range/run.sh +++ b/lightning/tests/lightning_disable_scheduler_by_key_range/run.sh @@ -17,7 +17,7 @@ set -eux export GO_FAILPOINTS="github.com/pingcap/tidb/lightning/pkg/server/EnableTestAPI=return" -export GO_FAILPOINTS="${GO_FAILPOINTS};github.com/pingcap/tidb/pkg/lightning/backend/local/ReadyForImportEngine=sleep(10000)" +export GO_FAILPOINTS="${GO_FAILPOINTS};github.com/pingcap/tidb/pkg/ingestor/ingestctrl/ReadyForImportEngine=sleep(10000)" run_lightning --backend='local' & shpid="$!" diff --git a/lightning/tests/lightning_fail_fast_on_nonretry_err/run.sh b/lightning/tests/lightning_fail_fast_on_nonretry_err/run.sh index 4c345c0eefe04..44df0c88a79a7 100755 --- a/lightning/tests/lightning_fail_fast_on_nonretry_err/run.sh +++ b/lightning/tests/lightning_fail_fast_on_nonretry_err/run.sh @@ -19,7 +19,7 @@ set -eu run_sql 'DROP DATABASE IF EXISTS csv' out_file_name="$TEST_DIR/sql_res.$TEST_NAME.txt" -export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/lightning/backend/local/WriteToTiKVNotEnoughDiskSpace=return(true)" +export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/ingestor/ingestctrl/WriteToTiKVNotEnoughDiskSpace=return(true)" start_time=$(date +'%s') run_lightning --backend local > $out_file_name 2>&1 || true export GO_FAILPOINTS="" diff --git a/lightning/tests/lightning_import_compress/run.sh b/lightning/tests/lightning_import_compress/run.sh index 1eef822a42632..24784156cd80d 100644 --- a/lightning/tests/lightning_import_compress/run.sh +++ b/lightning/tests/lightning_import_compress/run.sh @@ -18,7 +18,7 @@ set -eu CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/lightning/backend/local/LoggingImportBytes=return" +export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/ingestor/ingestctrl/LoggingImportBytes=return" mkdir -p "$TEST_DIR/data" diff --git a/lightning/tests/lightning_local_backend/run.sh b/lightning/tests/lightning_local_backend/run.sh index b213df957492c..b8706d81bafd6 100755 --- a/lightning/tests/lightning_local_backend/run.sh +++ b/lightning/tests/lightning_local_backend/run.sh @@ -37,7 +37,7 @@ grep -Fq 'table(s) [`cpeng`.`a`, `cpeng`.`b`] are not empty' $TEST_DIR/lightning # First, verify that inject with not leader error is fine. -export GO_FAILPOINTS='github.com/pingcap/tidb/pkg/lightning/backend/local/FailIngestMeta=1*return("notleader");github.com/pingcap/tidb/br/pkg/restore/split/failToSplit=5*return("");github.com/pingcap/tidb/pkg/lightning/backend/local/forceSplitRegion=5*return("")' +export GO_FAILPOINTS='github.com/pingcap/tidb/pkg/ingestor/ingestctrl/FailIngestMeta=1*return("notleader");github.com/pingcap/tidb/br/pkg/restore/split/failToSplit=5*return("");github.com/pingcap/tidb/pkg/ingestor/ingestctrl/forceSplitRegion=5*return("")' rm -f "$TEST_DIR/lightning-local.log" run_sql 'DROP DATABASE IF EXISTS cpeng;' run_lightning --backend local --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-local.log" --config "$CUR/config.toml" -L debug @@ -56,7 +56,7 @@ check_contains 'sum(c): 46' run_sql 'DROP DATABASE cpeng;' rm -f "/tmp/tidb_lightning_checkpoint_local_backend_test.pb" -export GO_FAILPOINTS='github.com/pingcap/tidb/pkg/lightning/backend/local/FailIngestMeta=2*return("epochnotmatch");github.com/pingcap/tidb/pkg/lightning/backend/local/NoLeader=1*return()' +export GO_FAILPOINTS='github.com/pingcap/tidb/pkg/ingestor/ingestctrl/FailIngestMeta=2*return("epochnotmatch");github.com/pingcap/tidb/pkg/ingestor/ingestctrl/NoLeader=1*return()' run_lightning --backend local --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-local.log" --config "$CUR/config.toml" diff --git a/lightning/tests/lightning_max_random/run.sh b/lightning/tests/lightning_max_random/run.sh index 9ffc06ef66c89..26beb7f16771c 100644 --- a/lightning/tests/lightning_max_random/run.sh +++ b/lightning/tests/lightning_max_random/run.sh @@ -44,7 +44,7 @@ cleanup() { cleanup -export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/lightning/backend/local/changeEpochVersion=1*return(-1)" +export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/ingestor/ingestctrl/changeEpochVersion=1*return(-1)" # auto_random_max = 2^{64-1-10}-1 # db.test contains key auto_random_max - 1 @@ -69,7 +69,7 @@ run_sql 'INSERT INTO db.test2(b) VALUES(55);' grep 'RequestTooOld' "$TEST_DIR/lightning.log" | grep -q 'needRescan' cleanup -export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/lightning/backend/local/changeEpochVersion=1*return(10)" +export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/ingestor/ingestctrl/changeEpochVersion=1*return(10)" # auto_random_max = 2^{64-1-10}-1 # db.test contains key auto_random_max - 1 diff --git a/lightning/tests/lightning_write_batch/run.sh b/lightning/tests/lightning_write_batch/run.sh index fe2f74262db18..008c1be5cd9b5 100644 --- a/lightning/tests/lightning_write_batch/run.sh +++ b/lightning/tests/lightning_write_batch/run.sh @@ -40,7 +40,7 @@ set -x # send-kv-pairs is deprecated, will not takes effect rm -rf $TEST_DIR/lightning.log -export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/lightning/backend/local/afterFlushKVs=return(true)" +export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/ingestor/ingestctrl/afterFlushKVs=return(true)" run_lightning --backend local -d "$TEST_DIR/data" --config "$CUR/kv-count.toml" check_contains 'afterFlushKVs count=100,' $TEST_DIR/lightning.log check_not_contains 'afterFlushKVs count=20,' $TEST_DIR/lightning.log @@ -59,7 +59,7 @@ done set -x rm -rf $TEST_DIR/lightning.log -export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/lightning/backend/local/afterFlushKVs=return(true)" +export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/ingestor/ingestctrl/afterFlushKVs=return(true)" run_lightning --backend local -d "$TEST_DIR/data" --config "$CUR/kv-size.toml" # each kv is 64b, so each kv is a batch check_contains 'afterFlushKVs count=1,' $TEST_DIR/lightning.log diff --git a/lightning/tests/lightning_write_timeout/run.sh b/lightning/tests/lightning_write_timeout/run.sh index 56bd3d728e0a3..69555bd0e7446 100644 --- a/lightning/tests/lightning_write_timeout/run.sh +++ b/lightning/tests/lightning_write_timeout/run.sh @@ -42,7 +42,7 @@ for i in {1..200000}; do done set -x -export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/lightning/backend/local/shortWaitNTimeout=5*return(1000)" +export GO_FAILPOINTS="github.com/pingcap/tidb/pkg/ingestor/ingestctrl/shortWaitNTimeout=5*return(1000)" run_lightning --backend local -d "$TEST_DIR/data" --config "$CUR/config.toml" check_lightning_log_contains 'experiencing a wait timeout while writing to TiKV' diff --git a/pkg/ddl/BUILD.bazel b/pkg/ddl/BUILD.bazel index 5e95cc1ab1e05..25e5d7681163e 100644 --- a/pkg/ddl/BUILD.bazel +++ b/pkg/ddl/BUILD.bazel @@ -113,10 +113,10 @@ go_library( "//pkg/infoschema/context", "//pkg/ingestor/engineapi", "//pkg/ingestor/globalsort", + "//pkg/ingestor/ingestctrl", "//pkg/keyspace", "//pkg/kv", "//pkg/lightning/backend", - "//pkg/lightning/backend/local", "//pkg/lightning/common", "//pkg/lightning/config", "//pkg/lightning/metric", diff --git a/pkg/ddl/backfilling.go b/pkg/ddl/backfilling.go index 825c0a813bbf5..a2d6aa02b0fb4 100644 --- a/pkg/ddl/backfilling.go +++ b/pkg/ddl/backfilling.go @@ -36,8 +36,8 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/expression/exprctx" "github.com/pingcap/tidb/pkg/expression/exprstatic" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/parser/ast" @@ -776,8 +776,8 @@ func (dc *ddlCtx) addIndexWithLocalIngest( } var ( - cfg *local.BackendConfig - bd *local.Backend + cfg *ingestctrl.BackendConfig + bd *ingestctrl.Backend err error ) if config.GetGlobalConfig().Store == config.StoreTypeTiKV { diff --git a/pkg/ddl/backfilling_dist_scheduler.go b/pkg/ddl/backfilling_dist_scheduler.go index d023da068254a..f950eebd84c50 100644 --- a/pkg/ddl/backfilling_dist_scheduler.go +++ b/pkg/ddl/backfilling_dist_scheduler.go @@ -36,8 +36,8 @@ import ( "github.com/pingcap/tidb/pkg/dxf/framework/scheduler" diststorage "github.com/pingcap/tidb/pkg/dxf/framework/storage" "github.com/pingcap/tidb/pkg/ingestor/globalsort" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/meta" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/objstore" @@ -763,7 +763,7 @@ func getRangeSplitter( pdCli := store.GetPDClient() tls, err := ingest.NewDDLTLS() if err == nil { - size, keys, err := local.GetRegionSplitSizeKeys(ctx, pdCli, tls) + size, keys, err := ingestctrl.GetRegionSplitSizeKeys(ctx, pdCli, tls) if err == nil { regionSplitSize = max(regionSplitSize, size) regionSplitKeys = max(regionSplitKeys, keys) diff --git a/pkg/ddl/backfilling_import_cloud.go b/pkg/ddl/backfilling_import_cloud.go index e0d5ef30b20f0..27e1a99148c7d 100644 --- a/pkg/ddl/backfilling_import_cloud.go +++ b/pkg/ddl/backfilling_import_cloud.go @@ -30,9 +30,9 @@ import ( "github.com/pingcap/tidb/pkg/dxf/framework/taskexecutor/execute" "github.com/pingcap/tidb/pkg/ingestor/engineapi" "github.com/pingcap/tidb/pkg/ingestor/globalsort" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/lightning/backend" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/config" lightningmetric "github.com/pingcap/tidb/pkg/lightning/metric" @@ -51,7 +51,7 @@ type cloudImportExecutor struct { ptbl table.PhysicalTable cloudStoreURI string backendCtx ingest.BackendCtx - backend *local.Backend + backend *ingestctrl.Backend metric *lightningmetric.Common engine atomic.Pointer[globalsort.Engine] summary *execute.SubtaskSummary diff --git a/pkg/ddl/backfilling_read_index.go b/pkg/ddl/backfilling_read_index.go index 24ce7e87042bb..e318a423aaf16 100644 --- a/pkg/ddl/backfilling_read_index.go +++ b/pkg/ddl/backfilling_read_index.go @@ -36,8 +36,8 @@ import ( "github.com/pingcap/tidb/pkg/dxf/framework/taskexecutor/execute" "github.com/pingcap/tidb/pkg/dxf/operator" "github.com/pingcap/tidb/pkg/ingestor/globalsort" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" lightningmetric "github.com/pingcap/tidb/pkg/lightning/metric" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/metrics" @@ -67,8 +67,8 @@ type readIndexStepExecutor struct { summary *execute.SubtaskSummary summaryMap sync.Map // subtaskID => readIndexSummary - backendCfg *local.BackendConfig - backend *local.Backend + backendCfg *ingestctrl.BackendConfig + backend *ingestctrl.Backend // pipeline of current running subtask, it's nil when no subtask is running. currPipe atomic.Pointer[operator.AsyncPipeline] diff --git a/pkg/ddl/index.go b/pkg/ddl/index.go index d030e41017b77..f3aec72121c00 100644 --- a/pkg/ddl/index.go +++ b/pkg/ddl/index.go @@ -51,10 +51,10 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/expression/exprstatic" "github.com/pingcap/tidb/pkg/infoschema" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/keyspace" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/lightning/backend" - "github.com/pingcap/tidb/pkg/lightning/backend/local" litconfig "github.com/pingcap/tidb/pkg/lightning/config" lightningmetric "github.com/pingcap/tidb/pkg/lightning/metric" "github.com/pingcap/tidb/pkg/meta" @@ -2959,8 +2959,8 @@ func (w *worker) addTableIndex( func checkDuplicateForUniqueIndex(ctx context.Context, t table.Table, reorgInfo *reorgInfo, store kv.Storage) (err error) { var ( backendCtx ingest.BackendCtx - cfg *local.BackendConfig - backend *local.Backend + cfg *ingestctrl.BackendConfig + backend *ingestctrl.Backend ) defer func() { if backendCtx != nil { diff --git a/pkg/ddl/ingest/BUILD.bazel b/pkg/ddl/ingest/BUILD.bazel index 0bba6f43954d1..756ab9baebdbd 100644 --- a/pkg/ddl/ingest/BUILD.bazel +++ b/pkg/ddl/ingest/BUILD.bazel @@ -24,11 +24,11 @@ go_library( "//pkg/ddl/logutil", "//pkg/ddl/session", "//pkg/ddl/util", + "//pkg/ingestor/ingestctrl", "//pkg/kv", "//pkg/lightning/backend", "//pkg/lightning/backend/encode", "//pkg/lightning/backend/kv", - "//pkg/lightning/backend/local", "//pkg/lightning/common", "//pkg/lightning/config", "//pkg/lightning/importdef", diff --git a/pkg/ddl/ingest/backend.go b/pkg/ddl/ingest/backend.go index c2a62bc622b8a..26f0d3ad2199f 100644 --- a/pkg/ddl/ingest/backend.go +++ b/pkg/ddl/ingest/backend.go @@ -24,10 +24,10 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" tikv "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/lightning/backend" "github.com/pingcap/tidb/pkg/lightning/backend/encode" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" lightning "github.com/pingcap/tidb/pkg/lightning/config" "github.com/pingcap/tidb/pkg/lightning/log" @@ -66,9 +66,9 @@ type BackendCtx interface { CheckpointOperator - // GetLocalBackend exposes local.Backend. It's only used in global sort based + // GetLocalBackend exposes ingestctrl.Backend. It's only used in global sort based // ingest. - GetLocalBackend() *local.Backend + GetLocalBackend() *ingestctrl.Backend // CollectRemoteDuplicateRows collects duplicate entry error for given index as // the supplement of Ingest. // @@ -100,9 +100,9 @@ type litBackendCtx struct { jobID int64 tbl table.Table // litBackendCtx doesn't manage the lifecycle of backend, caller should do it. - backend *local.Backend + backend *ingestctrl.Backend ctx context.Context - cfg *local.BackendConfig + cfg *ingestctrl.BackendConfig sysVars map[string]string flushing atomic.Bool @@ -341,13 +341,13 @@ func (bc *litBackendCtx) checkFlush() (shouldFlush bool, shouldImport bool) { } // GetLocalBackend returns the local backend. -func (bc *litBackendCtx) GetLocalBackend() *local.Backend { +func (bc *litBackendCtx) GetLocalBackend() *ingestctrl.Backend { return bc.backend } // GetDiskUsage returns current disk usage of underlying backend. func (bc *litBackendCtx) GetDiskUsage() uint64 { - _, _, bcDiskUsed, _ := local.CheckDiskQuota(bc.backend, math.MaxInt64) + _, _, bcDiskUsed, _ := ingestctrl.CheckDiskQuota(bc.backend, math.MaxInt64) return uint64(bcDiskUsed) } diff --git a/pkg/ddl/ingest/backend_mgr.go b/pkg/ddl/ingest/backend_mgr.go index e69ce2e926057..7a660a84ff6d2 100644 --- a/pkg/ddl/ingest/backend_mgr.go +++ b/pkg/ddl/ingest/backend_mgr.go @@ -26,8 +26,8 @@ import ( "github.com/pingcap/tidb/pkg/config" ddllogutil "github.com/pingcap/tidb/pkg/ddl/logutil" sess "github.com/pingcap/tidb/pkg/ddl/session" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/util/intest" @@ -115,7 +115,7 @@ func (b *BackendCtxBuilder) ForDuplicateCheck() *BackendCtxBuilder { var BackendCounterForTest = atomic.Int64{} // Build builds a BackendCtx. -func (b *BackendCtxBuilder) Build(cfg *local.BackendConfig, bd *local.Backend) (BackendCtx, error) { +func (b *BackendCtxBuilder) Build(cfg *ingestctrl.BackendConfig, bd *ingestctrl.Backend) (BackendCtx, error) { ctx, store, job := b.ctx, b.store, b.job jobSortPath, err := genJobSortPath(job.ID, b.checkDup) if err != nil { @@ -198,7 +198,7 @@ func genJobSortPath(jobID int64, checkDup bool) (string, error) { } // CreateLocalBackend creates a local backend for adding index. -func CreateLocalBackend(ctx context.Context, store kv.Storage, job *model.Job, hasUnique, checkDup bool, adjustedWorkerConcurrency int) (*local.BackendConfig, *local.Backend, error) { +func CreateLocalBackend(ctx context.Context, store kv.Storage, job *model.Job, hasUnique, checkDup bool, adjustedWorkerConcurrency int) (*ingestctrl.BackendConfig, *ingestctrl.Backend, error) { ctx = logutil.WithLogger(ctx, logutil.Logger(ctx)) jobSortPath, err := genJobSortPath(job.ID, checkDup) if err != nil { @@ -241,7 +241,7 @@ func CreateLocalBackend(ctx context.Context, store kv.Storage, job *model.Job, h //nolint: forcetypeassert pdCli := store.(kv.StorageWithPD).GetPDClient().(*tikv.CodecPDClient) - be, err := local.NewBackend(ctx, tls, *cfg, pdCli) + be, err := ingestctrl.NewBackend(ctx, tls, *cfg, pdCli) return cfg, be, err } @@ -250,8 +250,8 @@ const checkpointUpdateInterval = 10 * time.Minute func newBackendContext( ctx context.Context, jobID int64, - be *local.Backend, - cfg *local.BackendConfig, + be *ingestctrl.Backend, + cfg *ingestctrl.BackendConfig, vars map[string]string, memRoot MemRoot, etcdClient *clientv3.Client, diff --git a/pkg/ddl/ingest/config.go b/pkg/ddl/ingest/config.go index a765a631d20a2..a47b9072ef4b3 100644 --- a/pkg/ddl/ingest/config.go +++ b/pkg/ddl/ingest/config.go @@ -21,8 +21,8 @@ import ( "strconv" tidb "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/lightning/backend" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" lightning "github.com/pingcap/tidb/pkg/lightning/config" "github.com/pingcap/tidb/pkg/lightning/importdef" @@ -47,13 +47,13 @@ func genConfig( concurrency int, maxWriteSpeed int, globalSort bool, -) *local.BackendConfig { +) *ingestctrl.BackendConfig { workerConcurrency := int32(concurrency * 2) if ImporterRangeConcurrencyForTest != nil { workerConcurrency = ImporterRangeConcurrencyForTest.Load() * 2 } - cfg := &local.BackendConfig{ + cfg := &ingestctrl.BackendConfig{ LocalStoreDir: jobSortPath, ResourceGroupName: resourceGroup, MaxConnPerStore: concurrency, @@ -132,7 +132,7 @@ func generateLocalEngineConfig(ts uint64) *backend.EngineConfig { } // adjustImportMemory adjusts the lightning memory parameters according to the memory root's max limitation. -func adjustImportMemory(ctx context.Context, memRoot MemRoot, cfg *local.BackendConfig) { +func adjustImportMemory(ctx context.Context, memRoot MemRoot, cfg *ingestctrl.BackendConfig) { var scale int64 // Try aggressive resource usage successful. if tryAggressiveMemory(ctx, memRoot, cfg) { @@ -163,7 +163,7 @@ func adjustImportMemory(ctx context.Context, memRoot MemRoot, cfg *local.Backend } // tryAggressiveMemory lightning memory parameters according memory root's max limitation. -func tryAggressiveMemory(ctx context.Context, memRoot MemRoot, cfg *local.BackendConfig) bool { +func tryAggressiveMemory(ctx context.Context, memRoot MemRoot, cfg *ingestctrl.BackendConfig) bool { var defaultMemSize int64 defaultMemSize = int64(int(cfg.LocalWriterMemCacheSize) * cfg.GetWorkerConcurrency() / 2) defaultMemSize += int64(cfg.MemTableSize) diff --git a/pkg/ddl/ingest/mock.go b/pkg/ddl/ingest/mock.go index 9749d1401b1ef..73f1eb09b94c0 100644 --- a/pkg/ddl/ingest/mock.go +++ b/pkg/ddl/ingest/mock.go @@ -24,9 +24,9 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/ddl/logutil" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/lightning/backend" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessiontxn" @@ -153,8 +153,8 @@ func (m *MockBackendCtx) AdvanceWatermark(imported bool) error { } // GetLocalBackend returns the local backend. -func (m *MockBackendCtx) GetLocalBackend() *local.Backend { - b := &local.Backend{} +func (m *MockBackendCtx) GetLocalBackend() *ingestctrl.Backend { + b := &ingestctrl.Backend{} b.LocalStoreDir = filepath.Join(os.TempDir(), "mock_backend", strconv.FormatInt(m.jobID, 10)) return b } diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index a852f5457bcc1..94e5b4870bad0 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -48,9 +48,9 @@ go_library( "//pkg/infoschema/metrics", "//pkg/infoschema/perfschema", "//pkg/infoschema/validatorapi", + "//pkg/ingestor/ingestctrl", "//pkg/keyspace", "//pkg/kv", - "//pkg/lightning/backend/local", "//pkg/lightning/common", "//pkg/meta", "//pkg/meta/autoid", diff --git a/pkg/domain/domain.go b/pkg/domain/domain.go index a74af8ddbcc3a..07c6a0d732977 100644 --- a/pkg/domain/domain.go +++ b/pkg/domain/domain.go @@ -62,9 +62,9 @@ import ( infoschema_metrics "github.com/pingcap/tidb/pkg/infoschema/metrics" "github.com/pingcap/tidb/pkg/infoschema/perfschema" "github.com/pingcap/tidb/pkg/infoschema/validatorapi" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/keyspace" "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" lcom "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/meta" "github.com/pingcap/tidb/pkg/meta/autoid" @@ -1146,7 +1146,7 @@ func (do *Domain) InitDistTaskLoop() error { if err := kv.RunInNewTxn(ctx, do.store, true, func(_ context.Context, txn kv.Transaction) error { m := meta.NewMutator(txn) logger := logutil.BgLogger() - return local.InitializeRateLimiterParam(m, logger) + return ingestctrl.InitializeRateLimiterParam(m, logger) }); err != nil { logutil.BgLogger().Error("initialize global max batch split ranges failed", zap.Error(err)) } diff --git a/pkg/dxf/importinto/BUILD.bazel b/pkg/dxf/importinto/BUILD.bazel index 503600f791e96..ef932abc314cd 100644 --- a/pkg/dxf/importinto/BUILD.bazel +++ b/pkg/dxf/importinto/BUILD.bazel @@ -42,10 +42,10 @@ go_library( "//pkg/infoschema", "//pkg/ingestor/engineapi", "//pkg/ingestor/globalsort", + "//pkg/ingestor/ingestctrl", "//pkg/kv", "//pkg/lightning/backend", "//pkg/lightning/backend/kv", - "//pkg/lightning/backend/local", "//pkg/lightning/common", "//pkg/lightning/config", "//pkg/lightning/importdef", diff --git a/pkg/dxf/importinto/subtask_executor.go b/pkg/dxf/importinto/subtask_executor.go index 76db05617c293..215e261e5609a 100644 --- a/pkg/dxf/importinto/subtask_executor.go +++ b/pkg/dxf/importinto/subtask_executor.go @@ -22,9 +22,9 @@ import ( "github.com/pingcap/tidb/pkg/config/kerneltype" "github.com/pingcap/tidb/pkg/dxf/framework/taskexecutor/execute" "github.com/pingcap/tidb/pkg/executor/importer" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/lightning/backend" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/importdef" "github.com/pingcap/tidb/pkg/lightning/log" "github.com/pingcap/tidb/pkg/lightning/mydump" @@ -148,11 +148,11 @@ func (p *postProcessStepExecutor) postProcess(ctx context.Context, subtaskMeta * ctx = util.WithInternalSourceType(ctx, kv.InternalDistTask) if kerneltype.IsNextGen() { bfWeight := importer.GetBackoffWeight(plan) - mgr := local.NewTiKVChecksumManagerForImportInto(p.store, p.taskID, + mgr := ingestctrl.NewTiKVChecksumManagerForImportInto(p.store, p.taskID, uint(plan.DistSQLScanConcurrency), bfWeight, resourcegroup.DefaultResourceGroupName) defer mgr.Close() return importer.VerifyChecksum(ctx, plan, finalChecksum, logger, - func() (*local.RemoteChecksum, error) { + func() (*ingestctrl.RemoteChecksum, error) { ctxWithLogger := logutil.WithLogger(ctx, logger) return mgr.Checksum(ctxWithLogger, &importdef.TableInfo{ DB: plan.DBName, @@ -165,7 +165,7 @@ func (p *postProcessStepExecutor) postProcess(ctx context.Context, subtaskMeta * return p.taskTbl.WithNewSession(func(se sessionctx.Context) error { err = importer.VerifyChecksum(ctx, plan, finalChecksum, logger, - func() (*local.RemoteChecksum, error) { + func() (*ingestctrl.RemoteChecksum, error) { return importer.RemoteChecksumTableBySQL(ctx, se, plan, logger) }, ) diff --git a/pkg/executor/importer/BUILD.bazel b/pkg/executor/importer/BUILD.bazel index 9a23b31099cf0..6b67b6be9f90f 100644 --- a/pkg/executor/importer/BUILD.bazel +++ b/pkg/executor/importer/BUILD.bazel @@ -25,12 +25,12 @@ go_library( "//pkg/dxf/framework/taskexecutor/execute", "//pkg/expression", "//pkg/ingestor/globalsort", + "//pkg/ingestor/ingestctrl", "//pkg/keyspace", "//pkg/kv", "//pkg/lightning/backend", "//pkg/lightning/backend/encode", "//pkg/lightning/backend/kv", - "//pkg/lightning/backend/local", "//pkg/lightning/common", "//pkg/lightning/config", "//pkg/lightning/importdef", @@ -132,11 +132,11 @@ go_test( "//pkg/expression", "//pkg/infoschema", "//pkg/ingestor/globalsort", + "//pkg/ingestor/ingestctrl", "//pkg/keyspace", "//pkg/kv", "//pkg/lightning/backend/encode", "//pkg/lightning/backend/kv", - "//pkg/lightning/backend/local", "//pkg/lightning/common", "//pkg/lightning/config", "//pkg/lightning/log", diff --git a/pkg/executor/importer/import.go b/pkg/executor/importer/import.go index ab94b8b37fdf8..23227cc92b8f5 100644 --- a/pkg/executor/importer/import.go +++ b/pkg/executor/importer/import.go @@ -40,8 +40,8 @@ import ( "github.com/pingcap/tidb/pkg/dxf/framework/handle" "github.com/pingcap/tidb/pkg/dxf/framework/scheduler" "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" tidbkv "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/config" litlog "github.com/pingcap/tidb/pkg/lightning/log" @@ -1918,8 +1918,8 @@ func (e *LoadDataController) CreateColAssignSimpleExprs(ctx expression.BuildCont return createColAssignSimpleExprs(e.ColumnAssignments, ctx, &e.colAssignMu) } -func (e *LoadDataController) getLocalBackendCfg(keyspace, pdAddr, dataDir string) local.BackendConfig { - backendConfig := local.BackendConfig{ +func (e *LoadDataController) getLocalBackendCfg(keyspace, pdAddr, dataDir string) ingestctrl.BackendConfig { + backendConfig := ingestctrl.BackendConfig{ PDAddr: pdAddr, LocalStoreDir: dataDir, MaxConnPerStore: config.DefaultRangeConcurrency, diff --git a/pkg/executor/importer/importer_testkit_test.go b/pkg/executor/importer/importer_testkit_test.go index f2481a46b71f7..0e338a9472409 100644 --- a/pkg/executor/importer/importer_testkit_test.go +++ b/pkg/executor/importer/importer_testkit_test.go @@ -34,10 +34,10 @@ import ( "github.com/pingcap/tidb/pkg/dxf/framework/testutil" "github.com/pingcap/tidb/pkg/executor/importer" "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/lightning/backend/encode" backendkv "github.com/pingcap/tidb/pkg/lightning/backend/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/config" "github.com/pingcap/tidb/pkg/lightning/mydump" @@ -86,7 +86,7 @@ func TestVerifyChecksum(t *testing.T) { tk.MustExec("create table db.tb(id int)") tk.MustExec("insert into db.tb values(1)") - getRemoteChecksumFn := func() (*local.RemoteChecksum, error) { + getRemoteChecksumFn := func() (*ingestctrl.RemoteChecksum, error) { return importer.RemoteChecksumTableBySQL(ctx, tk.Session(), plan, logutil.BgLogger()) } @@ -133,7 +133,7 @@ func TestVerifyChecksum(t *testing.T) { ctx2, cancel := context.WithTimeout(ctx, time.Second) defer cancel() - err = importer.VerifyChecksum(ctx2, plan2, localChecksum, logutil.BgLogger(), func() (*local.RemoteChecksum, error) { + err = importer.VerifyChecksum(ctx2, plan2, localChecksum, logutil.BgLogger(), func() (*ingestctrl.RemoteChecksum, error) { return importer.RemoteChecksumTableBySQL(ctx2, tk.Session(), plan2, logutil.BgLogger()) }) require.ErrorContains(t, err, "Query execution was interrupted") diff --git a/pkg/executor/importer/table_import.go b/pkg/executor/importer/table_import.go index 0ebc0e21887f5..f302522ddf093 100644 --- a/pkg/executor/importer/table_import.go +++ b/pkg/executor/importer/table_import.go @@ -32,12 +32,12 @@ import ( "github.com/pingcap/failpoint" tidb "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/dxf/framework/proto" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/keyspace" tidbkv "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/lightning/backend" "github.com/pingcap/tidb/pkg/lightning/backend/encode" "github.com/pingcap/tidb/pkg/lightning/backend/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/config" "github.com/pingcap/tidb/pkg/lightning/importdef" @@ -71,7 +71,7 @@ import ( ) // NewTiKVModeSwitcher make it a var, so we can mock it in tests. -var NewTiKVModeSwitcher = local.NewTiKVModeSwitcher +var NewTiKVModeSwitcher = ingestctrl.NewTiKVModeSwitcher var ( // CheckDiskQuotaInterval is the default time interval to check disk quota. @@ -177,7 +177,7 @@ func GetRegionSplitSizeKeys(ctx context.Context) (regionSplitSize int64, regionS return 0, 0, errors.Trace(err) } defer pdCli.Close() - return local.GetRegionSplitSizeKeys(ctx, pdCli, tls) + return ingestctrl.GetRegionSplitSizeKeys(ctx, pdCli, tls) } // NewTableImporter creates a new table importer. @@ -214,7 +214,7 @@ func NewTableImporter( backendConfig := e.getLocalBackendCfg(kvStore.GetKeyspace(), tidbCfg.Path, dir) pdCli := kvStore.(tidbkv.StorageWithPD).GetPDClient().(*tikv.CodecPDClient) - localBackend, err := local.NewBackend(ctx, tls, backendConfig, pdCli) + localBackend, err := ingestctrl.NewBackend(ctx, tls, backendConfig, pdCli) if err != nil { return nil, err } @@ -250,7 +250,7 @@ type TableImporter struct { // it's the task id if we are running in distributed framework, else it's an // uuid. we use this id to create a unique directory for this importer. id string - backend *local.Backend + backend *ingestctrl.Backend kvStore tidbkv.Storage tableInfo *importdef.TableInfo // this table has a separate id allocator used to record the max row id allocated. @@ -279,7 +279,7 @@ func (s *storeHelper) GetTiKVCodec() tikv.Codec { return s.kvStore.GetCodec() } -var _ local.StoreHelper = (*storeHelper)(nil) +var _ ingestctrl.StoreHelper = (*storeHelper)(nil) // NewTableImporterForTest creates a new table importer for test. func NewTableImporterForTest(ctx context.Context, e *LoadDataController, id string, kvStore tidbkv.Storage) (*TableImporter, error) { @@ -297,7 +297,7 @@ func NewTableImporterForTest(ctx context.Context, e *LoadDataController, id stri } backendConfig := e.getLocalBackendCfg("", tidbCfg.Path, dir) - localBackend, err := local.NewBackendForTest(ctx, backendConfig, helper) + localBackend, err := ingestctrl.NewBackendForTest(ctx, backendConfig, helper) if err != nil { return nil, err } @@ -545,7 +545,7 @@ func (ti *TableImporter) OpenIndexEngine(ctx context.Context, engineID int32) (* } // todo: getTotalRawFileSize returns size of all data files, but in distributed framework, // we create one index engine for each engine, should reflect this in the future. - threshold := local.EstimateCompactionThreshold2(ti.getTotalRawFileSize(int64(idxCnt))) + threshold := ingestctrl.EstimateCompactionThreshold2(ti.getTotalRawFileSize(int64(idxCnt))) idxEngineCfg.Local = backend.LocalEngineConfig{ Compact: threshold > 0, CompactConcurrency: 4, @@ -570,7 +570,7 @@ func (ti *TableImporter) OpenDataEngine(ctx context.Context, engineID int32) (*b //if ti.tableMeta.IsRowOrdered { // dataEngineCfg.Local.Compact = true // dataEngineCfg.Local.CompactConcurrency = 4 - // dataEngineCfg.Local.CompactThreshold = local.CompactionUpperThreshold + // dataEngineCfg.Local.CompactThreshold = ingestctrl.CompactionUpperThreshold //} mgr := backend.MakeEngineManager(ti.backend) return mgr.OpenEngine(ctx, dataEngineCfg, ti.FullTableName(), engineID) @@ -582,7 +582,7 @@ func (ti *TableImporter) ImportAndCleanup(ctx context.Context, closedEngine *bac importErr := closedEngine.Import(ctx, ti.regionSplitSize, ti.regionSplitKeys) failpoint.InjectCall("mockDataEngineImportErr", &importErr) if common.ErrFoundDuplicateKeys.Equal(importErr) { - importErr = local.ConvertToErrFoundConflictRecords(importErr, ti.encTable) + importErr = ingestctrl.ConvertToErrFoundConflictRecords(importErr, ti.encTable) } if closedEngine.GetID() != common.IndexEngineID { // todo: change to a finer-grain progress later. @@ -594,7 +594,7 @@ func (ti *TableImporter) ImportAndCleanup(ctx context.Context, closedEngine *bac } // Backend returns the backend of the importer. -func (ti *TableImporter) Backend() *local.Backend { +func (ti *TableImporter) Backend() *ingestctrl.Backend { return ti.backend } @@ -637,7 +637,7 @@ func (ti *TableImporter) CheckDiskQuota(ctx context.Context) { case <-time.After(CheckDiskQuotaInterval): } - largeEngines, inProgressLargeEngines, totalDiskSize, totalMemSize := local.CheckDiskQuota(ti.backend, ti.diskQuota) + largeEngines, inProgressLargeEngines, totalDiskSize, totalMemSize := ingestctrl.CheckDiskQuota(ti.backend, ti.diskQuota) if len(largeEngines) == 0 && inProgressLargeEngines == 0 { unlockDiskQuota() continue @@ -676,7 +676,7 @@ func (ti *TableImporter) CheckDiskQuota(ctx context.Context) { int64(config.SplitRegionKeys)*int64(config.MaxSplitRegionSizeRatio), ); err != nil { if common.ErrFoundDuplicateKeys.Equal(err) { - err = local.ConvertToErrFoundConflictRecords(err, ti.encTable) + err = ingestctrl.ConvertToErrFoundConflictRecords(err, ti.encTable) } importErr = multierr.Append(importErr, err) } @@ -768,7 +768,7 @@ func (ti *TableImporter) ImportSelectedRows(ctx context.Context, se sessionctx.C }) if err = closedDataEngine.Import(ctx, ti.regionSplitSize, ti.regionSplitKeys); err != nil { if common.ErrFoundDuplicateKeys.Equal(err) { - err = local.ConvertToErrFoundConflictRecords(err, ti.encTable) + err = ingestctrl.ConvertToErrFoundConflictRecords(err, ti.encTable) } return 0, err } @@ -780,7 +780,7 @@ func (ti *TableImporter) ImportSelectedRows(ctx context.Context, se sessionctx.C } if err = closedIndexEngine.Import(ctx, ti.regionSplitSize, ti.regionSplitKeys); err != nil { if common.ErrFoundDuplicateKeys.Equal(err) { - err = local.ConvertToErrFoundConflictRecords(err, ti.encTable) + err = ingestctrl.ConvertToErrFoundConflictRecords(err, ti.encTable) } return 0, err } @@ -844,7 +844,7 @@ func PostProcess( } return VerifyChecksum(ctx, plan, localChecksum.MergedChecksum(), logger, - func() (*local.RemoteChecksum, error) { + func() (*ingestctrl.RemoteChecksum, error) { return RemoteChecksumTableBySQL(ctx, se, plan, logger) }, ) @@ -907,7 +907,7 @@ func RebaseAllocatorBases(ctx context.Context, kvStore tidbkv.Storage, maxIDs ma return errors.Trace(err) } -type remoteChecksumFunction func() (*local.RemoteChecksum, error) +type remoteChecksumFunction func() (*ingestctrl.RemoteChecksum, error) // VerifyChecksum verify the checksum of the table. func VerifyChecksum(ctx context.Context, plan *Plan, localChecksum verify.KVChecksum, logger *zap.Logger, getRemoteChecksumFn remoteChecksumFunction) error { @@ -950,13 +950,13 @@ func VerifyChecksum(ctx context.Context, plan *Plan, localChecksum verify.KVChec } // RemoteChecksumTableBySQL executes the SQL to get the remote checksum of the table. -func RemoteChecksumTableBySQL(ctx context.Context, se sessionctx.Context, plan *Plan, logger *zap.Logger) (*local.RemoteChecksum, error) { +func RemoteChecksumTableBySQL(ctx context.Context, se sessionctx.Context, plan *Plan, logger *zap.Logger) (*ingestctrl.RemoteChecksum, error) { var ( tableName = common.UniqueTable(plan.DBName, plan.TableInfo.Name.L) sql = "ADMIN CHECKSUM TABLE " + tableName maxErrorRetryCount = 3 distSQLScanConcurrencyFactor = 1 - remoteChecksum *local.RemoteChecksum + remoteChecksum *ingestctrl.RemoteChecksum txnErr error doneCh = make(chan struct{}) ) @@ -987,7 +987,7 @@ func RemoteChecksumTableBySQL(ctx context.Context, se sessionctx.Context, plan * logger.Warn("set tidb_backoff_weight failed", zap.Error(err)) } - newConcurrency := max(plan.DistSQLScanConcurrency/distSQLScanConcurrencyFactor, local.MinDistSQLScanConcurrency) + newConcurrency := max(plan.DistSQLScanConcurrency/distSQLScanConcurrencyFactor, ingestctrl.MinDistSQLScanConcurrency) logger.Info("checksum with adjusted distsql scan concurrency", zap.Int("concurrency", newConcurrency)) se.GetSessionVars().SetDistSQLScanConcurrency(newConcurrency) @@ -1012,7 +1012,7 @@ func RemoteChecksumTableBySQL(ctx context.Context, se sessionctx.Context, plan * // +---------+------------+---------------------+-----------+-------------+ // | test | t | 8520875019404689597 | 7296873 | 357601387 | // +---------+------------+------------- - remoteChecksum = &local.RemoteChecksum{ + remoteChecksum = &ingestctrl.RemoteChecksum{ Schema: rs[0].GetString(0), Table: rs[0].GetString(1), Checksum: rs[0].GetUint64(2), @@ -1031,9 +1031,9 @@ func RemoteChecksumTableBySQL(ctx context.Context, se sessionctx.Context, plan * } // GetBackoffWeight returns the backoff weight for the plan. -// returns max(local.DefaultBackoffWeight, plan.ImportantSysVars[vardef.TiDBBackOffWeight]) +// returns max(ingestctrl.DefaultBackoffWeight, plan.ImportantSysVars[vardef.TiDBBackOffWeight]) func GetBackoffWeight(plan *Plan) int { - backoffWeight := local.DefaultBackoffWeight + backoffWeight := ingestctrl.DefaultBackoffWeight if val, ok := plan.ImportantSysVars[vardef.TiDBBackOffWeight]; ok { if weight, err := strconv.Atoi(val); err == nil && weight > backoffWeight { backoffWeight = weight diff --git a/pkg/lightning/backend/local/BUILD.bazel b/pkg/ingestor/ingestctrl/BUILD.bazel similarity index 98% rename from pkg/lightning/backend/local/BUILD.bazel rename to pkg/ingestor/ingestctrl/BUILD.bazel index 676cc77d8c2d3..d71bd2d2dfe5e 100644 --- a/pkg/lightning/backend/local/BUILD.bazel +++ b/pkg/ingestor/ingestctrl/BUILD.bazel @@ -1,7 +1,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( - name = "local", + name = "ingestctrl", srcs = [ "checksum.go", "compress.go", @@ -22,7 +22,7 @@ go_library( "region_job.go", "tikv_mode.go", ], - importpath = "github.com/pingcap/tidb/pkg/lightning/backend/local", + importpath = "github.com/pingcap/tidb/pkg/ingestor/ingestctrl", visibility = ["//visibility:public"], deps = [ "//br/pkg/checksum", @@ -118,7 +118,7 @@ go_library( ) go_test( - name = "local_test", + name = "ingestctrl_test", timeout = "short", srcs = [ "checksum_test.go", @@ -136,7 +136,7 @@ go_test( "rate_limiter_test.go", "region_job_test.go", ], - embed = [":local"], + embed = [":ingestctrl"], flaky = True, race = "on", shard_count = 50, diff --git a/pkg/lightning/backend/local/checksum.go b/pkg/ingestor/ingestctrl/checksum.go similarity index 99% rename from pkg/lightning/backend/local/checksum.go rename to pkg/ingestor/ingestctrl/checksum.go index 4eb4326a0bcf1..9405100506a51 100644 --- a/pkg/lightning/backend/local/checksum.go +++ b/pkg/ingestor/ingestctrl/checksum.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "container/heap" diff --git a/pkg/lightning/backend/local/checksum_test.go b/pkg/ingestor/ingestctrl/checksum_test.go similarity index 99% rename from pkg/lightning/backend/local/checksum_test.go rename to pkg/ingestor/ingestctrl/checksum_test.go index e3bc5946ad1dc..84b844a453f96 100644 --- a/pkg/lightning/backend/local/checksum_test.go +++ b/pkg/ingestor/ingestctrl/checksum_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "context" diff --git a/pkg/lightning/backend/local/compress.go b/pkg/ingestor/ingestctrl/compress.go similarity index 98% rename from pkg/lightning/backend/local/compress.go rename to pkg/ingestor/ingestctrl/compress.go index 0ee75f7fc7874..dedd9c5a3772c 100644 --- a/pkg/lightning/backend/local/compress.go +++ b/pkg/ingestor/ingestctrl/compress.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "io" diff --git a/pkg/lightning/backend/local/compress_test.go b/pkg/ingestor/ingestctrl/compress_test.go similarity index 99% rename from pkg/lightning/backend/local/compress_test.go rename to pkg/ingestor/ingestctrl/compress_test.go index 4ee55afb3605d..1a9cf768df31a 100644 --- a/pkg/lightning/backend/local/compress_test.go +++ b/pkg/ingestor/ingestctrl/compress_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "bytes" diff --git a/pkg/lightning/backend/local/disk_quota.go b/pkg/ingestor/ingestctrl/disk_quota.go similarity index 99% rename from pkg/lightning/backend/local/disk_quota.go rename to pkg/ingestor/ingestctrl/disk_quota.go index 04e778e46bdcf..b4ba5328fabc5 100644 --- a/pkg/lightning/backend/local/disk_quota.go +++ b/pkg/ingestor/ingestctrl/disk_quota.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "cmp" diff --git a/pkg/lightning/backend/local/disk_quota_test.go b/pkg/ingestor/ingestctrl/disk_quota_test.go similarity index 99% rename from pkg/lightning/backend/local/disk_quota_test.go rename to pkg/ingestor/ingestctrl/disk_quota_test.go index b627288040d98..670474b14f006 100644 --- a/pkg/lightning/backend/local/disk_quota_test.go +++ b/pkg/ingestor/ingestctrl/disk_quota_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "testing" diff --git a/pkg/lightning/backend/local/duplicate.go b/pkg/ingestor/ingestctrl/duplicate.go similarity index 99% rename from pkg/lightning/backend/local/duplicate.go rename to pkg/ingestor/ingestctrl/duplicate.go index 3f0f952923429..67dc54e7b2f06 100644 --- a/pkg/lightning/backend/local/duplicate.go +++ b/pkg/ingestor/ingestctrl/duplicate.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "bytes" diff --git a/pkg/lightning/backend/local/duplicate_test.go b/pkg/ingestor/ingestctrl/duplicate_test.go similarity index 92% rename from pkg/lightning/backend/local/duplicate_test.go rename to pkg/ingestor/ingestctrl/duplicate_test.go index 7ddf05929518a..373bf819706be 100644 --- a/pkg/lightning/backend/local/duplicate_test.go +++ b/pkg/ingestor/ingestctrl/duplicate_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local_test +package ingestctrl_test import ( "bytes" @@ -21,10 +21,10 @@ import ( "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/tidb/pkg/ddl" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/keyspace" "github.com/pingcap/tidb/pkg/lightning/backend/encode" lkv "github.com/pingcap/tidb/pkg/lightning/backend/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/log" "github.com/pingcap/tidb/pkg/meta/model" @@ -62,7 +62,7 @@ func TestBuildDupTask(t *testing.T) { {&encode.SessionOptions{IndexID: info.Indices[1].ID}, false, []int64{info.Indices[1].ID}}, } for _, tc := range testCases { - dupMgr, err := local.NewDupeDetector( + dupMgr, err := ingestctrl.NewDupeDetector( tbl, "t", nil, @@ -76,7 +76,7 @@ func TestBuildDupTask(t *testing.T) { "lightning", ) require.NoError(t, err) - tasks, err := local.BuildDuplicateTaskForTest(dupMgr) + tasks, err := ingestctrl.BuildDuplicateTaskForTest(dupMgr) require.NoError(t, err) var hasRecordKey bool var gotIndexIDs []int64 @@ -111,7 +111,7 @@ func TestBuildDupTask(t *testing.T) { require.Equal(t, "idx2", tbl.Meta().Indices[2].Name.L) expectedIndexIDs := []int64{info.Indices[0].ID, info.Indices[2].ID} - dupMgr, err := local.NewDupeDetector( + dupMgr, err := ingestctrl.NewDupeDetector( tbl, "t", nil, @@ -125,7 +125,7 @@ func TestBuildDupTask(t *testing.T) { "lightning", ) require.NoError(t, err) - tasks, err := local.BuildDuplicateTaskForTest(dupMgr) + tasks, err := ingestctrl.BuildDuplicateTaskForTest(dupMgr) require.NoError(t, err) var hasRecordKey bool @@ -161,7 +161,7 @@ func TestBuildIndexDupTaskEncodesKeyspaceRange(t *testing.T) { require.NoError(t, err) indexInfo := info.Indices[0] - dupMgr, err := local.NewDupeDetector( + dupMgr, err := ingestctrl.NewDupeDetector( tbl, "t", nil, @@ -175,7 +175,7 @@ func TestBuildIndexDupTaskEncodesKeyspaceRange(t *testing.T) { "lightning", ) require.NoError(t, err) - tasks, err := local.BuildDuplicateTaskForTest(dupMgr) + tasks, err := ingestctrl.BuildDuplicateTaskForTest(dupMgr) require.NoError(t, err) require.Len(t, tasks, 1) @@ -250,7 +250,7 @@ func TestRetrieveKeyAndValueFromErrFoundDuplicateKeys(t *testing.T) { data1RowValue := kvPairs.Pairs[0].Val originalErr := common.ErrFoundDuplicateKeys.FastGenByArgs(data1RowKey, data1RowValue) - rawKey, rawValue, err := local.RetrieveKeyAndValueFromErrFoundDuplicateKeys(originalErr) + rawKey, rawValue, err := ingestctrl.RetrieveKeyAndValueFromErrFoundDuplicateKeys(originalErr) require.NoError(t, err) require.Equal(t, data1RowKey, rawKey) require.Equal(t, data1RowValue, rawValue) @@ -270,12 +270,12 @@ func TestConvertToErrFoundConflictRecordsSingleColumnsIndex(t *testing.T) { originalErr := common.ErrFoundDuplicateKeys.FastGenByArgs(data2RowKey, data2RowValue) - newErr := local.ConvertToErrFoundConflictRecords(originalErr, tbl) + newErr := ingestctrl.ConvertToErrFoundConflictRecords(originalErr, tbl) require.EqualError(t, newErr, "[Lightning:Restore:ErrFoundDataConflictRecords]found data conflict records in table a, primary key is '2', row data is '(2, 6, \"2.csv\", 102)'") originalErr = common.ErrFoundDuplicateKeys.FastGenByArgs(data3IndexKey, data3IndexValue) - newErr = local.ConvertToErrFoundConflictRecords(originalErr, tbl) + newErr = ingestctrl.ConvertToErrFoundConflictRecords(originalErr, tbl) require.EqualError(t, newErr, "[Lightning:Restore:ErrFoundIndexConflictRecords]found index conflict records in table a, index name is 'a.key_b', unique key is '[7]', primary key is '3'") } @@ -293,11 +293,11 @@ func TestConvertToErrFoundConflictRecordsMultipleColumnsIndex(t *testing.T) { originalErr := common.ErrFoundDuplicateKeys.FastGenByArgs(data2RowKey, data2RowValue) - newErr := local.ConvertToErrFoundConflictRecords(originalErr, tbl) + newErr := ingestctrl.ConvertToErrFoundConflictRecords(originalErr, tbl) require.EqualError(t, newErr, "[Lightning:Restore:ErrFoundDataConflictRecords]found data conflict records in table a, primary key is '2', row data is '(2, 6, \"2.csv\", 102)'") originalErr = common.ErrFoundDuplicateKeys.FastGenByArgs(data3IndexKey, data3IndexValue) - newErr = local.ConvertToErrFoundConflictRecords(originalErr, tbl) + newErr = ingestctrl.ConvertToErrFoundConflictRecords(originalErr, tbl) require.EqualError(t, newErr, "[Lightning:Restore:ErrFoundIndexConflictRecords]found index conflict records in table a, index name is 'a.key_bd', unique key is '[7 103]', primary key is '3'") } diff --git a/pkg/lightning/backend/local/engine.go b/pkg/ingestor/ingestctrl/engine.go similarity index 99% rename from pkg/lightning/backend/local/engine.go rename to pkg/ingestor/ingestctrl/engine.go index 3bc037412c172..ea4f0fb0e7001 100644 --- a/pkg/lightning/backend/local/engine.go +++ b/pkg/ingestor/ingestctrl/engine.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "bytes" diff --git a/pkg/lightning/backend/local/engine_mgr.go b/pkg/ingestor/ingestctrl/engine_mgr.go similarity index 99% rename from pkg/lightning/backend/local/engine_mgr.go rename to pkg/ingestor/ingestctrl/engine_mgr.go index f1dbb9a4a0c7c..32ca5087d9b71 100644 --- a/pkg/lightning/backend/local/engine_mgr.go +++ b/pkg/ingestor/ingestctrl/engine_mgr.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "context" diff --git a/pkg/lightning/backend/local/engine_mgr_test.go b/pkg/ingestor/ingestctrl/engine_mgr_test.go similarity index 99% rename from pkg/lightning/backend/local/engine_mgr_test.go rename to pkg/ingestor/ingestctrl/engine_mgr_test.go index 216d1d21eb0aa..55c78107c05ee 100644 --- a/pkg/lightning/backend/local/engine_mgr_test.go +++ b/pkg/ingestor/ingestctrl/engine_mgr_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "context" diff --git a/pkg/lightning/backend/local/engine_test.go b/pkg/ingestor/ingestctrl/engine_test.go similarity index 99% rename from pkg/lightning/backend/local/engine_test.go rename to pkg/ingestor/ingestctrl/engine_test.go index 2060ad53eaf97..32dceb4e797c5 100644 --- a/pkg/lightning/backend/local/engine_test.go +++ b/pkg/ingestor/ingestctrl/engine_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "context" diff --git a/pkg/lightning/backend/local/iterator.go b/pkg/ingestor/ingestctrl/iterator.go similarity index 99% rename from pkg/lightning/backend/local/iterator.go rename to pkg/ingestor/ingestctrl/iterator.go index a957bed08b6d7..1fa8e150b6c5d 100644 --- a/pkg/lightning/backend/local/iterator.go +++ b/pkg/ingestor/ingestctrl/iterator.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "github.com/cockroachdb/pebble" diff --git a/pkg/lightning/backend/local/iterator_test.go b/pkg/ingestor/ingestctrl/iterator_test.go similarity index 99% rename from pkg/lightning/backend/local/iterator_test.go rename to pkg/ingestor/ingestctrl/iterator_test.go index 00878dba1ca23..09e57422fcf01 100644 --- a/pkg/lightning/backend/local/iterator_test.go +++ b/pkg/ingestor/ingestctrl/iterator_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "bytes" diff --git a/pkg/lightning/backend/local/job_worker.go b/pkg/ingestor/ingestctrl/job_worker.go similarity index 99% rename from pkg/lightning/backend/local/job_worker.go rename to pkg/ingestor/ingestctrl/job_worker.go index 1b25153a89317..5f9d657b597a4 100644 --- a/pkg/lightning/backend/local/job_worker.go +++ b/pkg/ingestor/ingestctrl/job_worker.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "context" diff --git a/pkg/lightning/backend/local/job_worker_test.go b/pkg/ingestor/ingestctrl/job_worker_test.go similarity index 99% rename from pkg/lightning/backend/local/job_worker_test.go rename to pkg/ingestor/ingestctrl/job_worker_test.go index 43851dc26ea5b..bf71623b63bb2 100644 --- a/pkg/lightning/backend/local/job_worker_test.go +++ b/pkg/ingestor/ingestctrl/job_worker_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "context" @@ -118,7 +118,7 @@ func TestRegionJobBaseWorker(t *testing.T) { ) { // mock jobWg.Done() called in the worker testfailpoint.Enable(t, - "github.com/pingcap/tidb/pkg/lightning/backend/local/mockJobWgDone", + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/mockJobWgDone", fmt.Sprintf("return(%d)", generateCount)) workGroup, workerCtx := util.NewErrorGroupWithRecoverWithCtx(context.Background()) pool, jobWg, jobInCh, jobOutCh := newRegionJobWorkerPoolForTest(workerCtx, preRunFn, writeFn, ingestFn) diff --git a/pkg/lightning/backend/local/local.go b/pkg/ingestor/ingestctrl/local.go similarity index 99% rename from pkg/lightning/backend/local/local.go rename to pkg/ingestor/ingestctrl/local.go index 36e397e4e56a7..6d105bbc4970e 100644 --- a/pkg/lightning/backend/local/local.go +++ b/pkg/ingestor/ingestctrl/local.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "bytes" diff --git a/pkg/lightning/backend/local/local_check_test.go b/pkg/ingestor/ingestctrl/local_check_test.go similarity index 83% rename from pkg/lightning/backend/local/local_check_test.go rename to pkg/ingestor/ingestctrl/local_check_test.go index 933a366f697a2..6b5ad7fbe9304 100644 --- a/pkg/lightning/backend/local/local_check_test.go +++ b/pkg/ingestor/ingestctrl/local_check_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local_test +package ingestctrl_test import ( "context" @@ -24,8 +24,8 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb/br/pkg/restore/split" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/lightning/backend" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/lightning/mydump" "github.com/stretchr/testify/require" @@ -73,13 +73,13 @@ func TestCheckRequirementsTiFlash(t *testing.T) { } checkCtx := &backend.CheckCtx{DBMetas: dbMetas} - mock.ExpectQuery(local.TiFlashReplicaQueryForTest).WillReturnRows(sqlmock.NewRows([]string{"db", "tbl"}). + mock.ExpectQuery(ingestctrl.TiFlashReplicaQueryForTest).WillReturnRows(sqlmock.NewRows([]string{"db", "tbl"}). AddRow("db", "tbl"). AddRow("test", "t1"). AddRow("test1", "tbl")) mock.ExpectClose() - err = local.CheckTiFlashVersionForTest(ctx, db, checkCtx, *semver.New("4.0.2")) + err = ingestctrl.CheckTiFlashVersionForTest(ctx, db, checkCtx, *semver.New("4.0.2")) require.Regexp(t, "^lightning local backend doesn't support TiFlash in this TiDB version. conflict tables: \\[`test`.`t1`, `test1`.`tbl`\\]", err.Error()) } @@ -102,15 +102,15 @@ func TestGetRegionSplitSizeKeys(t *testing.T) { defer cancel() cli := split.NewFakePDClient(allStores, false, nil) defer func() { - local.SetGetSplitConfFromStoreFunc(local.GetSplitConfFromStore) + ingestctrl.SetGetSplitConfFromStoreFunc(ingestctrl.GetSplitConfFromStore) }() - local.SetGetSplitConfFromStoreFunc(func(ctx context.Context, host string, tls *common.TLS) (int64, int64, error) { + ingestctrl.SetGetSplitConfFromStoreFunc(func(ctx context.Context, host string, tls *common.TLS) (int64, int64, error) { if strings.Contains(host, "172.16.102.3:20180") { return int64(1), int64(2), nil } return 0, 0, errors.New("invalid connection") }) - splitSize, splitKeys, err := local.GetRegionSplitSizeKeys(ctx, cli, nil) + splitSize, splitKeys, err := ingestctrl.GetRegionSplitSizeKeys(ctx, cli, nil) require.NoError(t, err) require.Equal(t, int64(1), splitSize) require.Equal(t, int64(2), splitKeys) diff --git a/pkg/lightning/backend/local/local_freebsd.go b/pkg/ingestor/ingestctrl/local_freebsd.go similarity index 97% rename from pkg/lightning/backend/local/local_freebsd.go rename to pkg/ingestor/ingestctrl/local_freebsd.go index ea73142fb0205..f238414b42294 100644 --- a/pkg/lightning/backend/local/local_freebsd.go +++ b/pkg/ingestor/ingestctrl/local_freebsd.go @@ -14,7 +14,7 @@ //go:build freebsd -package local +package ingestctrl import ( "go.uber.org/zap" diff --git a/pkg/lightning/backend/local/local_test.go b/pkg/ingestor/ingestctrl/local_test.go similarity index 97% rename from pkg/lightning/backend/local/local_test.go rename to pkg/ingestor/ingestctrl/local_test.go index 32bbc4cb8fc4b..16d31aaaa36a3 100644 --- a/pkg/lightning/backend/local/local_test.go +++ b/pkg/ingestor/ingestctrl/local_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "bytes" @@ -1867,8 +1867,8 @@ func TestDoImport(t *testing.T) { maxRetryBackoffSecond = backup }) - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/skipSplitAndScatter", "return()") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/fakeRegionJobs", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/skipSplitAndScatter", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/fakeRegionJobs", "return()") // test that // - one job need rescan when ingest @@ -2119,8 +2119,8 @@ func TestRegionJobResetRetryCounter(t *testing.T) { maxRetryBackoffSecond = backup }) - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/skipSplitAndScatter", "return()") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/fakeRegionJobs", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/skipSplitAndScatter", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/fakeRegionJobs", "return()") // test that job need rescan when ingest @@ -2212,10 +2212,10 @@ func TestCtxCancelIsIgnored(t *testing.T) { maxRetryBackoffSecond = backup }) - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/skipSplitAndScatter", "return()") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/fakeRegionJobs", "return()") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/beforeGenerateJob", "sleep(1000)") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/WriteToTiKVNotEnoughDiskSpace", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/skipSplitAndScatter", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/fakeRegionJobs", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/beforeGenerateJob", "sleep(1000)") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/WriteToTiKVNotEnoughDiskSpace", "return()") initRegionKeys := [][]byte{{'c'}, {'d'}, {'e'}} fakeRegionJobs = map[[2]string]struct { @@ -2265,10 +2265,10 @@ func TestWorkerFailedWhenGeneratingJobs(t *testing.T) { maxRetryBackoffSecond = backup }) - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/skipSplitAndScatter", "return()") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/sendDummyJob", "return()") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/mockGetFirstAndLastKey", "return()") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/WriteToTiKVNotEnoughDiskSpace", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/skipSplitAndScatter", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/sendDummyJob", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/mockGetFirstAndLastKey", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/WriteToTiKVNotEnoughDiskSpace", "return()") initRegionKeys := [][]byte{{'c'}, {'d'}} @@ -2309,11 +2309,11 @@ func (r *recordScanRegionsHook) AfterScanRegions(infos []*split.RegionInfo, err } func TestExternalEngine(t *testing.T) { - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/skipSplitAndScatter", "return()") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/skipStartWorker", "return()") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/injectVariables", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/skipSplitAndScatter", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/skipStartWorker", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/injectVariables", "return()") testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/globalsort/LoadIngestDataBatchSize", "return(2)") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/skipOnDuplicateKeyCheck", "return(true)") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/skipOnDuplicateKeyCheck", "return(true)") ctx := context.Background() dir := t.TempDir() storageURI := "file://" + filepath.ToSlash(dir) @@ -2691,10 +2691,10 @@ func (r *refCountIngestData) IsCleaned() bool { // This test ensures that even if some empty jobs finish quickly, the ingestData won't be // cleaned unexpectedly before other jobs can access it. func TestRefAllJobsBeforeSending(t *testing.T) { - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/skipSplitAndScatter", "return()") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/skipStartWorker", "return()") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/injectVariables", "return()") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/fakeRegionJobs", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/skipSplitAndScatter", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/skipStartWorker", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/injectVariables", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/fakeRegionJobs", "return()") ctx := context.Background() local := &Backend{ diff --git a/pkg/lightning/backend/local/local_unix.go b/pkg/ingestor/ingestctrl/local_unix.go similarity index 99% rename from pkg/lightning/backend/local/local_unix.go rename to pkg/ingestor/ingestctrl/local_unix.go index 20695c6ebd6ab..da876e60245e9 100644 --- a/pkg/lightning/backend/local/local_unix.go +++ b/pkg/ingestor/ingestctrl/local_unix.go @@ -14,7 +14,7 @@ //go:build !windows -package local +package ingestctrl import ( "syscall" diff --git a/pkg/lightning/backend/local/local_unix_generic.go b/pkg/ingestor/ingestctrl/local_unix_generic.go similarity index 97% rename from pkg/lightning/backend/local/local_unix_generic.go rename to pkg/ingestor/ingestctrl/local_unix_generic.go index 2a465c4738c5e..abcf58d524ee7 100644 --- a/pkg/lightning/backend/local/local_unix_generic.go +++ b/pkg/ingestor/ingestctrl/local_unix_generic.go @@ -14,7 +14,7 @@ //go:build !freebsd && !windows -package local +package ingestctrl import "go.uber.org/zap" diff --git a/pkg/lightning/backend/local/local_windows.go b/pkg/ingestor/ingestctrl/local_windows.go similarity index 98% rename from pkg/lightning/backend/local/local_windows.go rename to pkg/ingestor/ingestctrl/local_windows.go index d42ca80607888..ef38e35c3ac4d 100644 --- a/pkg/lightning/backend/local/local_windows.go +++ b/pkg/ingestor/ingestctrl/local_windows.go @@ -14,7 +14,7 @@ //go:build windows -package local +package ingestctrl import ( "math" diff --git a/pkg/lightning/backend/local/localhelper.go b/pkg/ingestor/ingestctrl/localhelper.go similarity index 99% rename from pkg/lightning/backend/local/localhelper.go rename to pkg/ingestor/ingestctrl/localhelper.go index e92299ee09996..7791bd5c049b9 100644 --- a/pkg/lightning/backend/local/localhelper.go +++ b/pkg/ingestor/ingestctrl/localhelper.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "bytes" diff --git a/pkg/lightning/backend/local/localhelper_test.go b/pkg/ingestor/ingestctrl/localhelper_test.go similarity index 99% rename from pkg/lightning/backend/local/localhelper_test.go rename to pkg/ingestor/ingestctrl/localhelper_test.go index 89e044db46b57..f61bccd086f97 100644 --- a/pkg/lightning/backend/local/localhelper_test.go +++ b/pkg/ingestor/ingestctrl/localhelper_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "bytes" diff --git a/pkg/lightning/backend/local/main_test.go b/pkg/ingestor/ingestctrl/main_test.go similarity index 98% rename from pkg/lightning/backend/local/main_test.go rename to pkg/ingestor/ingestctrl/main_test.go index 6ee794c16a23b..d0892d6fb343d 100644 --- a/pkg/lightning/backend/local/main_test.go +++ b/pkg/ingestor/ingestctrl/main_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "testing" diff --git a/pkg/lightning/backend/local/rate_limiter.go b/pkg/ingestor/ingestctrl/rate_limiter.go similarity index 99% rename from pkg/lightning/backend/local/rate_limiter.go rename to pkg/ingestor/ingestctrl/rate_limiter.go index 6d14e662530f7..be8fdb5a969ba 100644 --- a/pkg/lightning/backend/local/rate_limiter.go +++ b/pkg/ingestor/ingestctrl/rate_limiter.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "context" diff --git a/pkg/lightning/backend/local/rate_limiter_param.go b/pkg/ingestor/ingestctrl/rate_limiter_param.go similarity index 99% rename from pkg/lightning/backend/local/rate_limiter_param.go rename to pkg/ingestor/ingestctrl/rate_limiter_param.go index ad0390f8b3f5e..7db8b9b770129 100644 --- a/pkg/lightning/backend/local/rate_limiter_param.go +++ b/pkg/ingestor/ingestctrl/rate_limiter_param.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "github.com/pingcap/errors" diff --git a/pkg/lightning/backend/local/rate_limiter_test.go b/pkg/ingestor/ingestctrl/rate_limiter_test.go similarity index 99% rename from pkg/lightning/backend/local/rate_limiter_test.go rename to pkg/ingestor/ingestctrl/rate_limiter_test.go index d73631226cf02..287f1bde6c906 100644 --- a/pkg/lightning/backend/local/rate_limiter_test.go +++ b/pkg/ingestor/ingestctrl/rate_limiter_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "context" diff --git a/pkg/lightning/backend/local/region_job.go b/pkg/ingestor/ingestctrl/region_job.go similarity index 99% rename from pkg/lightning/backend/local/region_job.go rename to pkg/ingestor/ingestctrl/region_job.go index ac9a48888aa22..ef3d6895d54d1 100644 --- a/pkg/lightning/backend/local/region_job.go +++ b/pkg/ingestor/ingestctrl/region_job.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "bytes" diff --git a/pkg/lightning/backend/local/region_job_test.go b/pkg/ingestor/ingestctrl/region_job_test.go similarity index 98% rename from pkg/lightning/backend/local/region_job_test.go rename to pkg/ingestor/ingestctrl/region_job_test.go index 29106f0005c62..dd2e2ce416e39 100644 --- a/pkg/lightning/backend/local/region_job_test.go +++ b/pkg/ingestor/ingestctrl/region_job_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "context" @@ -755,7 +755,7 @@ func TestWorkerPoolWithErrors(t *testing.T) { tests := []testCase{ { - fp: "github.com/pingcap/tidb/pkg/lightning/backend/local/mockRunJobSucceed", + fp: "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/mockRunJobSucceed", expr: "return", mockGeneratorErr: false, mockDrainerErr: false, @@ -763,7 +763,7 @@ func TestWorkerPoolWithErrors(t *testing.T) { opErr: "", }, { - fp: "github.com/pingcap/tidb/pkg/lightning/backend/local/mockRunJobSucceed", + fp: "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/mockRunJobSucceed", expr: "return", mockGeneratorErr: false, mockDrainerErr: true, @@ -771,7 +771,7 @@ func TestWorkerPoolWithErrors(t *testing.T) { opErr: "", }, { - fp: "github.com/pingcap/tidb/pkg/lightning/backend/local/mockRunJobSucceed", + fp: "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/mockRunJobSucceed", expr: "return", mockGeneratorErr: true, mockDrainerErr: false, @@ -779,7 +779,7 @@ func TestWorkerPoolWithErrors(t *testing.T) { opErr: "", }, { - fp: "github.com/pingcap/tidb/pkg/lightning/backend/local/injectPanicForRegionJob", + fp: "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/injectPanicForRegionJob", expr: "panic", mockGeneratorErr: false, mockDrainerErr: false, diff --git a/pkg/lightning/backend/local/tikv_mode.go b/pkg/ingestor/ingestctrl/tikv_mode.go similarity index 99% rename from pkg/lightning/backend/local/tikv_mode.go rename to pkg/ingestor/ingestctrl/tikv_mode.go index cd9506d7283df..bc35f35b28ebd 100644 --- a/pkg/lightning/backend/local/tikv_mode.go +++ b/pkg/ingestor/ingestctrl/tikv_mode.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package local +package ingestctrl import ( "context" diff --git a/pkg/lightning/log/filter_test.go b/pkg/lightning/log/filter_test.go index 53a3028ff134c..aacdab568d24f 100644 --- a/pkg/lightning/log/filter_test.go +++ b/pkg/lightning/log/filter_test.go @@ -51,6 +51,25 @@ func TestFilter(t *testing.T) { logger.Warn("the message", zap.String("stack", "github.com/pingcap/tidb/br/")) require.Len(t, buffer.Stripped(), 0) + + entry := zapcore.Entry{ + Level: zap.WarnLevel, + Message: "retryable write", + Caller: zapcore.EntryCaller{ + Defined: true, + Function: "github.com/pingcap/tidb/pkg/ingestor/ingestctrl.(*regionJobBaseWorker).runJob", + }, + } + + logger, buffer = log.MakeTestLogger() + core := log.NewFilterCore(logger.Logger.Core(), "/ingestor/ingestctrl") + require.NoError(t, core.Write(entry, nil)) + require.Contains(t, buffer.Stripped(), `"retryable write"`) + + logger, buffer = log.MakeTestLogger() + core = log.NewFilterCore(logger.Logger.Core(), "/ingestor/ingestctrl/") + require.NoError(t, core.Write(entry, nil)) + require.Len(t, buffer.Stripped(), 0) } // BenchmarkFilterStringsContains-16 16693887 66.68 ns/op diff --git a/pkg/lightning/log/log.go b/pkg/lightning/log/log.go index 38aa284e037ab..aa26fe5719335 100644 --- a/pkg/lightning/log/log.go +++ b/pkg/lightning/log/log.go @@ -101,6 +101,7 @@ func InitLogger(cfg *Config, _ string) error { core, "github.com/pingcap/tidb/br/", "/lightning/", + "/ingestor/ingestctrl", "main.main", "github.com/tikv/pd/client", ) diff --git a/pkg/server/handler/tikvhandler/BUILD.bazel b/pkg/server/handler/tikvhandler/BUILD.bazel index 643bd4b7e12ee..2310d5304e542 100644 --- a/pkg/server/handler/tikvhandler/BUILD.bazel +++ b/pkg/server/handler/tikvhandler/BUILD.bazel @@ -23,8 +23,8 @@ go_library( "//pkg/executor", "//pkg/infoschema", "//pkg/infoschema/context", + "//pkg/ingestor/ingestctrl", "//pkg/kv", - "//pkg/lightning/backend/local", "//pkg/meta", "//pkg/meta/metadef", "//pkg/meta/model", diff --git a/pkg/server/handler/tikvhandler/tikv_handler.go b/pkg/server/handler/tikvhandler/tikv_handler.go index de7e717778a56..51cff6f1040f4 100644 --- a/pkg/server/handler/tikvhandler/tikv_handler.go +++ b/pkg/server/handler/tikvhandler/tikv_handler.go @@ -43,8 +43,8 @@ import ( "github.com/pingcap/tidb/pkg/executor" "github.com/pingcap/tidb/pkg/infoschema" infoschemacontext "github.com/pingcap/tidb/pkg/infoschema/context" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/meta" "github.com/pingcap/tidb/pkg/meta/metadef" "github.com/pingcap/tidb/pkg/meta/model" @@ -2276,9 +2276,9 @@ func (h IngestConcurrencyHandler) ServeHTTP(w http.ResponseWriter, req *http.Req return m.SetIngestMaxBatchSplitRanges(int(value)) } updateGlobal = func(v float64) float64 { - old := local.CurrentMaxBatchSplitRanges.Load() + old := ingestctrl.CurrentMaxBatchSplitRanges.Load() intV := int(v) - local.CurrentMaxBatchSplitRanges.Store(&intV) + ingestctrl.CurrentMaxBatchSplitRanges.Store(&intV) return float64(*old) } case IngestParamMaxSplitRangesPerSec: @@ -2289,8 +2289,8 @@ func (h IngestConcurrencyHandler) ServeHTTP(w http.ResponseWriter, req *http.Req return m.SetIngestMaxSplitRangesPerSec(value) } updateGlobal = func(v float64) float64 { - old := local.CurrentMaxSplitRangesPerSec.Load() - local.CurrentMaxSplitRangesPerSec.Store(&v) + old := ingestctrl.CurrentMaxSplitRangesPerSec.Load() + ingestctrl.CurrentMaxSplitRangesPerSec.Store(&v) return *old } case IngestParamMaxPerSecond: @@ -2301,8 +2301,8 @@ func (h IngestConcurrencyHandler) ServeHTTP(w http.ResponseWriter, req *http.Req return m.SetIngestMaxPerSec(value) } updateGlobal = func(v float64) float64 { - old := local.CurrentMaxIngestPerSec.Load() - local.CurrentMaxIngestPerSec.Store(&v) + old := ingestctrl.CurrentMaxIngestPerSec.Load() + ingestctrl.CurrentMaxIngestPerSec.Store(&v) return *old } case IngestParamMaxInflight: @@ -2314,9 +2314,9 @@ func (h IngestConcurrencyHandler) ServeHTTP(w http.ResponseWriter, req *http.Req return m.SetIngestMaxInflight(int(value)) } updateGlobal = func(v float64) float64 { - old := local.CurrentMaxIngestInflight.Load() + old := ingestctrl.CurrentMaxIngestInflight.Load() intV := int(v) - local.CurrentMaxIngestInflight.Store(&intV) + ingestctrl.CurrentMaxIngestInflight.Store(&intV) return float64(*old) } default: diff --git a/tests/realtikvtest/addindextest/add_index_test.go b/tests/realtikvtest/addindextest/add_index_test.go index 08031dc8d282b..7cc9c3924c9b9 100644 --- a/tests/realtikvtest/addindextest/add_index_test.go +++ b/tests/realtikvtest/addindextest/add_index_test.go @@ -42,8 +42,8 @@ func enableFastAddIndexFailpoints(t *testing.T) { // by EpochNotMatch or other errors, and can make those cases faster. // we also reduce the retry backoff to make the test faster when region job // is re-queued, as CI ENV might be slow and cause more re-queues. - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/forceSplitRegion", "return(true)") - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/adjustRegionJobRetryBackoff", + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/forceSplitRegion", "return(true)") + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/adjustRegionJobRetryBackoff", func(backoff *time.Duration) { *backoff = time.Second }) diff --git a/tests/realtikvtest/addindextest1/BUILD.bazel b/tests/realtikvtest/addindextest1/BUILD.bazel index cdca0f96529c3..5c52cfb7ae81e 100644 --- a/tests/realtikvtest/addindextest1/BUILD.bazel +++ b/tests/realtikvtest/addindextest1/BUILD.bazel @@ -22,9 +22,9 @@ go_test( "//pkg/dxf/framework/taskexecutor", "//pkg/dxf/framework/testutil", "//pkg/errno", + "//pkg/ingestor/ingestctrl", "//pkg/keyspace", "//pkg/kv", - "//pkg/lightning/backend/local", "//pkg/meta/model", "//pkg/sessionctx/vardef", "//pkg/store", diff --git a/tests/realtikvtest/addindextest1/disttask_test.go b/tests/realtikvtest/addindextest1/disttask_test.go index d4c2bba9ad0d9..20ce896088735 100644 --- a/tests/realtikvtest/addindextest1/disttask_test.go +++ b/tests/realtikvtest/addindextest1/disttask_test.go @@ -38,8 +38,8 @@ import ( "github.com/pingcap/tidb/pkg/dxf/framework/taskexecutor" "github.com/pingcap/tidb/pkg/dxf/framework/testutil" "github.com/pingcap/tidb/pkg/errno" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/sessionctx/vardef" "github.com/pingcap/tidb/pkg/testkit" @@ -180,7 +180,7 @@ func TestAddIndexDistCancelWithPartition(t *testing.T) { tk.MustExec("split table t between (3) and (8646911284551352360) regions 50;") var once sync.Once - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/mockDMLExecutionAddIndexSubTaskFinish", func(*local.Backend) { + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/mockDMLExecutionAddIndexSubTaskFinish", func(*ingestctrl.Backend) { once.Do(func() { row := tk1.MustQuery("select job_id from mysql.tidb_ddl_job").Rows() require.Equal(t, 1, len(row)) @@ -259,7 +259,7 @@ func TestAddIndexDistCancel(t *testing.T) { enter := make(chan struct{}) testfailpoint.EnableCall( t, - "github.com/pingcap/tidb/pkg/lightning/backend/local/beforeExecuteRegionJob", + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/beforeExecuteRegionJob", func(ctx context.Context) { close(enter) select { @@ -304,7 +304,7 @@ func TestAddIndexDistPauseAndResume(t *testing.T) { var syncChan = make(chan struct{}) var counter atomic.Int32 - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/mockDMLExecutionAddIndexSubTaskFinish", func(*local.Backend) { + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/mockDMLExecutionAddIndexSubTaskFinish", func(*ingestctrl.Backend) { if counter.Add(1) <= 3 { row := tk1.MustQuery("select job_id from mysql.tidb_ddl_job").Rows() require.Equal(t, 1, len(row)) @@ -481,7 +481,7 @@ func TestAddIndexScheduleAway(t *testing.T) { close(afterCancel) }) var once sync.Once - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/mockDMLExecutionAddIndexSubTaskFinish", func(*local.Backend) { + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/mockDMLExecutionAddIndexSubTaskFinish", func(*ingestctrl.Backend) { once.Do(func() { tk1 := testkit.NewTestKit(t, store) tk1.MustExec("use test") diff --git a/tests/realtikvtest/addindextest2/BUILD.bazel b/tests/realtikvtest/addindextest2/BUILD.bazel index 12b716d0ccd41..690970ecab2ae 100644 --- a/tests/realtikvtest/addindextest2/BUILD.bazel +++ b/tests/realtikvtest/addindextest2/BUILD.bazel @@ -23,8 +23,8 @@ go_test( "//pkg/dxf/framework/testutil", "//pkg/dxf/operator", "//pkg/ingestor/globalsort", + "//pkg/ingestor/ingestctrl", "//pkg/kv", - "//pkg/lightning/backend/local", "//pkg/meta/model", "//pkg/objstore", "//pkg/session", diff --git a/tests/realtikvtest/addindextest2/alter_job_test.go b/tests/realtikvtest/addindextest2/alter_job_test.go index 504109a52e207..3495c4273d7aa 100644 --- a/tests/realtikvtest/addindextest2/alter_job_test.go +++ b/tests/realtikvtest/addindextest2/alter_job_test.go @@ -25,7 +25,7 @@ import ( "github.com/pingcap/tidb/pkg/dxf/framework/proto" "github.com/pingcap/tidb/pkg/dxf/framework/testutil" "github.com/pingcap/tidb/pkg/dxf/operator" - "github.com/pingcap/tidb/pkg/lightning/backend/local" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/testkit/testfailpoint" @@ -110,7 +110,7 @@ func TestAlterJobOnDXF(t *testing.T) { require.EqualValues(t, 6, writer.GetWorkerPoolSize()) }) var finishedSubtasks int - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/mockDMLExecutionAddIndexSubTaskFinish", func(be *local.Backend) { + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/mockDMLExecutionAddIndexSubTaskFinish", func(be *ingestctrl.Backend) { finishedSubtasks++ require.EqualValues(t, 1024, be.GetWriteSpeedLimit()) }) diff --git a/tests/realtikvtest/addindextest2/global_sort_test.go b/tests/realtikvtest/addindextest2/global_sort_test.go index ccb9bdc8a309f..7d28067b22d3a 100644 --- a/tests/realtikvtest/addindextest2/global_sort_test.go +++ b/tests/realtikvtest/addindextest2/global_sort_test.go @@ -810,10 +810,10 @@ func TestSplitRangeForTable(t *testing.T) { }) var addCnt, removeCnt atomic.Int32 - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/AddPartitionRangeForTable", func() { + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/AddPartitionRangeForTable", func() { addCnt.Add(1) }) - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/RemovePartitionRangeRequest", func() { + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/RemovePartitionRangeRequest", func() { removeCnt.Add(1) }) @@ -837,7 +837,7 @@ func TestSplitRangeForTable(t *testing.T) { // 2. Verify large table case (mocked by lowering threshold) // We only need to verify the logic once for the local backend logic. if tc.caseName == "local ingest" { - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/ForcePartitionRegionThreshold", func(threshold *int) { + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/ForcePartitionRegionThreshold", func(threshold *int) { *threshold = 0 }) addCnt.Store(0) @@ -886,10 +886,10 @@ func TestSplitRangeForPartitionTable(t *testing.T) { for i, tc := range testcases { t.Run(tc.caseName, func(t *testing.T) { var addCnt, removeCnt atomic.Int32 - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/AddPartitionRangeForTable", func() { + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/AddPartitionRangeForTable", func() { addCnt.Add(1) }) - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/RemovePartitionRangeRequest", func() { + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/RemovePartitionRangeRequest", func() { removeCnt.Add(1) }) @@ -910,7 +910,7 @@ func TestSplitRangeForPartitionTable(t *testing.T) { // 2. Verify large table case (mocked by lowering threshold) if tc.caseName == "local ingest" { - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/ForcePartitionRegionThreshold", func(threshold *int) { + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/ForcePartitionRegionThreshold", func(threshold *int) { *threshold = 0 }) addCnt.Store(0) diff --git a/tests/realtikvtest/addindextest3/BUILD.bazel b/tests/realtikvtest/addindextest3/BUILD.bazel index b3559877ed7d7..0e2fcb8de3602 100644 --- a/tests/realtikvtest/addindextest3/BUILD.bazel +++ b/tests/realtikvtest/addindextest3/BUILD.bazel @@ -24,8 +24,8 @@ go_test( "//pkg/dxf/framework/testutil", "//pkg/dxf/operator", "//pkg/errno", + "//pkg/ingestor/ingestctrl", "//pkg/kv", - "//pkg/lightning/backend/local", "//pkg/meta/model", "//pkg/parser/ast", "//pkg/resourcemanager/pool/workerpool", diff --git a/tests/realtikvtest/addindextest3/ingest_test.go b/tests/realtikvtest/addindextest3/ingest_test.go index a4bfa43aead3d..1cbf19591699f 100644 --- a/tests/realtikvtest/addindextest3/ingest_test.go +++ b/tests/realtikvtest/addindextest3/ingest_test.go @@ -32,7 +32,7 @@ import ( "github.com/pingcap/tidb/pkg/ddl/ingest" disttestutil "github.com/pingcap/tidb/pkg/dxf/framework/testutil" "github.com/pingcap/tidb/pkg/errno" - "github.com/pingcap/tidb/pkg/lightning/backend/local" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/testkit/testfailpoint" @@ -58,10 +58,10 @@ func TestAddIndexIngestMemoryUsage(t *testing.T) { tk.MustExec(`set global tidb_ddl_enable_fast_reorg=on;`) } - oldRunInTest := local.RunInTest - local.RunInTest = true + oldRunInTest := ingestctrl.RunInTest + ingestctrl.RunInTest = true t.Cleanup(func() { - local.RunInTest = oldRunInTest + ingestctrl.RunInTest = oldRunInTest }) tk.MustExec("create table t (a int, b int, c int);") @@ -81,7 +81,7 @@ func TestAddIndexIngestMemoryUsage(t *testing.T) { tk.MustExec("alter table t add unique index idx1(b);") tk.MustExec("admin check table t;") require.Equal(t, int64(0), ingest.LitMemRoot.CurrentUsage()) - require.NoError(t, local.LastAlloc.Load().CheckRefCnt()) + require.NoError(t, ingestctrl.LastAlloc.Load().CheckRefCnt()) } func TestAddIndexIngestLimitOneBackend(t *testing.T) { @@ -134,7 +134,7 @@ func TestAddIndexIngestLimitOneBackend(t *testing.T) { blockOnce := atomic.Bool{} testfailpoint.EnableCall( t, - "github.com/pingcap/tidb/pkg/lightning/backend/local/beforeExecuteRegionJob", + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/beforeExecuteRegionJob", func(ctx context.Context) { if !blockOnce.CompareAndSwap(false, true) { return @@ -505,7 +505,7 @@ func testAddIndexDiskQuotaTS(t *testing.T, tk *testkit.TestKit) { testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/wrapInBeginRollbackAfterFn", func() { counter-- }) - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/ReadyForImportEngine", func() { + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/ReadyForImportEngine", func() { assert.Equal(t, counter, 0) }) tk.MustExec("alter table t add index idx_test2(b);") @@ -547,7 +547,7 @@ func TestAddIndexAdvanceWatermarkFailed(t *testing.T) { tk.MustGetErrCode("alter table t add unique index idx(b);", errno.ErrDupEntry) testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ddl/ingest/mockAfterImportAllocTSFailed", "1*return") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/afterSetTSBeforeImportEngine", "1*return") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/afterSetTSBeforeImportEngine", "1*return") tk.MustGetErrCode("alter table t add unique index idx(b);", errno.ErrDupEntry) } @@ -567,10 +567,10 @@ func TestAddIndexTempDirDataRemoved(t *testing.T) { tk.MustExec("create table t (a int);") tk.MustExec("insert into t values (1), (1), (1);") - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/mockErrInMergeSSTs", "1*return(true)") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/mockErrInMergeSSTs", "1*return(true)") removeOnce := sync.Once{} removed := false - testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/beforeMergeSSTs", func() { + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/beforeMergeSSTs", func() { removeOnce.Do(func() { var filesToRemove []string filepath.WalkDir(tempDir, func(path string, d fs.DirEntry, err error) error { @@ -738,10 +738,10 @@ func TestAddIndexImportFailed(t *testing.T) { tk.MustExec(insertSQL) } - err := failpoint.Enable("github.com/pingcap/tidb/pkg/lightning/backend/local/mockWritePeerErr", "1*return") + err := failpoint.Enable("github.com/pingcap/tidb/pkg/ingestor/ingestctrl/mockWritePeerErr", "1*return") require.NoError(t, err) tk.MustExec("alter table t add index idx(a);") - err = failpoint.Disable("github.com/pingcap/tidb/pkg/lightning/backend/local/mockWritePeerErr") + err = failpoint.Disable("github.com/pingcap/tidb/pkg/ingestor/ingestctrl/mockWritePeerErr") require.NoError(t, err) tk.MustExec("admin check table t;") } @@ -803,9 +803,9 @@ func TestFirstLitSlowStart(t *testing.T) { t.Cleanup(func() { require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/ownerResignAfterDispatchLoopCheck")) }) - require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/lightning/backend/local/slowCreateFS", "return()")) + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ingestor/ingestctrl/slowCreateFS", "return()")) t.Cleanup(func() { - require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/lightning/backend/local/slowCreateFS")) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ingestor/ingestctrl/slowCreateFS")) }) var wg sync.WaitGroup @@ -872,17 +872,17 @@ func TestIssue55808(t *testing.T) { tk.MustExec("set global tidb_ddl_error_count_limit = default;") }() - backup := local.MaxWriteAndIngestRetryTimes - local.MaxWriteAndIngestRetryTimes = 1 + backup := ingestctrl.MaxWriteAndIngestRetryTimes + ingestctrl.MaxWriteAndIngestRetryTimes = 1 t.Cleanup(func() { - local.MaxWriteAndIngestRetryTimes = backup + ingestctrl.MaxWriteAndIngestRetryTimes = backup }) tk.MustExec("create table t (a int primary key, b int);") for i := range 4 { tk.MustExec(fmt.Sprintf("insert into t values (%d, %d);", i*10000, i*10000)) } - testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/lightning/backend/local/doIngestFailed", "return()") + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/doIngestFailed", "return()") err := tk.ExecToErr("alter table t add index idx(a);") require.ErrorContains(t, err, "injected error") } diff --git a/tests/realtikvtest/importintotest/BUILD.bazel b/tests/realtikvtest/importintotest/BUILD.bazel index 12b123531a52d..0fb03cff09d99 100644 --- a/tests/realtikvtest/importintotest/BUILD.bazel +++ b/tests/realtikvtest/importintotest/BUILD.bazel @@ -40,8 +40,8 @@ go_test( "//pkg/executor/importer", "//pkg/importsdk", "//pkg/infoschema", + "//pkg/ingestor/ingestctrl", "//pkg/kv", - "//pkg/lightning/backend/local", "//pkg/lightning/common", "//pkg/meta", "//pkg/parser/ast", diff --git a/tests/realtikvtest/importintotest/import_into_test.go b/tests/realtikvtest/importintotest/import_into_test.go index 046b5999dff56..81201e3cf936f 100644 --- a/tests/realtikvtest/importintotest/import_into_test.go +++ b/tests/realtikvtest/importintotest/import_into_test.go @@ -46,8 +46,8 @@ import ( "github.com/pingcap/tidb/pkg/dxf/importinto" "github.com/pingcap/tidb/pkg/executor/importer" "github.com/pingcap/tidb/pkg/infoschema" + "github.com/pingcap/tidb/pkg/ingestor/ingestctrl" "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/meta" "github.com/pingcap/tidb/pkg/parser/ast" @@ -947,7 +947,7 @@ func (s *mockGCSSuite) TestImportMode() { switcher.EXPECT().ToImportMode(gomock.Any(), gomock.Any()).DoAndReturn(toImportModeFn).Times(1) switcher.EXPECT().ToNormalMode(gomock.Any(), gomock.Any()).DoAndReturn(toNormalModeFn).Times(1) backup := importer.NewTiKVModeSwitcher - importer.NewTiKVModeSwitcher = func(*tls.Config, pdhttp.Client, *zap.Logger) local.TiKVModeSwitcher { + importer.NewTiKVModeSwitcher = func(*tls.Config, pdhttp.Client, *zap.Logger) ingestctrl.TiKVModeSwitcher { return switcher } s.T().Cleanup(func() { diff --git a/tests/realtikvtest/importintotest2/from_select_test.go b/tests/realtikvtest/importintotest2/from_select_test.go index 43f98fc458804..c1217e18b4f36 100644 --- a/tests/realtikvtest/importintotest2/from_select_test.go +++ b/tests/realtikvtest/importintotest2/from_select_test.go @@ -176,6 +176,6 @@ func (s *mockGCSSuite) TestDiskFullOnIngestFailFast() { } s.prepareAndUseDB("from_select") s.tk.MustExec("create table dt(id int unsigned)") - testfailpoint.Enable(s.T(), "github.com/pingcap/tidb/pkg/lightning/backend/local/diskFullOnIngest", `return(true)`) + testfailpoint.Enable(s.T(), "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/diskFullOnIngest", `return(true)`) s.ErrorContains(s.tk.ExecToErr("import into dt from select 1"), "tikv disk full") } diff --git a/tests/realtikvtest/importintotest4/global_sort_test.go b/tests/realtikvtest/importintotest4/global_sort_test.go index f718981fcbebc..c349e16d0c64a 100644 --- a/tests/realtikvtest/importintotest4/global_sort_test.go +++ b/tests/realtikvtest/importintotest4/global_sort_test.go @@ -416,14 +416,14 @@ func (s *mockGCSSuite) TestSplitRangeForTable() { s.tk.MustExec(`create table t (a bigint primary key, b varchar(100), c varchar(100), d int, key(a), key(c,d), key(d));`) - testfailpoint.EnableCall(s.T(), "github.com/pingcap/tidb/pkg/lightning/backend/local/ForcePartitionRegionThreshold", func(threshold *int) { + testfailpoint.EnableCall(s.T(), "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/ForcePartitionRegionThreshold", func(threshold *int) { *threshold = 0 }) var addCnt, removeCnt atomic.Int32 - testfailpoint.EnableCall(s.T(), "github.com/pingcap/tidb/pkg/lightning/backend/local/AddPartitionRangeForTable", func() { + testfailpoint.EnableCall(s.T(), "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/AddPartitionRangeForTable", func() { addCnt.Add(1) }) - testfailpoint.EnableCall(s.T(), "github.com/pingcap/tidb/pkg/lightning/backend/local/RemovePartitionRangeRequest", func() { + testfailpoint.EnableCall(s.T(), "github.com/pingcap/tidb/pkg/ingestor/ingestctrl/RemovePartitionRangeRequest", func() { removeCnt.Add(1) }) dom, err := session.GetDomain(s.store)