Skip to content

Force migration key for composite chunk boundaries #1761

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jakubpliszka merged 1 commit into from
Aug 28, 2026
Merged

Force migration key for composite chunk boundaries #1761

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go/logic/applier.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1106,6 +1106,7 @@ func (apl *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange bool
query, explodedArgs, err := buildFunc(
apl.migrationContext.DatabaseName,
apl.migrationContext.OriginalTableName,
apl.migrationContext.UniqueKey.Name,
&apl.migrationContext.UniqueKey.Columns,
apl.migrationContext.MigrationIterationRangeMinValues.AbstractValues(),
apl.migrationContext.MigrationRangeMaxValues.AbstractValues(),
Expand Down
24 changes: 12 additions & 12 deletions go/sql/builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,7 @@ func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableNa
return BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, includeRangeStartValues, transactionalTable, noWait)
}

func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) {
func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName, uniqueKey string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) {
if uniqueKeyColumns.Len() == 0 {
return "", explodedArgs, fmt.Errorf("got 0 columns in BuildUniqueKeyRangeEndPreparedQuery")
}
Expand All @@ -438,7 +438,7 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string
}

if uniqueKeyColumns.Len() == 2 {
return buildUniqueKeyRangeEndTwoColumnViaOffset(databaseName, tableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, startRangeComparisonSign, hint)
return buildUniqueKeyRangeEndTwoColumnViaOffset(databaseName, tableName, EscapeName(uniqueKey), uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, startRangeComparisonSign, hint)
}

rangeStartComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(uniqueKeyColumns, rangeStartArgs, startRangeComparisonSign)
Expand Down Expand Up @@ -483,7 +483,7 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string
return result, explodedArgs, nil
}

func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) {
func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName, uniqueKey string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) {
if uniqueKeyColumns.Len() == 0 {
return "", explodedArgs, fmt.Errorf("got 0 columns in BuildUniqueKeyRangeEndPreparedQuery")
}
Expand All @@ -496,7 +496,7 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str
}

if uniqueKeyColumns.Len() == 2 {
return buildUniqueKeyRangeEndTwoColumnViaTemptable(databaseName, tableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, startRangeComparisonSign, hint)
return buildUniqueKeyRangeEndTwoColumnViaTemptable(databaseName, tableName, EscapeName(uniqueKey), uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, startRangeComparisonSign, hint)
}

rangeStartComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(uniqueKeyColumns, rangeStartArgs, startRangeComparisonSign)
Expand Down Expand Up @@ -616,7 +616,7 @@ func buildTwoColumnUnionParts(
}

func buildUniqueKeyRangeEndTwoColumnViaOffset(
databaseName, tableName string,
databaseName, tableName, uniqueKey string,
uniqueKeyColumns *ColumnList,
rangeStartArgs, rangeEndArgs []interface{},
chunkSize int64,
Expand All @@ -629,15 +629,15 @@ func buildUniqueKeyRangeEndTwoColumnViaOffset(
}
col2StartOp := string(startRangeComparisonSign)
selectClause := m.col1Name + ", " + m.col2Name
fromClause := databaseName + "." + tableName
fromClause := fmt.Sprintf("%s.%s force index (%s)", databaseName, tableName, uniqueKey)
partSuffix := fmt.Sprintf("order by %s limit %d", m.orderByAsc, chunkSize)

if sameFirstColumnValue(rangeStartArgs, rangeEndArgs) {
result = fmt.Sprintf(`
select /* gh-ost %s.%s %s */
%s, %s
from
%s.%s
%s
where
(%s = %s and %s %s %s and %s <= %s)
order by
Expand All @@ -646,7 +646,7 @@ func buildUniqueKeyRangeEndTwoColumnViaOffset(
offset %d`,
databaseName, tableName, hint,
m.col1Name, m.col2Name,
databaseName, tableName,
fromClause,
m.col1Name, m.col1Val, m.col2Name, col2StartOp, m.col2Val, m.col2Name, m.col2Val,
m.orderByAsc,
chunkSize-1,
Expand Down Expand Up @@ -682,7 +682,7 @@ func buildUniqueKeyRangeEndTwoColumnViaOffset(
}

func buildUniqueKeyRangeEndTwoColumnViaTemptable(
databaseName, tableName string,
databaseName, tableName, uniqueKey string,
uniqueKeyColumns *ColumnList,
rangeStartArgs, rangeEndArgs []interface{},
chunkSize int64,
Expand All @@ -695,15 +695,15 @@ func buildUniqueKeyRangeEndTwoColumnViaTemptable(
}
col2StartOp := string(startRangeComparisonSign)
selectClause := m.col1Name + ", " + m.col2Name
fromClause := databaseName + "." + tableName
fromClause := fmt.Sprintf("%s.%s force index (%s)", databaseName, tableName, uniqueKey)
partSuffix := fmt.Sprintf("order by %s limit %d", m.orderByAsc, chunkSize)

if sameFirstColumnValue(rangeStartArgs, rangeEndArgs) {
result = fmt.Sprintf(`
select /* gh-ost %s.%s %s */ %s, %s
from (
select %s, %s
from %s.%s
from %s
where (%s = %s and %s %s %s and %s <= %s)
order by %s
limit %d
Expand All @@ -712,7 +712,7 @@ func buildUniqueKeyRangeEndTwoColumnViaTemptable(
limit 1`,
databaseName, tableName, hint, m.col1Name, m.col2Name,
m.col1Name, m.col2Name,
databaseName, tableName,
fromClause,
m.col1Name, m.col1Val, m.col2Name, col2StartOp, m.col2Val, m.col2Name, m.col2Val,
m.orderByAsc, chunkSize,
m.orderByDesc,
Expand Down
41 changes: 21 additions & 20 deletions go/sql/builder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,21 +423,22 @@ func TestBuildUniqueKeyRangeEndPreparedQueryViaOffset(t *testing.T) {
var chunkSize int64 = 500
{
// Different first-column values → 3-part UNION for efficient boundary seeks.
// A named (non-PRIMARY) key asserts the caller-supplied key reaches the force index hint.
uniqueKeyColumns := NewColumnList([]string{"name", "position"})
rangeStartArgs := []interface{}{3, 17}
rangeEndArgs := []interface{}{103, 117}

query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, originalTableName, "name_position_uidx", uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
require.NoError(t, err)
expected := `
select /* gh-ost mydb.tbl test */
name, position
from
((select name, position from mydb.tbl where name = ? and position > ? order by name asc, position asc limit 500)
((select name, position from mydb.tbl force index (name_position_uidx) where name = ? and position > ? order by name asc, position asc limit 500)
union all
(select name, position from mydb.tbl where name > ? and name < ? order by name asc, position asc limit 500)
(select name, position from mydb.tbl force index (name_position_uidx) where name > ? and name < ? order by name asc, position asc limit 500)
union all
(select name, position from mydb.tbl where name = ? and position <= ? order by name asc, position asc limit 500)) t
(select name, position from mydb.tbl force index (name_position_uidx) where name = ? and position <= ? order by name asc, position asc limit 500)) t
order by
name asc, position asc
limit 1
Expand All @@ -451,13 +452,13 @@ func TestBuildUniqueKeyRangeEndPreparedQueryViaOffset(t *testing.T) {
rangeStartArgs := []interface{}{3, 17}
rangeEndArgs := []interface{}{3, 117}

query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, originalTableName, "PRIMARY", uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
require.NoError(t, err)
expected := `
select /* gh-ost mydb.tbl test */
name, position
from
mydb.tbl
mydb.tbl force index (PRIMARY)
where
(name = ? and position > ? and position <= ?)
order by
Expand All @@ -479,18 +480,18 @@ func TestBuildUniqueKeyRangeEndPreparedQueryViaTemptable(t *testing.T) {
rangeStartArgs := []interface{}{3, 17}
rangeEndArgs := []interface{}{103, 117}

query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, originalTableName, "PRIMARY", uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
require.NoError(t, err)
expected := `
select /* gh-ost mydb.tbl test */ name, position
from (
select name, position
from
((select name, position from mydb.tbl where name = ? and position > ? order by name asc, position asc limit 500)
((select name, position from mydb.tbl force index (PRIMARY) where name = ? and position > ? order by name asc, position asc limit 500)
union all
(select name, position from mydb.tbl where name > ? and name < ? order by name asc, position asc limit 500)
(select name, position from mydb.tbl force index (PRIMARY) where name > ? and name < ? order by name asc, position asc limit 500)
union all
(select name, position from mydb.tbl where name = ? and position <= ? order by name asc, position asc limit 500)) t
(select name, position from mydb.tbl force index (PRIMARY) where name = ? and position <= ? order by name asc, position asc limit 500)) t
order by name asc, position asc
limit 500
) select_osc_chunk
Expand All @@ -505,13 +506,13 @@ func TestBuildUniqueKeyRangeEndPreparedQueryViaTemptable(t *testing.T) {
rangeStartArgs := []interface{}{3, 17}
rangeEndArgs := []interface{}{3, 117}

query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, originalTableName, "PRIMARY", uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
require.NoError(t, err)
expected := `
select /* gh-ost mydb.tbl test */ name, position
from (
select name, position
from mydb.tbl
from mydb.tbl force index (PRIMARY)
where (name = ? and position > ? and position <= ?)
order by name asc, position asc
limit 500
Expand All @@ -535,17 +536,17 @@ func TestBuildUniqueKeyRangeEndPreparedQueryTwoColumnEnum(t *testing.T) {
rangeStartArgs := []interface{}{"a", 17}
rangeEndArgs := []interface{}{"z", 117}

query, _, err := BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
query, _, err := BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, originalTableName, "PRIMARY", uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
require.NoError(t, err)
expected := `
select /* gh-ost mydb.tbl test */
name, position
from
((select name, position from mydb.tbl where name = ? and position > ? order by concat(name) asc, position asc limit 500)
((select name, position from mydb.tbl force index (PRIMARY) where name = ? and position > ? order by concat(name) asc, position asc limit 500)
union all
(select name, position from mydb.tbl where name > ? and name < ? order by concat(name) asc, position asc limit 500)
(select name, position from mydb.tbl force index (PRIMARY) where name > ? and name < ? order by concat(name) asc, position asc limit 500)
union all
(select name, position from mydb.tbl where name = ? and position <= ? order by concat(name) asc, position asc limit 500)) t
(select name, position from mydb.tbl force index (PRIMARY) where name = ? and position <= ? order by concat(name) asc, position asc limit 500)) t
order by
concat(name) asc, position asc
limit 1
Expand All @@ -560,18 +561,18 @@ func TestBuildUniqueKeyRangeEndPreparedQueryTwoColumnEnum(t *testing.T) {
rangeStartArgs := []interface{}{3, "a"}
rangeEndArgs := []interface{}{103, "z"}

query, _, err := BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
query, _, err := BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, originalTableName, "PRIMARY", uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
require.NoError(t, err)
expected := `
select /* gh-ost mydb.tbl test */ name, position
from (
select name, position
from
((select name, position from mydb.tbl where name = ? and position > ? order by name asc, concat(position) asc limit 500)
((select name, position from mydb.tbl force index (PRIMARY) where name = ? and position > ? order by name asc, concat(position) asc limit 500)
union all
(select name, position from mydb.tbl where name > ? and name < ? order by name asc, concat(position) asc limit 500)
(select name, position from mydb.tbl force index (PRIMARY) where name > ? and name < ? order by name asc, concat(position) asc limit 500)
union all
(select name, position from mydb.tbl where name = ? and position <= ? order by name asc, concat(position) asc limit 500)) t
(select name, position from mydb.tbl force index (PRIMARY) where name = ? and position <= ? order by name asc, concat(position) asc limit 500)) t
order by name asc, concat(position) asc
limit 500
) select_osc_chunk
Expand Down