Skip to content

Commit 791d963

Browse files
author
Shlomi Noach
committed
Character set recognition and manipulation
- Identifying textual characters sets; converting into specific type when applying dml events - Refactored `ColumnsList`: introducing `Column` type - Refactored `unsigned` handling, as part of `Column` - `Column` type supports `convertArg()`: converting value of argument according to column data type - DB URI attempts `utf8mb4,utf8,latin1` charsets in that order (first one to be recognized wins) - Local tests filter by pattern - Local tests append table schema on failure - Local tests do not have postpone flag file - Added character set local tests: `utf8`, `utf8mb4`, `latin1`
1 parent f646021 commit 791d963

File tree

10 files changed

+158
-97
lines changed

10 files changed

+158
-97
lines changed

‎build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
#
33
#
44

5-
RELEASE_VERSION="1.0.17"
5+
RELEASE_VERSION="1.0.18"
66

77
function build {
88
osname=$1

‎go/logic/applier.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,7 @@ func (this *Applier) ExecuteThrottleQuery() (int64, error) {
301301
// ReadMigrationMinValues returns the minimum values to be iterated on rowcopy
302302
func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error {
303303
log.Debugf("Reading migration range according to key: %s", uniqueKey.Name)
304-
query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, uniqueKey.Columns.Names)
304+
query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, uniqueKey.Columns.Names())
305305
if err != nil {
306306
return err
307307
}
@@ -322,7 +322,7 @@ func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error {
322322
// ReadMigrationMaxValues returns the maximum values to be iterated on rowcopy
323323
func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error {
324324
log.Debugf("Reading migration range according to key: %s", uniqueKey.Name)
325-
query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, uniqueKey.Columns.Names)
325+
query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, uniqueKey.Columns.Names())
326326
if err != nil {
327327
return err
328328
}
@@ -363,7 +363,7 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo
363363
query, explodedArgs, err := sql.BuildUniqueKeyRangeEndPreparedQuery(
364364
this.migrationContext.DatabaseName,
365365
this.migrationContext.OriginalTableName,
366-
this.migrationContext.UniqueKey.Columns.Names,
366+
this.migrationContext.UniqueKey.Columns.Names(),
367367
this.migrationContext.MigrationIterationRangeMinValues.AbstractValues(),
368368
this.migrationContext.MigrationRangeMaxValues.AbstractValues(),
369369
atomic.LoadInt64(&this.migrationContext.ChunkSize),
@@ -402,10 +402,10 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected
402402
this.migrationContext.DatabaseName,
403403
this.migrationContext.OriginalTableName,
404404
this.migrationContext.GetGhostTableName(),
405-
this.migrationContext.SharedColumns.Names,
406-
this.migrationContext.MappedSharedColumns.Names,
405+
this.migrationContext.SharedColumns.Names(),
406+
this.migrationContext.MappedSharedColumns.Names(),
407407
this.migrationContext.UniqueKey.Name,
408-
this.migrationContext.UniqueKey.Columns.Names,
408+
this.migrationContext.UniqueKey.Columns.Names(),
409409
this.migrationContext.MigrationIterationRangeMinValues.AbstractValues(),
410410
this.migrationContext.MigrationIterationRangeMaxValues.AbstractValues(),
411411
this.migrationContext.GetIteration() == 0,

‎go/logic/inspect.go‎

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -135,8 +135,8 @@ func (this *Inspector) InspectOriginalAndGhostTables() (err error) {
135135
// This additional step looks at which columns are unsigned. We could have merged this within
136136
// the `getTableColumns()` function, but it's a later patch and introduces some complexity; I feel
137137
// comfortable in doing this as a separate step.
138-
this.applyUnsignedColumns(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns)
139-
this.applyUnsignedColumns(this.migrationContext.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.GhostTableColumns, this.migrationContext.MappedSharedColumns)
138+
this.applyColumnTypes(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns)
139+
this.applyColumnTypes(this.migrationContext.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.GhostTableColumns, this.migrationContext.MappedSharedColumns)
140140

141141
return nil
142142
}
@@ -477,23 +477,31 @@ func (this *Inspector) getTableColumns(databaseName, tableName string) (*sql.Col
477477
return sql.NewColumnList(columnNames), nil
478478
}
479479

480-
// applyUnsignedColumns
481-
func (this *Inspector) applyUnsignedColumns(databaseName, tableName string, columnsLists ...*sql.ColumnList) error {
482-
query := fmt.Sprintf(`
483-
show columns from %s.%s
484-
`,
485-
sql.EscapeName(databaseName),
486-
sql.EscapeName(tableName),
487-
)
488-
err := sqlutils.QueryRowsMap(this.db, query, func(rowMap sqlutils.RowMap) error {
489-
columnName := rowMap.GetString("Field")
490-
if strings.Contains(rowMap.GetString("Type"), "unsigned") {
480+
// applyColumnTypes
481+
func (this *Inspector) applyColumnTypes(databaseName, tableName string, columnsLists ...*sql.ColumnList) error {
482+
query := `
483+
select
484+
*
485+
from
486+
information_schema.columns
487+
where
488+
table_schema=?
489+
and table_name=?
490+
`
491+
err := sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error {
492+
columnName := m.GetString("COLUMN_NAME")
493+
if strings.Contains(m.GetString("COLUMN_TYPE"), "unsigned") {
491494
for _, columnsList := range columnsLists {
492495
columnsList.SetUnsigned(columnName)
493496
}
494497
}
498+
if charset := m.GetString("CHARACTER_SET_NAME"); charset != "" {
499+
for _, columnsList := range columnsLists {
500+
columnsList.SetCharset(columnName, charset)
501+
}
502+
}
495503
return nil
496-
})
504+
}, databaseName, tableName)
497505
return err
498506
}
499507

@@ -583,7 +591,7 @@ func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys [
583591
// the ALTER is on the name itself...
584592
for _, originalUniqueKey := range originalUniqueKeys {
585593
for _, ghostUniqueKey := range ghostUniqueKeys {
586-
if originalUniqueKey.Columns.Equals(&ghostUniqueKey.Columns) {
594+
if originalUniqueKey.Columns.EqualsByNames(&ghostUniqueKey.Columns) {
587595
uniqueKeys = append(uniqueKeys, originalUniqueKey)
588596
}
589597
}
@@ -594,11 +602,11 @@ func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys [
594602
// getSharedColumns returns the intersection of two lists of columns in same order as the first list
595603
func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) {
596604
columnsInGhost := make(map[string]bool)
597-
for _, ghostColumn := range ghostColumns.Names {
605+
for _, ghostColumn := range ghostColumns.Names() {
598606
columnsInGhost[ghostColumn] = true
599607
}
600608
sharedColumnNames := []string{}
601-
for _, originalColumn := range originalColumns.Names {
609+
for _, originalColumn := range originalColumns.Names() {
602610
if columnsInGhost[originalColumn] || columnsInGhost[columnRenameMap[originalColumn]] {
603611
sharedColumnNames = append(sharedColumnNames, originalColumn)
604612
}

‎go/mysql/connection.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,11 @@ func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool {
4848
}
4949

5050
func (this *ConnectionConfig) GetDBUri(databaseName string) string {
51-
var ip = net.ParseIP(this.Key.Hostname)
51+
hostname := this.Key.Hostname
52+
var ip = net.ParseIP(hostname)
5253
if (ip != nil) && (ip.To4() == nil) {
5354
// Wrap IPv6 literals in square brackets
54-
return fmt.Sprintf("%s:%s@tcp([%s]:%d)/%s", this.User, this.Password, this.Key.Hostname, this.Key.Port, databaseName)
55-
} else {
56-
return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", this.User, this.Password, this.Key.Hostname, this.Key.Port, databaseName)
55+
hostname = fmt.Sprintf("[%s]", hostname)
5756
}
57+
return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4,utf8,latin1", this.User, this.Password, hostname, this.Key.Port, databaseName)
5858
}

‎go/sql/builder.go‎

Lines changed: 17 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -32,29 +32,6 @@ func EscapeName(name string) string {
3232
return fmt.Sprintf("`%s`", name)
3333
}
3434

35-
func fixArgType(arg interface{}, isUnsigned bool) interface{} {
36-
if !isUnsigned {
37-
return arg
38-
}
39-
// unsigned
40-
if i, ok := arg.(int8); ok {
41-
return uint8(i)
42-
}
43-
if i, ok := arg.(int16); ok {
44-
return uint16(i)
45-
}
46-
if i, ok := arg.(int32); ok {
47-
return uint32(i)
48-
}
49-
if i, ok := arg.(int64); ok {
50-
return strconv.FormatUint(uint64(i), 10)
51-
}
52-
if i, ok := arg.(int); ok {
53-
return uint(i)
54-
}
55-
return arg
56-
}
57-
5835
func buildPreparedValues(length int) []string {
5936
values := make([]string, length, length)
6037
for i := 0; i < length; i++ {
@@ -330,14 +307,14 @@ func BuildDMLDeleteQuery(databaseName, tableName string, tableColumns, uniqueKey
330307
if uniqueKeyColumns.Len() == 0 {
331308
return result, uniqueKeyArgs, fmt.Errorf("No unique key columns found in BuildDMLDeleteQuery")
332309
}
333-
for _, column := range uniqueKeyColumns.Names {
334-
tableOrdinal := tableColumns.Ordinals[column]
335-
arg := fixArgType(args[tableOrdinal], uniqueKeyColumns.IsUnsigned(column))
310+
for _, column := range uniqueKeyColumns.Columns() {
311+
tableOrdinal := tableColumns.Ordinals[column.Name]
312+
arg := column.convertArg(args[tableOrdinal])
336313
uniqueKeyArgs = append(uniqueKeyArgs, arg)
337314
}
338315
databaseName = EscapeName(databaseName)
339316
tableName = EscapeName(tableName)
340-
equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names)
317+
equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names())
341318
if err != nil {
342319
return result, uniqueKeyArgs, err
343320
}
@@ -367,13 +344,13 @@ func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedCol
367344
databaseName = EscapeName(databaseName)
368345
tableName = EscapeName(tableName)
369346

370-
for _, column := range mappedSharedColumns.Names {
371-
tableOrdinal := tableColumns.Ordinals[column]
372-
arg := fixArgType(args[tableOrdinal], mappedSharedColumns.IsUnsigned(column))
347+
for _, column := range mappedSharedColumns.Columns() {
348+
tableOrdinal := tableColumns.Ordinals[column.Name]
349+
arg := column.convertArg(args[tableOrdinal])
373350
sharedArgs = append(sharedArgs, arg)
374351
}
375352

376-
mappedSharedColumnNames := duplicateNames(mappedSharedColumns.Names)
353+
mappedSharedColumnNames := duplicateNames(mappedSharedColumns.Names())
377354
for i := range mappedSharedColumnNames {
378355
mappedSharedColumnNames[i] = EscapeName(mappedSharedColumnNames[i])
379356
}
@@ -415,26 +392,26 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol
415392
databaseName = EscapeName(databaseName)
416393
tableName = EscapeName(tableName)
417394

418-
for i, column := range sharedColumns.Names {
419-
mappedColumn := mappedSharedColumns.Names[i]
420-
tableOrdinal := tableColumns.Ordinals[column]
421-
arg := fixArgType(valueArgs[tableOrdinal], mappedSharedColumns.IsUnsigned(mappedColumn))
395+
for i, column := range sharedColumns.Columns() {
396+
mappedColumn := mappedSharedColumns.Columns()[i]
397+
tableOrdinal := tableColumns.Ordinals[column.Name]
398+
arg := mappedColumn.convertArg(valueArgs[tableOrdinal])
422399
sharedArgs = append(sharedArgs, arg)
423400
}
424401

425-
for _, column := range uniqueKeyColumns.Names {
426-
tableOrdinal := tableColumns.Ordinals[column]
427-
arg := fixArgType(whereArgs[tableOrdinal], uniqueKeyColumns.IsUnsigned(column))
402+
for _, column := range uniqueKeyColumns.Columns() {
403+
tableOrdinal := tableColumns.Ordinals[column.Name]
404+
arg := column.convertArg(whereArgs[tableOrdinal])
428405
uniqueKeyArgs = append(uniqueKeyArgs, arg)
429406
}
430407

431-
mappedSharedColumnNames := duplicateNames(mappedSharedColumns.Names)
408+
mappedSharedColumnNames := duplicateNames(mappedSharedColumns.Names())
432409
for i := range mappedSharedColumnNames {
433410
mappedSharedColumnNames[i] = EscapeName(mappedSharedColumnNames[i])
434411
}
435412
setClause, err := BuildSetPreparedClause(mappedSharedColumnNames)
436413

437-
equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names)
414+
equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names())
438415
result = fmt.Sprintf(`
439416
update /* gh-ost %s.%s */
440417
%s.%s

0 commit comments

Comments
 (0)