Version 4.1.2
This commit is contained in:
Neo committed
1 parent
3ba79295ee
commit
28bd025b35
1131 files changed
+63658
-20579
No files matched your search
@@ -16,7 +16,7 @@ namespace IPSUtf8;
|
||||
*/
|
||||
class Convert
|
||||
{
|
||||
const VERSION_ID = '1.1.17';
|
||||
const VERSION_ID = '1.1.18';
|
||||
|
||||
/**
|
||||
* @brief Data Store
|
||||
@@ -301,9 +301,13 @@ class Convert
|
||||
/* Even if the current charset is set to UTF-8, if the Table Charset is latin1, we need to use this method */
|
||||
/* ... but only if the actual column isn't using utf8 */
|
||||
$columnCollation = $schematic['definition']['columns'][$col]['collation'];
|
||||
if ( \IPSUtf8\Session::i()->current_charset != 'utf-8' OR ( $tableCharSet != 'utf8' AND ! in_array( $columnCollation, array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) ) OR ( $tableCharSet == 'utf8' AND mb_substr( $columnCollation, 0, 6 ) == 'latin1' ) )
|
||||
if ( defined( 'UTF8_INSERT_ONLY' ) AND UTF8_INSERT_ONLY === TRUE AND $tableCharSet == 'utf8' )
|
||||
{
|
||||
if ( \IPSUtf8\Session::i()->current_charset == 'iso-8859-1' AND $tableCharSet == 'latin1' )
|
||||
$select[] = 'CAST( `' . $col . '` AS BINARY )';
|
||||
}
|
||||
else if ( \IPSUtf8\Session::i()->current_charset != 'utf-8' OR ( $tableCharSet != 'utf8' AND ! in_array( $columnCollation, array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) ) OR ( $tableCharSet == 'utf8' AND mb_substr( $columnCollation, 0, 5 ) == 'latin' ) )
|
||||
{
|
||||
if ( \IPSUtf8\Session::i()->current_charset == 'iso-8859-1' OR mb_substr( $columnCollation, 0, 5 ) == 'latin' )
|
||||
{
|
||||
/* Casting as binary truncates non-latin characters, so if we are using real Latin1, just convert */
|
||||
$select[] = 'CONVERT( `' . $col . '` USING utf8 )';
|
||||
@@ -317,17 +321,19 @@ class Convert
|
||||
else
|
||||
{
|
||||
/**
|
||||
* @note This looks a little weird but bare with me. Semi-old databases (post-2.x but pre-4.x) can have UTF8 Tables and Columns, but contain Latin1 data.
|
||||
* We need to try and sniff this out and apply a different type of conversion, because in this instance you cannot just extract
|
||||
* as BINARY and then insert into the source. Instead, you need to convert to latin1, then to binary, then to utf8. But you can't
|
||||
* do that on actual UTF8 data because it will then truncate multibyte characters so we need to compare the conversion against what
|
||||
* is stored.
|
||||
*/
|
||||
This looks a little weird but bare with me. Semi-old databases (post-2.x but pre-4.x) can have UTF8 Tables and Columns, but contain Latin1 data.
|
||||
We need to try and sniff this out and apply a different type of conversion, because in this instance you cannot just extract
|
||||
as BINARY and then insert into the source. Instead, you need to convert to latin1, then to binary, then to utf8. But you can't
|
||||
do that on actual UTF8 data because it will then truncate multibyte characters so we need to compare the conversion against what
|
||||
is stored.
|
||||
Mind = blown.
|
||||
*/
|
||||
$collation = 'utf8_unicode_ci';
|
||||
|
||||
if ( $tableCharSet == 'utf8mb4' OR $columnCollation == 'utf8mb4_unicode_ci' )
|
||||
$using = 'utf8';
|
||||
if ( $columnCollation == 'utf8mb4_unicode_ci' )
|
||||
{
|
||||
$collation = 'utf8mb4_unicode_ci';
|
||||
$using = 'utf8mb4';
|
||||
}
|
||||
|
||||
$source_charset = 'latin1';
|
||||
@@ -336,7 +342,7 @@ class Convert
|
||||
$source_charset = SOURCE_DB_CHARSET;
|
||||
}
|
||||
|
||||
$select[] = 'CASE WHEN STRCMP(CONVERT(CONVERT(CONVERT(`' . $col . '` USING ' . $source_charset . ') USING BINARY) USING utf8), `' . $col . '` COLLATE ' . $collation . ') = 0 THEN CONVERT(`' . $col . '` USING BINARY) ELSE CONVERT(CONVERT(CONVERT(`' . $col . '` USING ' . $source_charset . ') USING BINARY) USING utf8) END';
|
||||
$select[] = 'CASE WHEN STRCMP(CONVERT(CONVERT(CONVERT(`' . $col . '` USING ' . $source_charset . ') USING BINARY) USING ' . $using . '), `' . $col . '` COLLATE ' . $collation . ') = 0 THEN CONVERT(`' . $col . '` USING BINARY) ELSE CONVERT(CONVERT(CONVERT(`' . $col . '` USING ' . $source_charset . ') USING BINARY) USING ' . $using . ') END';
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -452,6 +458,7 @@ class Convert
|
||||
{
|
||||
if ( isset( $row[ $col ] ) AND ! empty( $row[ $col ] ) )
|
||||
{
|
||||
$currentCharSet = ( defined( 'FORCE_CONVERT_CHARSET' ) AND FORCE_CONVERT_CHARSET != NULL ) ? FORCE_CONVERT_CHARSET : $currentCharSet;
|
||||
if ( static::isSerialized( $row[ $col ] ) AND \IPSUtf8\Text\Charset::i()->needsConverting( $row[ $col ], $currentCharSet, 'UTF-8' ) )
|
||||
{
|
||||
/* Store a copy in case it doesn't work */
|
||||
@@ -513,10 +520,13 @@ class Convert
|
||||
$tmp = $row; // Use a copy so the reference in array_walk doesn't overwrite the row to be inserted
|
||||
|
||||
/* Make sure it's an associative array as topic_views only has 1 column so DB driver returns an indexed array only */
|
||||
array_walk_recursive( $tmp, function( &$input, $key )
|
||||
if ( is_array( $tmp ) )
|
||||
{
|
||||
$input = htmlentities( (string) $input, ENT_QUOTES | ENT_IGNORE, 'utf-8', false );
|
||||
} );
|
||||
array_walk_recursive( $tmp, function( &$input, $key )
|
||||
{
|
||||
$input = htmlentities( (string) $input, ENT_QUOTES | ENT_IGNORE, 'utf-8', false );
|
||||
} );
|
||||
}
|
||||
|
||||
$batchBytes += mb_strlen( @json_encode( $tmp ), '8bit');
|
||||
unset( $tmp );
|
||||
@@ -649,13 +659,14 @@ class Convert
|
||||
{
|
||||
static::log( "Pre inserts for InnoDB table " . $table['name'] );
|
||||
|
||||
/* This doesn't seem necessary
|
||||
try
|
||||
{
|
||||
static::$utf->query("SET autocommit=0");
|
||||
static::$utf->query("SET unique_checks=0");
|
||||
static::$utf->query("SET foreign_key_checks=0");
|
||||
}
|
||||
catch ( Exception $e ) { }
|
||||
catch ( Exception $e ) { }*/
|
||||
}
|
||||
}
|
||||
|
||||
@@ -683,14 +694,15 @@ class Convert
|
||||
else if ( \strtolower( static::$utfTableEngine ) === 'innodb' )
|
||||
{
|
||||
static::log( "Post inserts for InnoDB table " . $table['name'] );
|
||||
|
||||
|
||||
/* This doesn't seem necessary
|
||||
try
|
||||
{
|
||||
static::$utf->query("SET autocommit=1");
|
||||
static::$utf->query("SET unique_checks=1");
|
||||
static::$utf->query("SET foreign_key_checks=1");
|
||||
}
|
||||
catch ( Exception $e ) { }
|
||||
catch ( Exception $e ) { }*/
|
||||
}
|
||||
}
|
||||
|
||||
@@ -881,27 +893,45 @@ class Convert
|
||||
|
||||
while ( $stmt->fetch() === true )
|
||||
{
|
||||
if( mb_substr( $name, 0, 6 ) === 'x_utf_' )
|
||||
{
|
||||
$tables[] = $name;
|
||||
}
|
||||
$tables[] = $name;
|
||||
}
|
||||
|
||||
foreach( $tables as $name )
|
||||
{
|
||||
$tableNameNoPrefix = $name;
|
||||
$tableName = mb_substr( $name, mb_strlen( static::$utf->prefix ) );
|
||||
$isConvertedTable = ( mb_substr( $name, 0, 6 ) === 'x_utf_' );
|
||||
$isOrigTable = ( mb_substr( $name, 0, 5 ) === 'orig_' );
|
||||
|
||||
if ( $tableName !== 'convert_session' AND $tableName !== 'convert_session_tables' )
|
||||
if ( $isOrigTable )
|
||||
{
|
||||
if( static::$utf->checkForTable( $tableName ) )
|
||||
{
|
||||
static::$db->query( "DROP TABLE IF EXISTS `" . $origPrefix . $tableName . '`' );
|
||||
static::log( "DROP TABLE IF EXISTS `" . $origPrefix . $tableName . '`' );
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
static::$utf->query( "RENAME TABLE `" . $tableNameNoPrefix . '` TO `' . $origPrefix . $tableName . '`' );
|
||||
static::log( "RENAME TABLE `" . $tableNameNoPrefix . '` TO `' . $origPrefix . $tableName . '`' );
|
||||
if ( ! $isConvertedTable and $origPrefix )
|
||||
{
|
||||
$tableNameNoPrefix = mb_substr( $name, mb_strlen( $origPrefix ) );
|
||||
}
|
||||
|
||||
/* Rename original tables */
|
||||
if ( ( ! $isConvertedTable ) and mb_substr( $name, 0, mb_strlen( $origPrefix ) ) === $origPrefix )
|
||||
{
|
||||
static::$db->query( "RENAME TABLE `" . $origPrefix . $tableNameNoPrefix . '` TO `orig_' . $origPrefix . $tableNameNoPrefix . '`' );
|
||||
|
||||
static::log( "RENAME TABLE `" . $origPrefix . $tableNameNoPrefix . '` TO `orig_' . $origPrefix . $tableNameNoPrefix . '`' );
|
||||
}
|
||||
|
||||
/* Grab x_utf_ prefixed tables */
|
||||
if ( $isConvertedTable )
|
||||
{
|
||||
$tableName = mb_substr( $name, mb_strlen( static::$utf->prefix ) );
|
||||
|
||||
/* Rename the new one */
|
||||
if ( $tableName !== 'convert_session' AND $tableName !== 'convert_session_tables' )
|
||||
{
|
||||
static::$utf->query( "RENAME TABLE `" . $name . '` TO `' . $origPrefix . $tableName . '`' );
|
||||
|
||||
static::log( "RENAME TABLE `" . $name . '` TO `' . $origPrefix . $tableName . '`' );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -984,13 +1014,33 @@ class Convert
|
||||
/* We don't need to drop any indexes - just create a copy of the table and insert ignore */
|
||||
if ( $hasUnique === TRUE )
|
||||
{
|
||||
static::$utf->query( "RENAME TABLE `" . static::$db->prefix . $name . "` TO `" . static::$db->prefix . $name . "_temp`" );
|
||||
static::$utf->query( "CREATE TABLE `" . static::$db->prefix . $name . "` LIKE `" . static::$db->prefix . $name . "_temp`" );
|
||||
static::$utf->query( "INSERT IGNORE INTO `" . static::$db->prefix . $name . "` SELECT * FROM `" . static::$db->prefix . $name . "_temp`" );
|
||||
static::$utf->query( "DROP TABLE `" . static::$db->prefix . $name . "_temp`" );
|
||||
/* Let's try an alter first - below may not be necessary */
|
||||
try
|
||||
{
|
||||
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
|
||||
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
|
||||
}
|
||||
catch( \IPSUtf8\Db\Exception $e )
|
||||
{
|
||||
/* Did not work - try the long way */
|
||||
static::$utf->query( "RENAME TABLE `" . static::$db->prefix . $name . "` TO `" . static::$db->prefix . $name . "_temp`" );
|
||||
static::log( $name . " has Unique Index - temp table created from main." );
|
||||
static::$utf->query( "CREATE TABLE `" . static::$db->prefix . $name . "` LIKE `" . static::$db->prefix . $name . "_temp`" );
|
||||
static::log( $name . " created from temp table." );
|
||||
static::$utf->query( "REPLACE INTO `" . static::$db->prefix . $name . "` SELECT * FROM `" . static::$db->prefix . $name . "_temp`" );
|
||||
static::log( $name . " data copied to new table." );
|
||||
static::$utf->query( "DROP TABLE `" . static::$db->prefix . $name . "_temp`" );
|
||||
static::log( $name . "_temp dropped" );
|
||||
|
||||
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
|
||||
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
|
||||
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
|
||||
}
|
||||
|
||||
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
|
||||
|
||||
if ( count( $cols ) )
|
||||
{
|
||||
@@ -1565,7 +1615,7 @@ class Convert
|
||||
foreach( \IPSUtf8\Session::i()->tables as $table => $definition )
|
||||
{
|
||||
/* If the table itself is not utf8_unicode_ci, don't bother check columns */
|
||||
if ( $definition['collation'] AND !in_array( $definition['collation'], array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) )
|
||||
if ( $definition['definition']['collation'] AND !in_array( $definition['definition']['collation'], array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) )
|
||||
{
|
||||
static::$nonUtf8Collations[] = $table;
|
||||
continue;
|
||||
|
||||
Reference in new issue
Block a user