sydney/IPB
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 4.1.2

Browse Source
This commit is contained in:
Neo committed 2025-12-19 05:32:19 -08:00
1 parent 3ba79295ee
commit 28bd025b35
1131 files changed
+63658 -20579
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

admin/convertutf8/init.php
+1
View File
@@ -76,6 +76,7 @@ class IPSUtf8
'FORCE_CONVERT_CHARSET' => NULL,
'FORCE_CONVERT_METHOD' => NULL,
'SOURCE_DB_CHARSET' => NULL, /* If set, will force the source database connection to use this charset */
'UTF8_INSERT_ONLY' => FALSE, /* If true, and sql_charset is present and set to utf8 in conf_global.php, then the "fast" method will simply insert without converting. Useful for mixed collation databases where the utf8 tables are real utf8 */
);
}
admin/convertutf8/js/app.js
+1
View File
@@ -113,6 +113,7 @@ var ips = {};
data: data,
url: url,
timeout: ips.getSetting('maxExecTime') !== false ? parseInt( ips.getSetting('maxExecTime') * 1000 ) : 30000,
cache: false,
success: function( data, textStatus, jqXHR )
{
success( data, textStatus, jqXHR );
admin/convertutf8/modules/browser/archive.php
+9 -1
View File
@@ -32,7 +32,15 @@ class archive extends \IPSUtf8\Dispatcher\Controller
$this->welcome();
break;
case 'completed':
$this->completed();
/* Regardless of current status, if the archive table is not utf8 we need to convert it */
if ( \IPSUtf8\Convert\Archive::i()->getNonUtf8Tables( TRUE ) !== NULL )
{
$this->welcome();
}
else
{
$this->completed();
}
break;
}
}
admin/convertutf8/modules/browser/browser.php
+3 -3
View File
@@ -58,7 +58,7 @@ class browser extends \IPSUtf8\Dispatcher\Controller
$json = \IPSUtf8\Session::i()->json;
$percent = round( ($json['convertedCount'] / \IPSUtf8\Convert::i()->getTotalRowsToConvert()) * 100, 2 );
$msg = "Processing " . \IPSUtf8\Session::i()->current_table . ' (Total: ' . $percent . '%)';
$msg = "Processing " . \IPSUtf8\Session::i()->current_table . '<br>Row: ' . \IPSUtf8\Session::i()->current_row . ' of ' . \IPSUtf8\Session::i()->tables[ \IPSUtf8\Session::i()->current_table ]['count'] . ' - Total: ' . $percent . '%';
if ( ! \IPSUtf8\Request::i()->isAjax() )
{
@@ -169,14 +169,14 @@ class browser extends \IPSUtf8\Dispatcher\Controller
exit();
}
if ( \IPSUtf8\Convert::i()->databaseIsUtf8() )
if ( \IPSUtf8\Convert::i()->databaseIsUtf8() AND !count( \IPSUtf8\Convert::i()->getNonUtf8CollationTables() ) )
{
\IPSUtf8\Output\Browser::i()->error( 'The database tables are UTF-8, collations are correct and there is nothing to convert. You can <strong><a href="../upgrade">proceed with the upgrade</strong>.' );
exit();
}
$json = \IPSUtf8\Session::i()->json;
$percent = ( \IPSUtf8\Convert::i()->getTotalRowsToConvert() ) ? round( ($json['convertedCount'] / \IPSUtf8\Convert::i()->getTotalRowsToConvert() ) * 100, 2 ) : 100;
$percent = ( \IPSUtf8\Convert::i()->getTotalRowsToConvert() ) ? sprintf ("%6.2f", round( ($json['convertedCount'] / \IPSUtf8\Convert::i()->getTotalRowsToConvert() ) * 100, 2 ) ) : 100;
$isUtf8 = (boolean) ( \IPSUtf8\Convert::i()->databaseIsUtf8() );
$processing = (boolean) ( \IPSUtf8\Session::i()->status === 'processing' );
admin/convertutf8/readme.html
+1 -1
View File
@@ -55,7 +55,7 @@
</p>
<p><a class="btn btn-lg btn-info" href="index.php?controller=browser">Start Web Converter</a></p>
<div class="footer">
<p>&copy; 2015 Invision Power Services, Inc.</p>
<p>&copy; <script type="text/javascript">document.write( new Date().getFullYear() );</script><noscript>2015</noscript> Invision Power Services, Inc.</p>
</div>
</div>
<script src="js/jquery.min.js"></script>
admin/convertutf8/system/Convert/Convert.php
+88 -38
View File
@@ -16,7 +16,7 @@ namespace IPSUtf8;
*/
class Convert
{
const VERSION_ID = '1.1.17';
const VERSION_ID = '1.1.18';
/**
* @brief Data Store
@@ -301,9 +301,13 @@ class Convert
/* Even if the current charset is set to UTF-8, if the Table Charset is latin1, we need to use this method */
/* ... but only if the actual column isn't using utf8 */
$columnCollation = $schematic['definition']['columns'][$col]['collation'];
if ( \IPSUtf8\Session::i()->current_charset != 'utf-8' OR ( $tableCharSet != 'utf8' AND ! in_array( $columnCollation, array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) ) OR ( $tableCharSet == 'utf8' AND mb_substr( $columnCollation, 0, 6 ) == 'latin1' ) )
if ( defined( 'UTF8_INSERT_ONLY' ) AND UTF8_INSERT_ONLY === TRUE AND $tableCharSet == 'utf8' )
{
if ( \IPSUtf8\Session::i()->current_charset == 'iso-8859-1' AND $tableCharSet == 'latin1' )
$select[] = 'CAST( `' . $col . '` AS BINARY )';
}
else if ( \IPSUtf8\Session::i()->current_charset != 'utf-8' OR ( $tableCharSet != 'utf8' AND ! in_array( $columnCollation, array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) ) OR ( $tableCharSet == 'utf8' AND mb_substr( $columnCollation, 0, 5 ) == 'latin' ) )
{
if ( \IPSUtf8\Session::i()->current_charset == 'iso-8859-1' OR mb_substr( $columnCollation, 0, 5 ) == 'latin' )
{
/* Casting as binary truncates non-latin characters, so if we are using real Latin1, just convert */
$select[] = 'CONVERT( `' . $col . '` USING utf8 )';
@@ -317,17 +321,19 @@ class Convert
else
{
/**
* @note This looks a little weird but bare with me. Semi-old databases (post-2.x but pre-4.x) can have UTF8 Tables and Columns, but contain Latin1 data.
* We need to try and sniff this out and apply a different type of conversion, because in this instance you cannot just extract
* as BINARY and then insert into the source. Instead, you need to convert to latin1, then to binary, then to utf8. But you can't
* do that on actual UTF8 data because it will then truncate multibyte characters so we need to compare the conversion against what
* is stored.
*/
This looks a little weird but bare with me. Semi-old databases (post-2.x but pre-4.x) can have UTF8 Tables and Columns, but contain Latin1 data.
We need to try and sniff this out and apply a different type of conversion, because in this instance you cannot just extract
as BINARY and then insert into the source. Instead, you need to convert to latin1, then to binary, then to utf8. But you can't
do that on actual UTF8 data because it will then truncate multibyte characters so we need to compare the conversion against what
is stored.
Mind = blown.
*/
$collation = 'utf8_unicode_ci';
if ( $tableCharSet == 'utf8mb4' OR $columnCollation == 'utf8mb4_unicode_ci' )
$using = 'utf8';
if ( $columnCollation == 'utf8mb4_unicode_ci' )
{
$collation = 'utf8mb4_unicode_ci';
$using = 'utf8mb4';
}
$source_charset = 'latin1';
@@ -336,7 +342,7 @@ class Convert
$source_charset = SOURCE_DB_CHARSET;
}
$select[] = 'CASE WHEN STRCMP(CONVERT(CONVERT(CONVERT(`' . $col . '` USING ' . $source_charset . ') USING BINARY) USING utf8), `' . $col . '` COLLATE ' . $collation . ') = 0 THEN CONVERT(`' . $col . '` USING BINARY) ELSE CONVERT(CONVERT(CONVERT(`' . $col . '` USING ' . $source_charset . ') USING BINARY) USING utf8) END';
$select[] = 'CASE WHEN STRCMP(CONVERT(CONVERT(CONVERT(`' . $col . '` USING ' . $source_charset . ') USING BINARY) USING ' . $using . '), `' . $col . '` COLLATE ' . $collation . ') = 0 THEN CONVERT(`' . $col . '` USING BINARY) ELSE CONVERT(CONVERT(CONVERT(`' . $col . '` USING ' . $source_charset . ') USING BINARY) USING ' . $using . ') END';
}
}
else
@@ -452,6 +458,7 @@ class Convert
{
if ( isset( $row[ $col ] ) AND ! empty( $row[ $col ] ) )
{
$currentCharSet = ( defined( 'FORCE_CONVERT_CHARSET' ) AND FORCE_CONVERT_CHARSET != NULL ) ? FORCE_CONVERT_CHARSET : $currentCharSet;
if ( static::isSerialized( $row[ $col ] ) AND \IPSUtf8\Text\Charset::i()->needsConverting( $row[ $col ], $currentCharSet, 'UTF-8' ) )
{
/* Store a copy in case it doesn't work */
@@ -513,10 +520,13 @@ class Convert
$tmp = $row; // Use a copy so the reference in array_walk doesn't overwrite the row to be inserted
/* Make sure it's an associative array as topic_views only has 1 column so DB driver returns an indexed array only */
array_walk_recursive( $tmp, function( &$input, $key )
if ( is_array( $tmp ) )
{
$input = htmlentities( (string) $input, ENT_QUOTES | ENT_IGNORE, 'utf-8', false );
} );
array_walk_recursive( $tmp, function( &$input, $key )
{
$input = htmlentities( (string) $input, ENT_QUOTES | ENT_IGNORE, 'utf-8', false );
} );
}
$batchBytes += mb_strlen( @json_encode( $tmp ), '8bit');
unset( $tmp );
@@ -649,13 +659,14 @@ class Convert
{
static::log( "Pre inserts for InnoDB table " . $table['name'] );
/* This doesn't seem necessary
try
{
static::$utf->query("SET autocommit=0");
static::$utf->query("SET unique_checks=0");
static::$utf->query("SET foreign_key_checks=0");
}
catch ( Exception $e ) { }
catch ( Exception $e ) { }*/
}
}
@@ -683,14 +694,15 @@ class Convert
else if ( \strtolower( static::$utfTableEngine ) === 'innodb' )
{
static::log( "Post inserts for InnoDB table " . $table['name'] );
/* This doesn't seem necessary
try
{
static::$utf->query("SET autocommit=1");
static::$utf->query("SET unique_checks=1");
static::$utf->query("SET foreign_key_checks=1");
}
catch ( Exception $e ) { }
catch ( Exception $e ) { }*/
}
}
@@ -881,27 +893,45 @@ class Convert
while ( $stmt->fetch() === true )
{
if( mb_substr( $name, 0, 6 ) === 'x_utf_' )
{
$tables[] = $name;
}
$tables[] = $name;
}
foreach( $tables as $name )
{
$tableNameNoPrefix = $name;
$tableName = mb_substr( $name, mb_strlen( static::$utf->prefix ) );
$isConvertedTable = ( mb_substr( $name, 0, 6 ) === 'x_utf_' );
$isOrigTable = ( mb_substr( $name, 0, 5 ) === 'orig_' );
if ( $tableName !== 'convert_session' AND $tableName !== 'convert_session_tables' )
if ( $isOrigTable )
{
if( static::$utf->checkForTable( $tableName ) )
{
static::$db->query( "DROP TABLE IF EXISTS `" . $origPrefix . $tableName . '`' );
static::log( "DROP TABLE IF EXISTS `" . $origPrefix . $tableName . '`' );
}
continue;
}
static::$utf->query( "RENAME TABLE `" . $tableNameNoPrefix . '` TO `' . $origPrefix . $tableName . '`' );
static::log( "RENAME TABLE `" . $tableNameNoPrefix . '` TO `' . $origPrefix . $tableName . '`' );
if ( ! $isConvertedTable and $origPrefix )
{
$tableNameNoPrefix = mb_substr( $name, mb_strlen( $origPrefix ) );
}
/* Rename original tables */
if ( ( ! $isConvertedTable ) and mb_substr( $name, 0, mb_strlen( $origPrefix ) ) === $origPrefix )
{
static::$db->query( "RENAME TABLE `" . $origPrefix . $tableNameNoPrefix . '` TO `orig_' . $origPrefix . $tableNameNoPrefix . '`' );
static::log( "RENAME TABLE `" . $origPrefix . $tableNameNoPrefix . '` TO `orig_' . $origPrefix . $tableNameNoPrefix . '`' );
}
/* Grab x_utf_ prefixed tables */
if ( $isConvertedTable )
{
$tableName = mb_substr( $name, mb_strlen( static::$utf->prefix ) );
/* Rename the new one */
if ( $tableName !== 'convert_session' AND $tableName !== 'convert_session_tables' )
{
static::$utf->query( "RENAME TABLE `" . $name . '` TO `' . $origPrefix . $tableName . '`' );
static::log( "RENAME TABLE `" . $name . '` TO `' . $origPrefix . $tableName . '`' );
}
}
}
@@ -984,13 +1014,33 @@ class Convert
/* We don't need to drop any indexes - just create a copy of the table and insert ignore */
if ( $hasUnique === TRUE )
{
static::$utf->query( "RENAME TABLE `" . static::$db->prefix . $name . "` TO `" . static::$db->prefix . $name . "_temp`" );
static::$utf->query( "CREATE TABLE `" . static::$db->prefix . $name . "` LIKE `" . static::$db->prefix . $name . "_temp`" );
static::$utf->query( "INSERT IGNORE INTO `" . static::$db->prefix . $name . "` SELECT * FROM `" . static::$db->prefix . $name . "_temp`" );
static::$utf->query( "DROP TABLE `" . static::$db->prefix . $name . "_temp`" );
/* Let's try an alter first - below may not be necessary */
try
{
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
}
catch( \IPSUtf8\Db\Exception $e )
{
/* Did not work - try the long way */
static::$utf->query( "RENAME TABLE `" . static::$db->prefix . $name . "` TO `" . static::$db->prefix . $name . "_temp`" );
static::log( $name . " has Unique Index - temp table created from main." );
static::$utf->query( "CREATE TABLE `" . static::$db->prefix . $name . "` LIKE `" . static::$db->prefix . $name . "_temp`" );
static::log( $name . " created from temp table." );
static::$utf->query( "REPLACE INTO `" . static::$db->prefix . $name . "` SELECT * FROM `" . static::$db->prefix . $name . "_temp`" );
static::log( $name . " data copied to new table." );
static::$utf->query( "DROP TABLE `" . static::$db->prefix . $name . "_temp`" );
static::log( $name . "_temp dropped" );
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
}
}
else
{
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
}
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
if ( count( $cols ) )
{
@@ -1565,7 +1615,7 @@ class Convert
foreach( \IPSUtf8\Session::i()->tables as $table => $definition )
{
/* If the table itself is not utf8_unicode_ci, don't bother check columns */
if ( $definition['collation'] AND !in_array( $definition['collation'], array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) )
if ( $definition['definition']['collation'] AND !in_array( $definition['definition']['collation'], array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) )
{
static::$nonUtf8Collations[] = $table;
continue;
admin/convertutf8/system/Db/Db.php
+29 -7
View File
@@ -61,6 +61,11 @@ class Db extends \mysqli
* @brief Multiton Store
*/
protected static $multitons;
/**
* @brief Return Query
*/
public $returnQuery = FALSE;
/**
* Get instance
@@ -111,7 +116,30 @@ class Db extends \mysqli
{
throw new \IPSUtf8\Db\Exception( $error, self::$multitons[ $identifier ]->connect_errno );
}
/* If we succeeded, set the charset */
if ( mb_substr( $identifier, 0, 4 ) == 'utf8' )
{
/* UTF8MB4? */
if ( isset( $connectionSettings['sql_utf8mb4'] ) and $connectionSettings['sql_utf8mb4'] )
{
self::$multitons[ $identifier ]->charset = 'utf8mb4';
self::$multitons[ $identifier ]->collation = 'utf8mb4_unicode_ci';
self::$multitons[ $identifier ]->binaryCollation = 'utf8mb4_bin';
}
/* If we succeeded, set the charset */
if ( !self::$multitons[ $identifier ]->set_charset( self::$multitons[ $identifier ]->charset ) )
{
/* Fallback to UTF8 */
self::$multitons[ $identifier ]->charset = 'utf8';
self::$multitons[ $identifier ]->collation = 'utf8_unicode_ci';
self::$multitons[ $identifier ]->binaryCollation = 'utf8_bin';
self::$multitons[ $identifier ]->set_charset( 'utf8' );
}
}
/* Set the prefix */
if ( isset( $connectionSettings['sql_tbl_prefix'] ) )
{
@@ -148,12 +176,6 @@ class Db extends \mysqli
*/
public $prefix = '';
/**
* @brief Return the query instead of executing it
* @note Only designed to work with methods that call query() vs prepared statements
*/
public $returnQuery = FALSE;
/**
* Run a query
*
admin/convertutf8/system/Output/Browser.php
+1
View File
@@ -103,6 +103,7 @@ class Browser extends \IPSUtf8\Output
/* Flush and exit */
@ob_end_flush();
@flush();
exit;
}