Version 5.0.6

applications/convert/sources/Software/Core/Vanilla.php

@@ -14,7 +14,9 @@ namespace IPS\convert\Software\Core;
 
 use DateTimeZone;
 use erusev\Parsedown;
+use IPS\Application;
 use IPS\Content\Search\Index;
+use IPS\convert\App;
 use IPS\convert\Software;
 use IPS\Data\Cache;
 use IPS\Data\Store;
@@ -45,6 +47,81 @@ if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
  */
 class Vanilla extends Software
 {
+	/**
+	 * Constructor
+	 *
+	 * @param	App	$app	The application to reference for database and other information.
+	 * @param	bool				$needDB	Establish a DB connection
+	 * @return	void
+	 * @throws	InvalidArgumentException
+	 */
+	public function __construct( App $app, bool $needDB=TRUE )
+	{
+		parent::__construct( $app, $needDB );
+
+		/* Check for temp table */
+		if ( $needDB )
+		{
+			if( !Db::i()->checkForTable( 'convert_vanilla_temp' ) )
+			{
+				Db::i()->createTable( array(
+					'name'		=> 'convert_vanilla_temp',
+					'columns'	=> array(
+						'id' => array(
+							'name'			=> 'id',
+							'type'			=> 'int',
+							'length'		=> 10,
+							'allow_null'	=> false,
+							'auto_increment'	=> true
+						),
+						'media_id' => array(
+							'name'			=> 'media_id',
+							'type'			=> 'int',
+							'length'		=> 10,
+							'allow_null'	=> false,
+							'auto_increment'	=> false
+						),
+						'link_type' => array(
+							'name'			=> 'link_type',
+							'type'			=> 'varchar',
+							'length'		=> 255,
+							'allow_null'	=> true,
+							'default'		=> null
+						),
+						'content_id' => array(
+							'name'			=> 'content_id',
+							'type'			=> 'int',
+							'length'		=> 10,
+							'allow_null'	=> false,
+							'default'		=> 0
+						),
+						'post_id' => array(
+							'name'			=> 'post_id',
+							'type'			=> 'int',
+							'length'		=> 10,
+							'allow_null'	=> false,
+							'default'		=> 0
+						),
+					),
+					'indexes'	=> array(
+						'PRIMARY' => array(
+							'type'		=> 'primary',
+							'name'		=> 'PRIMARY',
+							'columns'	=> array( 'id', 'media_id' ),
+							'length'	=> array( NULL )
+						),
+						'lookup' => array(
+							'type'		=> 'index',
+							'name'		=> 'lookup',
+							'columns'	=> array( 'media_id', 'link_type' ),
+							'length'	=> array( NULL, NULL )
+						),
+					)
+				)	);
+			}
+		}
+	}
+
 	/**
 	 * Software Name
 	 *
@@ -198,6 +275,10 @@ class Vanilla extends Software
 			'convertPrivateMessageReplies'	=> array(
 				'table'		=> 'ConversationMessage',
 				'where'		=> NULL,
+			),
+			'convertAttachments'	=> array(
+				'table'		=> 'Media',
+				'where'		=> [ "ForeignTable IN ('embed')" ]
 			)
 		);
 	}
@@ -316,6 +397,9 @@ class Vanilla extends Software
 		/* Attachments */
 		Task::queue( 'core', 'RebuildAttachmentThumbnails', array( 'app' => $this->app->app_id ), 1, array( 'app' ) );
 
+		/* Remove temp table */
+		Db::i()->dropTable('convert_vanilla_temp');
+
 		return array( "f_search_index_rebuild", "f_clear_caches", "f_rebuild_pms", "f_signatures_rebuild", "f_rebuild_attachments" );
 	}
 
@@ -327,12 +411,13 @@ class Vanilla extends Software
 	/**
 	 * Pre-process content for the Invision Community text parser
 	 *
-	 * @param	string			$post	The post
-	 * @param	string|null		$className Classname passed by post-conversion rebuild
-	 * @param	int|null		$contentId ID passed by post-conversion rebuild
+	 * @param	string			The post
+	 * @param	string|null		Content Classname passed by post-conversion rebuild
+	 * @param	int|null		Content ID passed by post-conversion rebuild
+	 * @param	App|null		App object if available
 	 * @return	string			The converted post
 	 */
-	public static function fixPostData( string $post, ?string $className=null, ?int $contentId=null ): string
+	public static function fixPostData( string $post, ?string $className=null, ?int $contentId=null, ?App $app=null ): string
 	{
 		/* Is this Markdown? */
 		if( mb_substr( $post, 0, 15 ) == '<!--Markdown-->' )
@@ -347,67 +432,57 @@ class Vanilla extends Software
 			$post = str_replace( '<!--Markdown-->', '', $post );
 			$post = static::$_parseDown->text( $post );
 		}
-		elseif( mb_substr( $post, 0, 3 ) == '[{"' ) // Probably a delta
+		/* If rich, we've already processed it, but need to skip the legacy formatting */
+		elseif( mb_substr( $post, 0, 11 ) == '<!--Rich-->' )
 		{
-			$postData = str_replace( [ "\r\n","\n" ], '', $post );
-			$postData = json_decode( $postData, TRUE );
+			$post = str_replace( '<!--Rich-->', '', $post );
+		}
+		/* Deltas still tha haven't been previously processed */
+		elseif( mb_substr( $post, 0, 3 ) == '[{"' )
+		{
+			$post = static::processQuill( $post, $app );
+		}
+		/* LEGACY CONVERSIONS BELOW FOR OLDER CONTENT */
+		else
+		{
+			/* Mentions */
+			$matches = [];
+			preg_match_all( '/@("|&quot;)([^@"]+)("|&quot;)/i', $post, $matches );
 
-			$post = '';
-			foreach( $postData as $data )
+			if( count( $matches ) )
 			{
-				if( is_array( $data['insert'] ) )
+				foreach( $matches[0] as $key => $tag )
 				{
-					switch ( $data['insert']['embed-external']['data']['embedType'] )
+					$member = Member::load( $matches[2][ $key ], 'name' );
+					if( !$member->member_id )
 					{
-						case 'quote':
-							$post .= '[quote name="' . $data['insert']['embed-external']['data']['insertUser']['name'] . '" timestamp="' . strtotime( $data['insert']['embed-external']['data']['dateInserted'] ) . '"]' . $data['insert']['embed-external']['data']['body'] . "[/quote]";
-							break;
+						continue;
 					}
-				}
-				else
-				{
-					$post .= $data['insert'] . "<br>";
+
+					$post = str_replace( $tag, "[mention={$member->member_id}]{$member->name}[/mention]", $post );
 				}
 			}
+
+			/**
+			 * Vanilla has a quotes plugin that seems to have changed formats quite often. - We'll try to match as many formats as we can
+			 */
+			$post = preg_replace( '/(?:<blockquote\s+(?:class=\"(?:User)?Quote\")?(?:\s+rel=\"(?:[^\"]+)\")?>)(?:\s+)?<div class=\"QuoteAuthor\">([^\"]+)<\/div>(?:\s+)?<div class=\"QuoteText\">(?:<p>)?(.*?)(?:<\/p>)?<\/div>(?:\s+)?<\/blockquote>/i',
+				'[quote name="$1"]$2[/quote]' ,
+				$post );
+
+			$post = preg_replace( '/(?:<blockquote\s+(?:class=\"Quote (?:User)?Quote\")?(?:\s+rel=\"(?:[^\"]+)\")?>)(?:\s+)?<div class=\"QuoteText\">(?:<p>)?(.*?)(?:<\/p>)?<\/div>(?:\s+)?<\/blockquote>/i',
+				'[quote]$1[/quote]' ,
+				$post );
+
+			$post = preg_replace( '/<blockquote\s+rel=\"([^\"]+)\"?>(.*?)<\/blockquote>/i',
+				'[quote name="$1"]$2[/quote]' ,
+				$post );
+
+			$post = preg_replace( '/<blockquote>(.*?)<\/blockquote>/ims',
+				'[quote]$1[/quote]' ,
+				$post );
 		}
 
-		/* Mentions */
-		$matches = [];
-		preg_match_all( '/@("|&quot;)([^@"]+)("|&quot;)/i', $post, $matches );
-
-		if( count( $matches ) )
-		{
-			foreach( $matches[0] as $key => $tag )
-			{
-				$member = Member::load( $matches[2][ $key ], 'name' );
-				if( !$member->member_id )
-				{
-					continue;
-				}
-
-				$post = str_replace( $tag, "[mention={$member->member_id}]{$member->name}[/mention]", $post );
-			}
-		}
-
-		/**
-		 * Vanilla has a quotes plugin that seems to have changed formats quite often. - We'll try to match as many formats as we can
-		 */
-		$post = preg_replace( '/(?:<blockquote\s+(?:class=\"(?:User)?Quote\")?(?:\s+rel=\"(?:[^\"]+)\")?>)(?:\s+)?<div class=\"QuoteAuthor\">([^\"]+)<\/div>(?:\s+)?<div class=\"QuoteText\">(?:<p>)?(.*?)(?:<\/p>)?<\/div>(?:\s+)?<\/blockquote>/i',
-			'[quote name="$1"]$2[/quote]' ,
-			$post );
-
-		$post = preg_replace( '/(?:<blockquote\s+(?:class=\"Quote (?:User)?Quote\")?(?:\s+rel=\"(?:[^\"]+)\")?>)(?:\s+)?<div class=\"QuoteText\">(?:<p>)?(.*?)(?:<\/p>)?<\/div>(?:\s+)?<\/blockquote>/i',
-			'[quote]$1[/quote]' ,
-			$post );
-
-		$post = preg_replace( '/<blockquote\s+rel=\"([^\"]+)\"?>(.*?)<\/blockquote>/i',
-			'[quote name="$1"]$2[/quote]' ,
-			$post );
-
-		$post = preg_replace( '/<blockquote>(.*?)<\/blockquote>/ims',
-			'[quote]$1[/quote]' ,
-			$post );
-
 		return $post;
 	}
 
@@ -458,11 +533,15 @@ class Vanilla extends Software
 
 		$uploadsPath = $this->app->_session['more_info']['convertMembers']['attach_location'];
 
-		$users = $this->fetch( array( 'User', 'u' ), 'u.UserID', array( "u.Deleted<>?", 1 ), 'u.*, ur.RoleID' );
-		$users->join( array( 'UserRole', 'ur' ), 'ur.UserID=u.UserID' );
+		$users = $this->fetch( array( 'User', 'u' ), 'u.UserID', array( "u.Deleted<>?", 1 ) );
 
 		$data = iterator_to_array( $users );
-		$this->app->preCacheLinks( $data, [ 'core_groups' => 'RoleID' ] );
+		$userIds = array_column( $data, 'UserID' );
+		$roles = array_fill_keys( $userIds, [] );
+		foreach( $this->db->select( '*', 'UserRole', [ $this->db->in( 'UserID', $userIds ) ] ) as $role )
+		{
+			$roles[ $role['UserID'] ][] = $role['RoleID'];
+		}
 
 		foreach( $data AS $row )
 		{
@@ -482,22 +561,54 @@ class Vanilla extends Software
 			/* Work out banned stuff */
 			$tempBan = ( $row['Banned'] == 1 ) ? -1 : 0;
 
+			/* Get IP */
+			try
+			{
+				$ipAddress = ( $row['InsertIPAddress'] AND !str_contains( $row['InsertIPAddress'], '.' ) ) ? long2ip( hexdec( bin2hex( $row['InsertIPAddress'] ) ) ) : $row['InsertIPAddress'];
+			}
+			catch( \ErrorException $e )
+			{
+				$ipAddress = '';
+			}
+
+			if( !empty( $row['RoleID'] ) )
+			{
+				array_unshift( $roles[ $row['UserID'] ], $row['RoleID'] );
+			}
+
+			$allRoles = array_unique( $roles[ $row['UserID'] ] );
+			$primaryRole = NULL;
+			$secondaryRoles = [];
+
+			/* Find a primary role. */
+			if( count( $allRoles ) )
+			{
+				$primaryRole = array_shift( $allRoles );
+			}
+
+			/* Any remaining roles will be secondary */
+			if( count( $allRoles ) )
+			{
+				$secondaryRoles = $allRoles;
+			}
+
 			/* Array of basic data */
 			$info = array(
 				'member_id'       => $row['UserID'],
 				'email'           => $row['Email'],
 				'name'            => $row['Name'],
 				'password'        => $row['Password'],
-				'member_group_id'    => $row['RoleID'] ?: NULL,
+				'member_group_id'    => $primaryRole,
+				'mgroup_others'		=> $secondaryRoles,
 				'joined'          => static::mysqlToDateTime( $row['DateInserted'] ),
-				'ip_address'      => $row['InsertIPAddress'],
+				'ip_address'      => $ipAddress,
 				'temp_ban'        => $tempBan,
 				'bday_day'        => $bdayDay,
 				'bday_month'      => $bdayMonth,
 				'bday_year'       => $bdayYear,
 				'msg_count_new'   => (int) $row['CountUnreadDiscussions'],
 				'msg_count_total' => (int) $row['CountDiscussions'],
-				'last_visit'      => static::mysqlToDateTime( $row['DateLastActive'] ),
+				'last_visit'      => $row['DateLastActive'] ? static::mysqlToDateTime( $row['DateLastActive'] ) : NULL,
 				'timezone'        => !empty( $row['HourOffset'] )
 					? (int) $row['HourOffset']
 					: new DateTimeZone(
@@ -570,10 +681,10 @@ class Vanilla extends Software
 			{
 				$maps[ $r['UserID'] ] = array(
 					'map_user_id'           => $r['UserID'],
-					'map_read_time'         => static::mysqlToDateTime( $r['DateLastViewed'] ),
+					'map_read_time'         => $r['DateLastViewed'] ? static::mysqlToDateTime( $r['DateLastViewed'] ) : NULL,
 					//'map_folder_id'        => TODO: Create a Bookmarks folder for "Bookmarked" conversations?
 					'map_user_active'       => $r['Deleted'] == 0 ? 1 : 0,
-					'map_last_topic_reply'  => static::mysqlToDateTime( $r['DateConversationUpdated'] )
+					'map_last_topic_reply'  => $r['DateConversationUpdated'] ? static::mysqlToDateTime( $r['DateConversationUpdated'] ) : NULL
 				);
 			}
 
@@ -602,6 +713,20 @@ class Vanilla extends Software
 			{
 				$row['Body'] = '<!--Markdown-->' . $row['Body'];
 			}
+			elseif( $row['Format'] == 'Rich' )
+			{
+				$row['Body'] = static::processQuill( $row['Body'], $this->app, 'core_message_posts', $row['ConversationID'], $row['MessageID'] );
+			}
+
+			/* Get IP */
+			try
+			{
+				$ipAddress = ( $row['InsertIPAddress'] AND !str_contains( $row['InsertIPAddress'], '.' ) ) ? long2ip( hexdec( bin2hex( $row['InsertIPAddress'] ) ) ) : $row['InsertIPAddress'];
+			}
+			catch( \ErrorException $e )
+			{
+				$ipAddress = '';
+			}
 
 			$libraryClass->convertPrivateMessageReply( array(
 				'msg_id'            => $row['MessageID'],
@@ -609,13 +734,106 @@ class Vanilla extends Software
 				'msg_date'          => static::mysqlToDateTime( $row['DateInserted'] ),
 				'msg_post'          => $row['Body'],
 				'msg_author_id'     => $row['InsertUserID'],
-				'msg_ip_address'    => $row['InsertIPAddress']
+				'msg_ip_address'    => $ipAddress
 			) );
 			
 			$libraryClass->setLastKeyValue( $row['MessageID'] );
 		}
 	}
 
+	/**
+	 * @brief   temporarily store post content
+	 */
+	protected array $_postContent = array();
+
+	/**
+	 * Convert attachments
+	 *
+	 * @return	void
+	 */
+	public function convertAttachments(): void
+	{
+		$libraryClass = $this->getLibrary();
+		$libraryClass::setKey( 'MediaID' );
+		$this->_postContent = [];
+
+		foreach( $this->fetch( 'Media', 'MediaID', [ $this->db->in( 'ForeignTable', [ 'embed' ] ) ] ) AS $row )
+		{
+			$mediaRows = iterator_to_array( 
+				\IPS\Db::i()->select( 
+					'*', 
+					'convert_vanilla_temp', 
+					[ 'media_id=? AND ' . \IPS\Db::i()->in( 'link_type', [ 'core_message_posts' ] ), $row['MediaID'] ] 
+			) );
+
+			// Nothing to do, attachment must be in another content type
+			if( !count( $mediaRows ) )
+			{
+				$libraryClass->setLastKeyValue( $row['MediaID'] );
+				continue;
+			}
+
+			$map = [
+				'id1'	=> $mediaRows[0]['content_id'],
+				'id2'	=> $mediaRows[0]['post_id']
+			];
+			
+			$map['id1_type']		= 'core_message_topics';
+			$map['id2_type']		= 'core_message_posts';
+			$map['location_key']	= 'core_Messaging';
+			$map['id1_from_parent'] = FALSE;
+			$map['id2_from_parent'] = FALSE;
+			$map['id3_skip_link'] = TRUE;
+
+			/* File extension */
+			$ext = explode( '.', $row['Path'] );
+			$ext = array_pop( $ext );
+
+			$info = array(
+				'attach_id'			=> $row['MediaID'],
+				'attach_file'		=> $row['Name'],
+				'attach_date'		=> static::mysqlToDateTime( $row['DateInserted'] ),
+				'attach_member_id'	=> $row['InsertUserID'],
+				'attach_hits'		=> 0,
+				'attach_ext'		=> $ext,
+				'attach_filesize'	=> $row['Size'],
+			);
+
+			$uploadPath = static::parseMediaLocation( $row['Path'], $this->app->_session['more_info']['convertMembers']['attach_location'] );
+
+			$attachId = $libraryClass->convertAttachment( $info, $map, $uploadPath );
+
+			/* Do some re-jiggery on the post itself to make sure attachment displays */
+			if ( $attachId !== FALSE )
+			{
+				foreach( $mediaRows as $mediaRow )
+				{
+					try
+					{
+						$pid = $this->app->getLink( $map['id2'], 'core_message_posts' );
+
+						if( !isset( $this->_postContent[ $pid ] ) )
+						{
+							$this->_postContent[ $pid ] = Db::i()->select( 'msg_post', 'core_message_posts', array( "msg_id=?", $pid ) )->first();
+						}
+
+						$this->_postContent[ $pid ] = str_replace( "[ATTACH={$row['MediaID']}]", '[attachment=' . $attachId . ':name]', $this->_postContent[ $pid ] );
+					}
+					catch( UnderflowException|OutOfRangeException $e ) {}
+				}
+			}
+
+			unset( $mediaRows );
+			$libraryClass->setLastKeyValue( $row['MediaID'] );
+		}
+
+		/* Do the updates */
+		foreach( $this->_postContent as $pid => $content )
+		{
+			Db::i()->update( 'core_message_posts', array( 'msg_post' => $content ), array( 'msg_id=?', $pid ) );
+		}
+	}
+
 	/**
 	 * Process a login
 	 *
@@ -642,4 +860,38 @@ class Vanilla extends Software
 			return FALSE;
 		}
 	}
+
+	public static function processQuill( string $postJson, ?App $app=null, ?string $type=null, ?int $contentId=null, ?int $commentId=null ): string
+	{
+		/* Need to wrap so it's an expected format */
+		$post = '{"ops":' . $postJson . '}';
+
+		if ( strpos( $post, '"header":' ) !== false )
+		{
+			$post = preg_replace('/{"header":{"level":(\d+),"ref":"([^"]*)"}}/', '{"header":$1}', $post );
+		}
+
+		$data = json_decode( $post, true );
+
+		if ( !isset( $data['ops'][ count( $data['ops'] ) - 1 ]['insert'] ) OR is_array( $data['ops'][ count( $data['ops'] ) - 1 ]['insert'] ) OR \substr( $data['ops'][ count( $data['ops'] ) - 1 ]['insert'], -1 ) !== "\n" )
+		{
+			$data['ops'][] = [ "insert" => "\n" ];
+		}
+
+		/* Get the autoloader for the Quill library */
+		require_once Application::load( 'convert' )->getApplicationPath() . '/sources/Tools/Quill/vendor/autoload.php';
+
+		$lexer = new \nadar\quill\Lexer( $data );
+		$lexer->registerListener( new \IPS\convert\Tools\Vanilla\QuillListeners\Emoji() );
+		$lexer->registerListener( new \IPS\convert\Tools\Vanilla\QuillListeners\EmbedExternal( $type, $contentId, $commentId ) );
+		$lexer->registerListener( new \IPS\convert\Tools\Vanilla\QuillListeners\Mention( $app ) );
+		try
+		{
+			return '<!--Rich-->' . $lexer->render();
+		}
+		catch( \ErrorException $e )
+		{
+			return $postJson;
+		}
+	}
 }