Version 4.1.19

system/File/Amazon.php

@@ -2,9 +2,9 @@
 /**
  * @brief		File Handler: Amazon S3
  * @author		<a href='http://www.invisionpower.com'>Invision Power Services, Inc.</a>
- * @copyright	(c) 2001 - SVN_YYYY Invision Power Services, Inc.
+ * @copyright	(c) 2001 - 2016 Invision Power Services, Inc.
  * @license		http://www.invisionpower.com/legal/standards/
- * @package		IPS Social Suite
+ * @package		IPS Community Suite
  * @since		12 Jul 2013
  * @version		SVN_VERSION_NUMBER
  */
@@ -44,6 +44,7 @@ class _Amazon extends \IPS\File
 		
 		return array(
 			'bucket'		=> 'Text',
+			'endpoint'		=> array( 'type' => 'Text', 'default' => 's3.amazonaws.com' ),
 			'bucket_path'   => 'Text',
 			'access_key'	=> 'Text',
 			'secret_key'	=> 'Text',
@@ -53,6 +54,11 @@ class _Amazon extends \IPS\File
 			'custom_url' => array( 'type' => 'Text', 'default' => '' )
 		);
 	}
+
+	/**
+	 * @brief	Temporarily stored endpoint - when testing settings, we may need to update it automagically
+	 */
+	protected static $updatedEndpoint	= NULL;
 	
 	/**
 	 * Test Settings
@@ -88,6 +94,12 @@ class _Amazon extends \IPS\File
 		{
 			throw new \DomainException( \IPS\Member::loggedIn()->language()->addToStack( 'file_storage_test_error_amazon_d403', FALSE, array( 'sprintf' => array( $values['bucket'], $response->httpResponseCode ) ) ) );
 		}
+
+		if( static::$updatedEndpoint !== NULL )
+		{
+			$values['endpoint']	= static::$updatedEndpoint;
+			static::$updatedEndpoint = NULL;
+		}
 		
 		if ( ! $values['toggle'] )
 		{
@@ -168,22 +180,48 @@ class _Amazon extends \IPS\File
 	{
 		if ( $this->storageExtension and ! array_key_exists( $this->storageExtension, static::$gzipExtensions ) )
 		{
-			$bits     = explode( '_', $this->storageExtension );
-			$class    = '\IPS\\' . $bits[0] . '\extensions\core\FileStorage\\' . $bits[1];
-		
-			if ( isset( $class::$storeGzipExtensions ) and is_array( $class::$storeGzipExtensions ) and count( $class::$storeGzipExtensions ) )
+			static::$gzipExtensions[ $this->storageExtension ] = array();
+
+			if( mb_strpos( $this->storageExtension, '_' ) !== FALSE )
 			{
-				static::$gzipExtensions[ $this->storageExtension ] = $class::$storeGzipExtensions;
-			}
-			else
-			{
-				static::$gzipExtensions[ $this->storageExtension ] = array();
+				$bits     = explode( '_', $this->storageExtension );
+				$class    = '\IPS\\' . $bits[0] . '\extensions\core\FileStorage\\' . $bits[1];
+			
+				if ( isset( $class::$storeGzipExtensions ) and is_array( $class::$storeGzipExtensions ) and count( $class::$storeGzipExtensions ) )
+				{
+					static::$gzipExtensions[ $this->storageExtension ] = $class::$storeGzipExtensions;
+				}
 			}
 		}
 		
 		return $this->storageExtension ? static::$gzipExtensions[ $this->storageExtension ] : array();
 	}
 	
+	/**
+	 * Is this a private file?
+	 * This means that it is PUT with bucket owner read-only permissions which means it needs a signed URL to download
+	 *
+	 * @return boolean
+	 */
+	public function isPrivate()
+	{
+		if ( $this->storageExtension )
+		{
+			if ( mb_strpos( $this->storageExtension, '_' ) !== FALSE )
+			{
+				$bits     = explode( '_', $this->storageExtension );
+				$class    = '\IPS\\' . $bits[0] . '\extensions\core\FileStorage\\' . $bits[1];
+			
+				if ( isset( $class::$isPrivate ) )
+				{
+					return $class::$isPrivate;
+				}
+			}
+		}
+		
+		return false;
+	}
+	
 	/**
 	 * AWS does not gzip content when serving it, so if we want gzip compressed JS and CSS, we need to store a copy ourselves.
 	 *
@@ -202,7 +240,7 @@ class _Amazon extends \IPS\File
 	 */
 	public function baseUrl()
 	{
-		return preg_replace( '#^http(s)?://#', '//', ( empty( $this->configuration['custom_url'] ) ) ? rtrim( static::buildBaseUrl( $this->configuration ), '/' ) : $this->configuration['custom_url'] );
+		return preg_replace( '#^http(s)?://#', '//', rtrim( ( empty( $this->configuration['custom_url'] ) ) ? static::buildBaseUrl( $this->configuration ) : $this->configuration['custom_url'], '/' ) );
 	}
 	
 	/**
@@ -229,21 +267,42 @@ class _Amazon extends \IPS\File
 	public function save()
 	{
 		$this->container = trim( $this->container, '/' );
-		$this->url = $this->baseUrl() . "/{$this->container}/{$this->filename}";
-		$response  = static::makeRequest( "{$this->container}/{$this->filename}", 'PUT', $this->configuration, $this->configurationId, (string) $this->contents() );
+		$this->url = $this->baseUrl() . ( $this->container ? "/{$this->container}" : '' ) . "/{$this->filename}";
+		$path = $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}";
+		$response  = static::makeRequest( $path, 'PUT', $this->configuration, $this->configurationId, (string) $this->contents(), $this->storageExtension, FALSE, $this->isPrivate() );
 
 		if ( $response->httpResponseCode != 200 )
 		{
+			try
+			{
+				$xml = $response->decodeXml();
+				if ( isset( $xml->Code ) and isset( $xml->Message ) )
+				{
+					\IPS\Log::log( $xml->Code . ' ' . $xml->Message . ' ' . $path, 'Amazon' );
+				}
+			}
+			catch( \Exception $e ) { }
+			
 			throw new \RuntimeException('COULD_NOT_SAVE_FILE');
 		}
 		
 		/* Write the gzip version */
 		if ( $this->needsGzipVersion() )
 		{
-			$response  = static::makeRequest( "{$this->container}/{$this->filename}.gz", 'PUT', $this->configuration, $this->configurationId, gzencode( (string) $this->contents() ) );
+			$response  = static::makeRequest( "{$path}.gz", 'PUT', $this->configuration, $this->configurationId, gzencode( (string) $this->contents() ) );
 	
 			if ( $response->httpResponseCode != 200 )
 			{
+				try
+				{
+					$xml = $response->decodeXml();
+					if ( isset( $xml->Code ) and isset( $xml->Message ) )
+					{
+						\IPS\Log::log( $xml->Code . ' ' . $xml->Message . ' ' . $path, 'Amazon' );
+					}
+				}
+				catch( \Exception $e ) { }
+			
 				throw new \RuntimeException('COULD_NOT_SAVE_FILE');
 			}
 		}
@@ -259,7 +318,7 @@ class _Amazon extends \IPS\File
 	{
 		if ( $this->contents === NULL or $refresh === TRUE )
 		{
-			$this->contents = (string) static::makeRequest( "{$this->container}/{$this->filename}", 'GET', $this->configuration, $this->configurationId );
+			$this->contents = (string) static::makeRequest( $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}", 'GET', $this->configuration, $this->configurationId );
 		}
 		return $this->contents;
 	}
@@ -272,25 +331,36 @@ class _Amazon extends \IPS\File
 	public function delete()
 	{
 		$this->container = trim( $this->container, '/' );
+		$path = $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}";
 
-		$response = static::makeRequest( "{$this->container}/{$this->filename}", 'DELETE', $this->configuration, $this->configurationId );
-		
-		if ( $response->httpResponseCode == 204 )
+		try
 		{
-			/* Got a gzip version? */
-			if ( $this->needsGzipVersion() )
+			$response = static::makeRequest( $path, 'DELETE', $this->configuration, $this->configurationId );
+			
+			/* Log deletion request */
+			$this->log( "file_deletion", 'delete', NULL, 'log' );
+
+			if ( $response->httpResponseCode == 204 )
 			{
-				static::makeRequest( "{$this->container}/{$this->filename}.gz", 'DELETE', $this->configuration, $this->configurationId );
+				/* Got a gzip version? */
+				if ( $this->needsGzipVersion() )
+				{
+					static::makeRequest( "{$path}.gz", 'DELETE', $this->configuration, $this->configurationId );
+				}
+				
+				/* Ok */
+				return;
 			}
 			
-			/* Ok */
-			return;
+			if ( $response->httpResponseCode != 200 )
+			{
+				$this->log( 'COULD_NOT_DELETE_FILE', 'delete', array( $response->httpResponseCode, $response->httpResponseText ), 'error' );
+			}
 		}
-		
-		if ( $response->httpResponseCode != 200 )
+		catch( \IPS\Http\Request\Exception $e )
 		{
-			$this->log( 'COULD_NOT_DELETE_FILE', 'delete', array( $response->httpResponseCode, $response->httpResponseText ), 'error' );
-			throw new \RuntimeException('COULD_NOT_DELETE_FILE');
+			/* If there was a problem deleting the file, don't stop code execution just because of that */
+			$this->log( 'HTTP_ERROR_DELETE_FILE', 'delete', array( $e->getCode(), $e->getMessage() ), 'error' );
 		}
 	}
 	
@@ -324,8 +394,19 @@ class _Amazon extends \IPS\File
 			
 			static::makeRequest( $result->Key, 'DELETE', $this->configuration, $this->configurationId );
 		}
+
+		/* Log deletion request */
+		$realContainer = $this->container;
+		$this->container = $container;
+		$this->log( "container_deletion", 'delete', NULL, 'log' );
+		$this->container = $realContainer;
 	}
 
+	/**
+	 * @brief Cached filesize
+	 */
+	protected $_cachedFilesize	= NULL;
+
 	/**
 	 * Get filesize (in bytes)
 	 *
@@ -333,20 +414,67 @@ class _Amazon extends \IPS\File
 	 */
 	public function filesize()
 	{
+		if( $this->_cachedFilesize !== NULL )
+		{
+			return $this->_cachedFilesize;
+		}
+
 		$this->container = trim( $this->container, '/' );
 
-		$response = static::makeRequest( "{$this->container}/{$this->filename}", 'HEAD', $this->configuration, $this->configurationId );
+		$response = static::makeRequest( $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}", 'HEAD', $this->configuration, $this->configurationId );
 
 		if ( $response->httpResponseCode != 200 OR !isset( $response->httpHeaders['Content-Length'] ) )
 		{
 			return parent::filesize();
 		}
 		
-		return $response->httpHeaders['Content-Length'];
+		$this->_cachedFilesize = $response->httpHeaders['Content-Length'];
+
+		return $this->_cachedFilesize;
 	}
 
 	/* !Amazon Utility Methods */
 	
+	/**
+	 * Generate a temporary download URL the user can be redirected to
+	 *
+	 * @param	$validForSeconds	int	The number of seconds the link should be valid for
+	 * @return	\IPS\Http\Url
+	 */
+	public function generateTemporaryDownloadUrl( $validForSeconds = 1200 )
+	{
+		$fileUrl = ( $this->container ? ( rawurlencode( $this->container ) . '/' . rawurlencode( $this->filename ) ) : rawurlencode( $this->filename ) );
+		$url = \IPS\Http\Url::external( static::buildBaseUrl( $this->configuration ) . $fileUrl );
+		
+		$headers = array();
+		$queryString = array(
+			'X-Amz-Expires'					=> $validForSeconds,
+			'response-content-disposition'	=> 'attachment; filename*=UTF-8\'\'' . rawurlencode( $this->originalFilename )
+		);
+		$signature = $this->signature( $this->configuration, 'GET', $fileUrl, $headers, $queryString, NULL, TRUE );
+		$queryString['X-Amz-Signature'] = $signature;
+		
+		$url = $url->setQueryString( $queryString );
+		
+		$response = $url->request()->head();
+		if ( $response->httpResponseCode == 400 )
+		{	
+			$xml = $url->request()->get()->decodeXml();
+			if ( !isset( $xml->Region ) )
+			{
+				throw new \RuntimeException;
+			}
+			
+			$this->configuration['region'] = (string) $xml->Region;
+			\IPS\Db::i()->update( 'core_file_storage', array( 'configuration' => json_encode( $this->configuration ) ), array( 'id=?', $this->configurationId ) );
+			unset( \IPS\Data\Store::i()->storageConfigurations );
+			return $this->generateTemporaryDownloadUrl( $validForSeconds );
+			
+		}
+				
+		return $url;
+	}
+	
 	/**
 	 * Sign and make request
 	 *
@@ -355,13 +483,19 @@ class _Amazon extends \IPS\File
 	 * @param	array 		$configuration		The configuration for this instance
 	 * @param	int			$configurationId	The configuration ID
 	 * @param	string|null	$content			The content to send
+	 * @param	bool		$skipExtraChecks	Skips the endpoint check (to prevent infinite looping)
 	 * @return	\IPS\Http\Response
 	 * @throws	\IPS\Http\Request\Exception
 	 */
-	protected static function makeRequest( $uri, $verb, $configuration, $configurationId, $content=NULL )
+	protected static function makeRequest( $uri, $verb, $configuration, $configurationId, $content=NULL, $storageExtension=NULL, $skipExtraChecks=FALSE, $isPrivate=false )
 	{
+		/* Amazon requires filename characters to be properly encoded - let's urlencode the filename here */
+		$uriPieces	= explode( '/', $uri );
+		$filename	= array_pop( $uriPieces );
+		$uri		= ltrim( implode( '/', $uriPieces ) . '/' . rawurlencode( $filename ), '/' );
+		
 		/* Build a request */
-		$request = \IPS\Http\Url::external( static::buildBaseUrl( $configuration ) . $uri )->request();
+		$request = \IPS\Http\Url::external( static::buildBaseUrl( $configuration ) . $uri )->request( \IPS\LONG_REQUEST_TIMEOUT, NULL, FALSE ); # Amazon will send a 301 header code, but no Location header, if we need to try another endpoint
 		
 		/* When using virtual hosted–style buckets with SSL, the SSL wild card certificate only matches buckets that do not contain periods. To work around this, use HTTP or write your own certificate verification logic. @link http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html */
 		if ( \IPS\Request::i()->isSecure() and mb_strstr( $configuration['bucket'], '.' ) )
@@ -369,75 +503,220 @@ class _Amazon extends \IPS\File
 			$request->sslCheck( FALSE );
 		}
 		
-		$date    = date('r');
-		/* Make sure the file has the correct mime type, even if it is gzipped */
+		/* Set headers. Make sure the file has the correct mime type, even if it is gzipped */
 		$mimeUri = ( mb_substr( $uri, -3 ) === '.gz' ) ? mb_substr( $uri, 0, -3 ) : $uri;
-		$headers = array( 'Date' => $date, 'Content-Type' => \IPS\File::getMimeType( $mimeUri ), 'Content-MD5' => base64_encode( md5( $content, TRUE ) ), 'x-amz-acl' => 'public-read' );
-		
+		$headers = array(
+			'Content-Type'	=> \IPS\File::getMimeType( $mimeUri ),
+			'Content-MD5'	=> base64_encode( md5( $content, TRUE ) ),
+			'X-Amz-Acl'		=> ( $isPrivate ? 'bucket-owner-read' : 'public-read' )
+		);
 		if ( $mimeUri !== $uri )
 		{
 			$headers['Content-Encoding'] = 'gzip';
 		}
-		
+
+		/* If uploading a file, need to specify length and cache control */
 		if( mb_strtoupper( $verb ) === 'PUT' )
 		{
 			$headers['Content-Length']	= \strlen( $content );
+
+			$cacheSeconds = 3600;
+			$ext = mb_substr( $mimeUri, ( mb_strrpos( $mimeUri, '.' ) + 1 ) );
+			
+			if ( preg_match( "/\." . preg_quote( $ext, '/' ) . "\.([a-zA-Z0-9]{32})\." . preg_quote( $ext, '/' ) . "$/", $mimeUri ) )
+			{
+				/* This is an obfuscated file, so it will change each time the file is updated so it is safe to cache for longer */
+				$cacheSeconds = 3600 * 24 * 365;
+			}
+
+			/* Custom Cache-Control */
+			if( $storageExtension !== NULL AND mb_strpos( $storageExtension, '_' ) !== FALSE )
+			{
+				$bits     = explode( '_', $storageExtension );
+				$class    = '\IPS\\' . $bits[0] . '\extensions\core\FileStorage\\' . $bits[1];
+
+				if ( isset( $class::$cacheControlTtl ) and $class::$cacheControlTtl )
+				{
+					$cacheSeconds = $class::$cacheControlTtl;
+				}
+			}
+			
+			$headers['Cache-Control'] = 'max-age=' . $cacheSeconds;
+			$headers['Expires'] = date( 'r', time() + $cacheSeconds );
 		}
 		
-		$cacheSeconds = 3600;
-		$ext = mb_substr( $mimeUri, ( mb_strrpos( $mimeUri, '.' ) + 1 ) );
-		
-		if ( preg_match( "/\." . preg_quote( $ext ) . "\.([a-zA-Z0-9]{32})\." . preg_quote( $ext ) . "$/", $mimeUri ) )
-		{
-			/* This is an obfuscated file, so it will change each time the file is updated so it is safe to cache for longer */
-			$cacheSeconds = 3600 * 24 * 365;
-		}
-		
-		$headers['Cache-Control'] = 'max-age=' . $cacheSeconds;
-		$headers['Expires'] = date( 'r', time() + $cacheSeconds );
-		
 		/* We need to strip query string parameters for the signature, but not always (e.g. a subresource such as ?acl needs to be included and multi-
 			object delete requests must include the query string params).  Let the callee decide to do this or not. */
 		if( isset( $configuration['_strip_querystring'] ) AND $configuration['_strip_querystring'] === TRUE )
 		{
 			$uri = preg_replace( "/^(.*?)\?.*$/", "$1", $uri );
 		}
-	
-		$string = implode( "\n", array(
-			mb_strtoupper( $verb ),
-			$headers['Content-MD5'],
-			$headers['Content-Type'],
-			$date,
-			"x-amz-acl:{$headers['x-amz-acl']}",
-			"/{$configuration['bucket']}" . ( ( isset( $configuration['bucket_path'] ) AND ! empty( $configuration['bucket_path'] ) ) ? '/' . trim( $configuration['bucket_path'], '/' ) . '/' : '/' ) . $uri
-		) );
 		
-		/* Build a signature */
-		$signature = base64_encode( hash_hmac( 'sha1', $string, $configuration['secret_key'], true ) );
-
 		/* Sign the request */
-		$headers['Authorization'] = "AWS {$configuration['access_key']}:{$signature}";
+		$queryString = array();
+		$authorization = static::signature( $configuration, $verb, $uri, $headers, $queryString, $content );
+		$headers['Authorization'] = $authorization;
+		unset( $headers['Host'] );
 		$request->setHeaders( $headers );
-
+		
 		/* Make the request */
 		$verb = mb_strtolower( $verb );
-
 		$response = $request->$verb( $content );
 
-		/* Need a different region? */
-		if ( $response->httpResponseCode == 307 AND $configurationId )
+		/* If we are skipping extra checks, return response now */
+		if( $skipExtraChecks )
+		{
+			return $response;
+		}
+		
+		/* Change endpoint if necessary */
+		if ( $response->httpResponseCode == 301 )
 		{
 			$xml = $response->decodeXml();
-			$configuration['region'] = mb_substr( $xml->Endpoint, mb_strlen( $configuration['bucket'] ) + 1, -mb_strlen( '.amazonaws.com' ) );
-			\IPS\Db::i()->update( 'core_file_storage', array( 'configuration' => json_encode( $configuration ) ), array( 'id=?', $configurationId ) );
-			unset( \IPS\Data\Store::i()->storageConfigurations );
-			return static::makeRequest( $uri, $verb, $configuration, $configurationId, $content );
+			if ( isset( $xml->Endpoint ) )
+			{
+				/* We have an endpoint, but if we called s3.amazonaws.com then it might be wrong. Try to detect the correct one. */
+				$configuration['endpoint'] = 's3-us-west-1.amazonaws.com';
+
+				$endpointResponse	= static::makeRequest( $uri, $verb, $configuration, $configurationId, $content, NULL, TRUE );
+				$update				= FALSE;
+
+				/* If the response code is 200, we got lucky and that's our endpoint */
+				if( $endpointResponse->httpResponseCode == 200 )
+				{
+					$update = TRUE;
+				}
+				/* If it's a 301 response, we should be able to pull out the correct endpoint now */
+				elseif( $endpointResponse->httpResponseCode == 301 )
+				{
+					$xml = $endpointResponse->decodeXml();
+					if ( isset( $xml->Endpoint ) )
+					{
+						/* Strip out the bucket from the endpoint */
+						$configuration['endpoint'] = preg_replace( '/^' . preg_quote( $configuration['bucket'], '/' ) . '\./', '', (string) $xml->Endpoint );
+						$update = TRUE;
+					}
+				}
+
+				/* If we need to update, do it now and return the result */
+				if( $update === TRUE )
+				{
+					static::$updatedEndpoint	= $configuration['endpoint'];
+
+					if ( $configurationId )
+					{
+						\IPS\Db::i()->update( 'core_file_storage', array( 'configuration' => json_encode( $configuration ) ), array( "id=?", $configurationId ) );
+						unset( \IPS\Data\Store::i()->storageConfigurations );
+					}
+				}
+
+				return static::makeRequest( $uri, $verb, $configuration, $configurationId, $content );
+			}
+		}
+		
+		/* Change region if necessary */
+		if ( $response->httpResponseCode == 400 )
+		{
+			try
+			{
+				$xml = $response->decodeXml();
+				if ( isset( $xml->Region ) )
+				{
+					$configuration['region'] = (string) $xml->Region;
+					if ( $configurationId )
+					{
+						\IPS\Db::i()->update( 'core_file_storage', array( 'configuration' => json_encode( $configuration ) ), array( 'id=?', $configurationId ) );
+						unset( \IPS\Data\Store::i()->storageConfigurations );
+					}
+					return static::makeRequest( $uri, $verb, $configuration, $configurationId, $content );
+				}
+			}
+			catch ( \Exception $e ) { }
 		}
 
 		/* Return */
-		return $response;
+		return $response;		
 	}
+	
+	/**
+	 * Generate a v4 signature
+	 *
+	 * @param	array 		$configuration					The configuration for this instance
+	 * @param	string		$verb							The HTTP verb that will be used in the request
+	 * @param	string		$uri							The URI (relative to the bucket)
+	 * @param	array		$header							The request headers as an array
+	 * @param	array		$queryString					The query string as an array
+	 * @param	string|null	$content						The content to send
+	 * @param	bool		$signatureIsForQueryString		If true, signature will be generated for query string. If false, header.
+	 * @return	string
+	 */
+	protected static function signature( $configuration, $verb, $uri, &$headers = array(), &$queryString = array(), $content = NULL, $signatureIsForQueryString=FALSE )
+	{
+		/* Work out some basic stuff */
+		$time = time();
+		$region = ( isset( $configuration['region'] ) ? $configuration['region'] : 'us-east-1' );
+		$scope = date( 'Ymd', $time ) . '/' . $region . '/s3/aws4_request';
+		$contentSha256 = ( $signatureIsForQueryString and !$content ) ? 'UNSIGNED-PAYLOAD' : hash( 'sha256', $content );
+				
+		/* Figure out the canonical headers and query string */
+		$headers['Host'] = ( isset( $configuration['endpoint'] ) ? $configuration['endpoint'] : "s3.amazonaws.com" );
+		if ( $signatureIsForQueryString )
+		{
+			$queryString['X-Amz-Algorithm'] = 'AWS4-HMAC-SHA256';
+			$queryString['X-Amz-Content-Sha256'] = $contentSha256;
+			$queryString['X-Amz-Credential'] = $configuration['access_key'] . '/' . $scope;
+			$queryString['X-Amz-Date'] = gmdate( 'Ymd', $time ) . 'T' . gmdate( 'His', $time ) . 'Z';
+			$queryString['X-Amz-SignedHeaders'] = implode( ';', array_map( 'mb_strtolower', array_keys( $headers ) ) );
+		}
+		else
+		{
+			$headers['X-Amz-Content-Sha256'] = $contentSha256;
+			$headers['X-Amz-Date'] = gmdate( 'Ymd', $time ) . 'T' . gmdate( 'His', $time ) . 'Z';
+		}
+		ksort( $queryString );
+		ksort( $headers );
+		$canonicalHeadersAsString = '';
+		foreach ( $headers as $k => $v )
+		{
+			$canonicalHeadersAsString .= mb_strtolower( $k ) . ':' . trim( $v ) . "\n";
+		}
 
+		/* Task 1: Create a Canonical Request */
+		$canonicalRequest = implode( "\n", array(
+			mb_strtoupper( $verb ),
+			'/' . $configuration['bucket'] . static::bucketPath( $configuration ) . '/' . ltrim( $uri, '/' ),
+			http_build_query( $queryString, '', '&', PHP_QUERY_RFC3986 ),
+			$canonicalHeadersAsString,
+			implode( ';', array_map( 'mb_strtolower', array_keys( $headers ) ) ),
+			$contentSha256
+		) );
+						
+		/* Task 2: Create a String to Sign */
+		$stringToSign = implode( "\n", array(
+			'AWS4-HMAC-SHA256',
+			gmdate( 'Ymd', $time ) . 'T' . gmdate( 'His', $time ) . 'Z',
+			$scope,
+			hash( 'sha256', $canonicalRequest )
+		) );
+						
+		/* Task 3: Calculate Signature */
+		$dateKey = hash_hmac( 'sha256', date( 'Ymd', $time ), 'AWS4' . $configuration['secret_key'], true );
+		$dateRegionKey = hash_hmac( 'sha256', $region, $dateKey, true );
+		$dateRegionServiceKey = hash_hmac( 'sha256', 's3', $dateRegionKey, true );
+		$signingKey = hash_hmac( 'sha256', 'aws4_request', $dateRegionServiceKey, true );
+		
+		/* Return */
+		$signature = hash_hmac( 'sha256', $stringToSign, $signingKey );
+		if ( $signatureIsForQueryString )
+		{
+			return $signature;
+		}
+		else
+		{
+			return "AWS4-HMAC-SHA256 Credential={$configuration['access_key']}/{$scope},SignedHeaders=" . implode( ';', array_map( 'mb_strtolower', array_keys( $headers ) ) ) . ",Signature={$signature}";
+		}
+	}
+	
 	/**
 	 * Build up the base Amazon URL
 	 * @param   array   $configuration  Configuration data
@@ -446,11 +725,29 @@ class _Amazon extends \IPS\File
 	public static function buildBaseUrl( $configuration )
 	{
 		return (
-			\IPS\Request::i()->isSecure() ? "https" : "http" )
-			. "://{$configuration['bucket']}"
-			. ( ( isset( $configuration['region'] ) AND ! empty( $configuration['region'] ) ) ? ".{$configuration['region']}" : '.s3' )
-			. ".amazonaws.com"
-			. ( ( isset( $configuration['bucket_path'] ) AND ! empty( $configuration['bucket_path'] ) ) ? '/' . trim( $configuration['bucket_path'], '/' ) . '/' : '/' );
+			\IPS\Request::i()->isSecure() ? "https" : "http" ) . "://"
+			. ( isset( $configuration['endpoint'] ) ? $configuration['endpoint'] : "s3.amazonaws.com" )
+			. "/{$configuration['bucket']}"
+			. static::bucketPath( $configuration )
+			. '/';
+	}
+	
+	/**
+	 * Get bucket path
+	 *
+	 * @param   array   $configuration  Configuration data
+	 * @return	string
+	 */
+	protected static function bucketPath( $configuration )
+	{
+		if ( isset( $configuration['bucket_path'] ) AND ! empty( $configuration['bucket_path'] ) )
+		{
+			$bucketPath = trim( $configuration['bucket_path'], '/' );
+			$bucketPath = rawurlencode( $bucketPath ); // The bucket path needs to be mostly url-encoded
+			$bucketPath = str_replace( '%2F', '/', $bucketPath ); // Except for slashes because it can be multiple-levels deep
+			return "/{$bucketPath}";
+		}
+		return '';
 	}
 
 	/**

system/File/Database.php

@@ -2,9 +2,9 @@
 /**
  * @brief		File Handler: Database
  * @author		<a href='http://www.invisionpower.com'>Invision Power Services, Inc.</a>
- * @copyright	(c) 2001 - SVN_YYYY Invision Power Services, Inc.
+ * @copyright	(c) 2001 - 2016 Invision Power Services, Inc.
  * @license		http://www.invisionpower.com/legal/standards/
- * @package		IPS Social Suite
+ * @package		IPS Community Suite
  * @since		06 May 2013
  * @version		SVN_VERSION_NUMBER
  */
@@ -89,6 +89,16 @@ class _Database extends \IPS\File
 		return (string) \IPS\Http\Url::internal( "applications/core/interface/file/index.php?file=", 'none', NULL, array(), \IPS\Http\Url::PROTOCOL_RELATIVE );
 	}
 	
+	/**
+	 * Encode the file name for the fully qualified URL
+	 *
+	 * @return string
+	 */
+	public function encodeFileUrl( $filename )
+	{
+		return \IPS\Http\Url::encodeComponent( \IPS\Http\Url::COMPONENT_QUERY_VALUE, $filename );
+	}
+	
 	/**
 	 * Load File Data
 	 *
@@ -160,7 +170,7 @@ class _Database extends \IPS\File
 			'container'	=> $this->container
 		) );
 
-		$this->url = new \IPS\Http\Url( $this->fullyQualifiedUrl( "{$this->container}/{$this->filename}" ) );
+		$this->url = \IPS\Http\Url::internal( "applications/core/interface/file/index.php?file={$this->container}/{$this->filename}", 'none', NULL, array(), \IPS\Http\Url::PROTOCOL_RELATIVE );
 	}
 	
 	/**
@@ -171,6 +181,9 @@ class _Database extends \IPS\File
 	public function delete()
 	{
 		\IPS\Db::i()->delete( 'core_files', array( 'filename=? AND container=?', $this->filename, $this->container ) );
+
+		/* Log deletion request */
+		$this->log( "file_deletion", 'delete', NULL, 'log' );
 	}
 	
 	/**
@@ -182,6 +195,12 @@ class _Database extends \IPS\File
 	public function deleteContainer( $container )
 	{
 		\IPS\Db::i()->delete( 'core_files', array( 'container=?', $container ) );
+
+		/* Log deletion request */
+		$realContainer = $this->container;
+		$this->container = $container;
+		$this->log( "container_deletion", 'delete', NULL, 'log' );
+		$this->container = $realContainer;
 	}
 
 	/**
@@ -203,7 +222,7 @@ class _Database extends \IPS\File
 		$checked	= 0;
 
 		/* Loop over files */
-		foreach( \IPS\DB::i()->select( '*', 'core_files', array(), 'id ASC', array( $fileIndex, 100 ) ) as $file )
+		foreach( \IPS\Db::i()->select( '*', 'core_files', array(), 'id ASC', array( $fileIndex, 100 ) ) as $file )
 		{
 			$checked++;
 

system/File/File.php

@@ -2,9 +2,9 @@
 /**
  * @brief		File Class
  * @author		<a href='http://www.invisionpower.com'>Invision Power Services, Inc.</a>
- * @copyright	(c) 2001 - SVN_YYYY Invision Power Services, Inc.
+ * @copyright	(c) 2001 - 2016 Invision Power Services, Inc.
  * @license		http://www.invisionpower.com/legal/standards/
- * @package		IPS Social Suite
+ * @package		IPS Community Suite
  * @since		19 Feb 2013
  * @version		SVN_VERSION_NUMBER
  */
@@ -52,12 +52,13 @@ abstract class _File
 	 * @brief	When moving files, do not delete the original immediately but log for later deletion
 	 */
 	const MOVE_DELETE_NOW = 2;
-	
+
 	/**
 	 * Get class
 	 *
 	 * @param	string|int		            $storageExtension	Storage extension or configuration ID
 	 * @param   string|\IPS\Http\Url|NULL  $url                 URL to validate storage configs with
+	 * @throws 	\RuntimeException
 	 * @return	\IPS\File
 	 */
 	public static function getClass( $storageExtension, $url=NULL )
@@ -133,7 +134,7 @@ abstract class _File
 	/**
 	 * Load storage configurations
 	 *
-	 * @return	void
+	 * @return	array
 	 */
 	public static function getConfigurations()
 	{
@@ -149,6 +150,8 @@ abstract class _File
 				\IPS\Data\Store::i()->storageConfigurations = static::$storageConfigurations;
 			}
 		}
+
+		return static::$storageConfigurations;
 	}
 
 	/**
@@ -181,10 +184,10 @@ abstract class _File
 		}
 		
 		$class->storageExtension = $storageExtension;
-		
+						
 		/* Image-specific stuff */
 		$ext = mb_substr( $filename, ( mb_strrpos( $filename, '.' ) + 1 ) );
-		if( in_array( $ext, \IPS\Image::$imageExtensions ) and ( !$isSafe or \IPS\Image::exifSupported() ) )
+		if( in_array( mb_strtolower( $ext ), \IPS\Image::$imageExtensions ) and ( !$isSafe or \IPS\Image::exifSupported() ) )
 		{
 			/* Get contents */
 			if( $data === NULL AND $filePath !== NULL )
@@ -199,7 +202,7 @@ abstract class _File
 			{
 				/* Try to just strip the EXIF */
 				$image = \IPS\Image::create( $data );
-				$image->resize( 100, 100 );
+				$image->resize( $image->width, $image->height );
 				$data = (string) $image;
 				
 				/* And if it still fails, throw an error */
@@ -295,7 +298,7 @@ abstract class _File
 				if( $callback !== NULL )
 				{
 					$contents	= file_get_contents( $file['tmp_name'] );
-					$contents	= call_user_func( $callback, $contents );
+					$contents	= call_user_func( $callback, $contents, $file['name'] );
 
 					$fileObjects[]	= static::create( $storageLocation, $file['name'], $contents, $container, FALSE, NULL, $obscure );
 				}
@@ -465,9 +468,16 @@ abstract class _File
 	 * @param	string					$storageExtension	Storage extension
 	 * @param	string|\IPS\Http|Url	$url				URL to file
 	 * @return	\IPS\File
+	 * @throws	\OutOfRangeException
 	 */
 	public static function get( $storageExtension, $url )
 	{
+		/* Don't allow something/../../../../etc/passwd */
+		if( mb_strpos( rawurldecode( $url ), '../' ) !== FALSE )
+		{
+			throw new \OutOfRangeException( 'INVALID_PATH' );
+		}
+
 		$class = static::getClass( $storageExtension, $url );
 		$class->storageExtension = $storageExtension;
 		$class->url = $url;
@@ -545,8 +555,6 @@ abstract class _File
 			
 			foreach( static::$storageConfigurations as $id => $data  )
 			{
-				$secondTest = NULL;
-				$thirdTest  = NULL;
 				$class      = static::getClass( $data['id'] );
 				
 				$urlRelative  = preg_replace( '#^http(s)?://#', '//', $url );
@@ -654,8 +662,8 @@ abstract class _File
 		else
 		{
 			$pointer	= 0;
-
-			while( $pointer < \strlen( $contents ) )
+			$contentsLength = \strlen( $contents );
+			while( $pointer < $contentsLength )
 			{
 				print \substr( $contents, $pointer, $throttle );
 				$pointer	+= $throttle;
@@ -685,7 +693,7 @@ abstract class _File
 			ob_end_clean();
 		}
 
-		if( $throttle === NULL AND !($start AND $length) AND function_exists('readfile') )
+		if( $throttle === NULL AND $start === NULL AND function_exists('readfile') )
 		{
 			readfile( $file );
 		}
@@ -695,7 +703,7 @@ abstract class _File
 			{
 				$read	= ( $throttle !== NULL ) ? $throttle : 4096;
 
-				if( $start AND $length )
+				if( $start !== NULL AND $length )
 				{
 					fseek( $fh, $start );
 
@@ -830,8 +838,13 @@ abstract class _File
 		{
 			$filename = mb_substr( $filename, 0, ( mb_strrpos( $filename, '.' ) ) ) . '_' . $ext;
 		}
+		
+		while( preg_match( '#\.(?!(' . implode( '|', static::$safeFileExtensions ) . '))([a-z0-9]{2,4})(\.|$)#i', $filename, $matches ) )
+		{
+			$filename = str_replace( '.' . $matches[2], '_' . $matches[2], $filename );
+		}
 
-		return str_replace( array( ' ', '#' ), '_', $filename ) . '.' . md5( uniqid() ) . ( ( $safe ) ? '.' . $ext : '' );
+		return str_replace( array( ' ', '#' ), '_', $filename ) . '.' . md5( uniqid( \IPS\SUITE_UNIQUE_KEY ) . mt_rand() ) . ( ( $safe ) ? '.' . $ext : '' );
 	}
 
 	/**
@@ -843,7 +856,6 @@ abstract class _File
 	protected function unObscureFilename( $filename )
 	{
 		$ext = mb_substr( $filename, ( mb_strrpos( $filename, '.' ) + 1 ) );
-		
 		if ( mb_strlen( $ext ) == 32 )
 		{
 			preg_match( '#(.*)_([A-Z0-9\-]{2,10})\.([A-Z0-9]{32})#i', $filename, $matches );
@@ -852,9 +864,19 @@ abstract class _File
 			{
 				return $matches[1] . '.' . $matches[2];
 			}
+			else
+			{
+				/* Fix foo.pdf.hash */
+				preg_match( '#(.*)\.([A-Z0-9\-]{2,10})\.([A-Z0-9]{32})#i', $filename, $matches );
+		
+				if ( isset( $matches[1] ) and isset( $matches[2] ) )
+				{
+					return $matches[1] . '.' . $matches[2];
+				}
+			}
 		}
-
-		if( preg_match( "/([a-zA-Z0-9]{32})\." . preg_quote( $ext ) . "$/", $filename ) )
+		
+		if( preg_match( "/\.([a-zA-Z0-9]{32})\." . preg_quote( $ext ) . "$/", $filename ) )
 		{
 			return mb_substr( $filename, 0, mb_strlen( $filename ) - ( 34 + mb_strlen( $ext ) ) );
 		}
@@ -881,7 +903,7 @@ abstract class _File
 		$exploded = explode( '/', $url );
 		$this->filename = array_pop( $exploded );		
 		$this->originalFilename = $this->unObscureFilename( $this->filename );
-		$this->url = new \IPS\Http\Url( $this->fullyQualifiedUrl( $this->url ) );
+		$this->url = \IPS\Http\Url::createFromString( $this->fullyQualifiedUrl( $this->url ), FALSE, TRUE );
 
 		/* Upon upgrade we don't rename every file, so we need to account for this */
 		if( mb_strpos( $this->originalFilename, '.' ) === FALSE )
@@ -933,7 +955,7 @@ abstract class _File
 	 */
 	public function __toString()
 	{
-		return $this->container . '/' . $this->filename;
+		return $this->container ? ( $this->container . '/' . $this->filename ) : $this->filename;
 	}
 	
 	/**
@@ -975,12 +997,22 @@ abstract class _File
 	{
 		if ( mb_substr( $url, 0, 4 ) !== 'http' AND mb_substr( $url, 0, 2 ) !== '//' )
 		{
-			$url = $this->baseUrl() . '/' . $url;
+			$url = $this->baseUrl() . '/' . $this->encodeFileUrl( $url );
 		}
 		
 		return $url;
 	}
 	
+	/**
+	 * Encode the file name for the fully qualified URL
+	 *
+	 * @return string
+	 */
+	public function encodeFileUrl( $filename )
+	{
+		return \IPS\Http\Url::encodeComponent( \IPS\Http\Url::COMPONENT_PATH, $filename );
+	}
+	
 	/**
 	 * Move file to a different storage location
 	 *
@@ -1094,12 +1126,15 @@ abstract class _File
 	 * Make into an attachment
 	 *
 	 * @param	string	$postKey	Post key
+	 * @param	\IPS\Member|NULL	Member who uploaded the attachment
 	 * @return	array
 	 * @throws	\DomainException
 	 */
-	public function makeAttachment( $postKey )
+	public function makeAttachment( $postKey, \IPS\Member $member = NULL )
 	{
-		$ext = mb_substr( $this->originalFilename, mb_strrpos( $this->originalFilename, '.' ) + 1 );
+		$ext = mb_substr( $this->originalFilename, mb_strrpos($this->originalFilename, '.') + 1 );
+
+		$memberId = ( $member and $member->member_id ) ? $member->member_id : 0;
 
 		$data = array(
 			'attach_ext'			=> $ext,
@@ -1112,7 +1147,7 @@ abstract class _File
 			'attach_hits'			=> 0,
 			'attach_date'			=> time(),
 			'attach_post_key'		=> $postKey,
-			'attach_member_id'		=> \IPS\Member::loggedIn()->member_id ?: 0,
+			'attach_member_id'		=> $memberId,
 			'attach_filesize'		=> $this->filesize(),
 			'attach_img_width'		=> 0,
 			'attach_img_height'		=> 0,
@@ -1184,7 +1219,7 @@ abstract class _File
 	}
 	
 	/**
-	 * Claim Attachments
+	 * Claim Attachments and clear autosave content
 	 *
 	 * @param	string		$autoSaveKey	Auto-save key
 	 * @param	int|NULL	$id1			ID 1	
@@ -1208,6 +1243,9 @@ abstract class _File
 				), array( 'temp=?', md5( $autoSaveKey . $lang->id ) ) );
 				
 				\IPS\Db::i()->update( 'core_attachments', array( 'attach_post_key' => '' ), array( 'attach_post_key=?', md5( $autoSaveKey . $lang->id . ':' . session_id() ) ) );
+				
+				
+				\IPS\Request::i()->setClearAutosaveCookie( $autoSaveKey . $lang->id );
 			}
 		}
 		else
@@ -1220,6 +1258,8 @@ abstract class _File
 			), array( 'temp=?', md5( $autoSaveKey ) ) );
 			
 			\IPS\Db::i()->update( 'core_attachments', array( 'attach_post_key' => '' ), array( 'attach_post_key=?', md5( $autoSaveKey . ':' . session_id() ) ) );
+			
+			\IPS\Request::i()->setClearAutosaveCookie( $autoSaveKey );
 		}
 	}
 	
@@ -1246,23 +1286,18 @@ abstract class _File
 			}
 		}
 		\IPS\Db::i()->delete( 'core_attachments_map', $where );
-		
-		/* Cleanup - this is deliberately slightly different to the one in \IPS\Text\Parser as it will only delete attachments which have been "saved" (i.e. are not in the middle of being posted) */
-		foreach ( \IPS\Db::i()->select( '*', 'core_attachments', array( array( 'attach_id NOT IN(?) AND attach_post_key=?', \IPS\Db::i()->select( 'DISTINCT attachment_id', 'core_attachments_map' ), '' ) ) ) as $attachment )
-		{
-			try
-			{
-				\IPS\Db::i()->delete( 'core_attachments', array( 'attach_id=?', $attachment['attach_id'] ) );
-				\IPS\File::get( 'core_Attachment', $attachment['attach_location'] )->delete();
-				if ( $attachment['attach_thumb_location'] )
-				{
-					\IPS\File::get( 'core_Attachment', $attachment['attach_thumb_location'] )->delete();
-				}
-			}
-			catch ( \Exception $e ) { }
-		}
 	}
 
+	/**
+	 * @brief This can be used to force a thumbnail name
+	 */
+	public $thumbnailName = NULL;
+
+	/**
+	 * @brief This can be used to force a thumbnail container
+	 */
+	public $thumbnailContainer = NULL;
+
 	/**
 	 * Make a thumbnail of the file - copies file, resizes and returns new file object
 	 *
@@ -1295,10 +1330,10 @@ abstract class _File
 		static::$thumbnailDimensions = array( $image->width, $image->height );
 		
 		/* What are we calling this? */		
-		$thumbnailName = mb_substr( $this->originalFilename, 0, mb_strrpos( $this->originalFilename, '.' ) ) . '.thumb' . mb_substr( $this->originalFilename, mb_strrpos( $this->originalFilename, '.' ) );
-		
+		$thumbnailName = $this->thumbnailName ?: mb_substr( $this->originalFilename, 0, mb_strrpos( $this->originalFilename, '.' ) ) . '.thumb' . mb_substr( $this->originalFilename, mb_strrpos( $this->originalFilename, '.' ) );
+
 		/* Create and return */
-		return \IPS\File::create( $storageExtension, $thumbnailName, $image );
+		return \IPS\File::create( $storageExtension, $thumbnailName, $image, $this->thumbnailContainer, FALSE, NULL, $this->thumbnailName ? FALSE : TRUE );
 	}
 
 	/**
@@ -1378,7 +1413,8 @@ abstract class _File
 		/* We only need to check the first 1kb of the file...some programs will use more, but this is the most common */
 		$firstBytes	= \substr( $data, 0, 1024 );
 
-		if( preg_match( '#(<script|<html|<head|<title|<body|<pre|<table|<a\s+href|<img|<plaintext|<cross\-domain\-policy)(\s|=|>)#si', $firstBytes, $matches ) )
+		/* @see https://mimesniff.spec.whatwg.org/#identifying-a-resource-with-an-unknown-mime-type */
+		if( preg_match( '#(<\!DOCTYPE\s+HTML|<script|<html|<head|<iframe|<h1|<div|<font|<table|<title|<style|<body|<pre|<table|<br|<a\s+href|<img|<plaintext|<cross\-domain\-policy|<\!\-\-|<\?xml)(\s|=|>)#si', $firstBytes, $matches ) )
 		{
 			return TRUE;
 		}
@@ -1395,9 +1431,9 @@ abstract class _File
 	{
 		$extension	= mb_strtolower( mb_substr( $filename, mb_strrpos( $filename, '.' ) + 1 ) );
 		
-		if ( array_key_exists( $extension, self::$mimeTypes ) )
+		if ( array_key_exists( $extension, static::$mimeTypes ) )
 		{
-			return self::$mimeTypes[ $extension ];
+			return static::$mimeTypes[ $extension ];
 		}
 		
 		return 'application/x-unknown'; // This, slightly unusual, type is needed to stop some browsers adding random extensions
@@ -1420,6 +1456,9 @@ abstract class _File
 
 		/* Get the last character, which may be 'm' or may be a number */
 		$last	= mb_strtolower( $size[ \strlen( $size ) - 1 ] );
+		
+		/* Convert $size to a number */
+		$size = intval( preg_replace( '/[^0-9]/', '', $size ) );
 
 		/* Adjust value as necessary - note that we do not break intentionally */
 		switch( $last )
@@ -1435,6 +1474,23 @@ abstract class _File
 		return (float) $size;
 	}
 	
+	/**
+	 * Get output for API
+	 *
+	 * @return	array
+	 * @apiresponse	string	name	The filename
+	 * @apiresponse	string	url		URL to where file is stored
+	 * @apiresponse	int		size	Filesize in bytes
+	 */
+	public function apiOutput()
+	{
+		return array(
+			'name'	=> $this->originalFilename,
+			'url'	=> (string) $this->url,
+			'size'	=> $this->filesize()
+		);
+	}
+	
 	/* !Mime-Type Map */
 	
 	/**

system/File/FileSystem.php

@@ -2,9 +2,9 @@
 /**
  * @brief		File Handler: File System
  * @author		<a href='http://www.invisionpower.com'>Invision Power Services, Inc.</a>
- * @copyright	(c) 2001 - SVN_YYYY Invision Power Services, Inc.
+ * @copyright	(c) 2001 - 2016 Invision Power Services, Inc.
  * @license		http://www.invisionpower.com/legal/standards/
- * @package		IPS Social Suite
+ * @package		IPS Community Suite
  * @since		06 May 2013
  * @version		SVN_VERSION_NUMBER
  */
@@ -53,9 +53,9 @@ class _FileSystem extends \IPS\File
 	 */
 	public static function testSettings( &$values )
 	{
-		$values['dir'] = rtrim( $values['dir'], '/' );
+		$values['dir'] = rtrim( $values['dir'], '\\/' );
 		$testDir = str_replace( '{root}', \IPS\ROOT_PATH, $values['dir'] );
-		$values['url'] = trim( str_replace( \IPS\ROOT_PATH, '', $testDir ), '/' );
+		$values['url'] = trim( str_replace( \IPS\ROOT_PATH, '', $testDir ), '\\/' );
 		
 		if ( empty( $values['toggle'] ) )
 		{
@@ -105,7 +105,7 @@ class _FileSystem extends \IPS\File
 	 */
 	public static function moveCheck( $configuration, $oldConfiguration )
 	{
-		if ( str_replace( '{root}', \IPS\ROOT_PATH, $configuration['dir'] ) !== str_replace( '{root}', \IPS\ROOT_PATH, $oldConfiguration['dir'] ) )
+		if ( str_replace( '{root}', \IPS\ROOT_PATH, preg_replace( '#/{1,}/#', '/', rtrim( $configuration['dir'], '/' ) ) ) !== str_replace( '{root}', \IPS\ROOT_PATH, preg_replace( '#/{1,}/#', '/', rtrim( $oldConfiguration['dir'], '/' ) ) ) )
 		{
 			return TRUE;
 		}
@@ -173,7 +173,7 @@ class _FileSystem extends \IPS\File
 		{
 			$url = str_replace( 'http://', 'https://', $url );
 		}
-		return $url;
+		return rtrim( $url, '/' );
 	}
 	
 	/**
@@ -249,7 +249,9 @@ class _FileSystem extends \IPS\File
 		{
 			if( file_exists( $this->configuration['dir'] . '/' . $this->container . '/' . $this->filename ) )
 			{
-				if ( ! ( $this->contents = @file_get_contents( $this->configuration['dir'] . '/' . $this->container . '/' . $this->filename ) ) )
+				$this->contents = @file_get_contents( $this->configuration['dir'] . '/' . $this->container . '/' . $this->filename );
+				
+				if ( $this->contents === FALSE )
 				{
 					throw new \RuntimeException( 'COULD_NOT_OPEN_FILE' );
 				}
@@ -307,19 +309,25 @@ class _FileSystem extends \IPS\File
 		/* Save the file */
 		if( $this->temporaryFilePath )
 		{
-			if( !@\rename( $this->temporaryFilePath, "{$folder}/{$this->filename}" ) OR @chmod( "{$folder}/{$this->filename}", \IPS\IPS_FILE_PERMISSION ) === FALSE )
+			if( !@\rename( $this->temporaryFilePath, "{$folder}/{$this->filename}" ) )
 			{
+				\IPS\Log::log( "Could not move file {$folder}/{$this->filename}" , 'FileSystem' );
 				throw new \RuntimeException( 'COULD_NOT_MOVE_FILE' );
 			}
+
+			@chmod( "{$folder}/{$this->filename}", \IPS\IPS_FILE_PERMISSION );
 		}
 		else
 		{
 			if ( $contents = $this->contents() )
 			{				
-				if ( !@\file_put_contents( "{$folder}/{$this->filename}", $contents ) OR @chmod( "{$folder}/{$this->filename}", \IPS\IPS_FILE_PERMISSION ) === FALSE )
+				if ( !@\file_put_contents( "{$folder}/{$this->filename}", $contents ) )
 				{					
+					\IPS\Log::log( "Could not write file {$folder}/{$this->filename}" , 'FileSystem' );
 					throw new \RuntimeException( 'COULD_NOT_WRITE_FILE' );
 				}
+
+				@chmod( "{$folder}/{$this->filename}", \IPS\IPS_FILE_PERMISSION );
 			}
 			else
 			{
@@ -328,14 +336,14 @@ class _FileSystem extends \IPS\File
 			}
 		}
 		
-		/* Set the URL */
-		$this->url = new \IPS\Http\Url( $this->fullyQualifiedUrl( "{$this->container}/{$this->filename}" ) );
-
 		/* Clear zend opcache if enabled */
-		if( function_exists('opcache_reset') )
+		if ( function_exists( 'opcache_invalidate' ) )
 		{
-			@opcache_reset();
+			@opcache_invalidate( "{$folder}/{$this->filename}" );
 		}
+		
+		/* Set the URL */
+		$this->url = \IPS\Http\Url::createFromString( $this->fullyQualifiedUrl( "{$this->container}/{$this->filename}" ), FALSE );
 	}
 		
 	/**
@@ -345,14 +353,17 @@ class _FileSystem extends \IPS\File
 	 */
 	public function delete()
 	{
+		/* Log deletion request */
+		$this->log( "file_deletion", 'delete', NULL, 'log' );
+
 		if( file_exists( "{$this->configuration['dir']}/{$this->container}/{$this->filename}" ) )
 		{
 			$result = @unlink( "{$this->configuration['dir']}/{$this->container}/{$this->filename}" );
 
 			/* Clear zend opcache if enabled */
-			if( function_exists('opcache_reset') )
+			if ( function_exists( 'opcache_invalidate' ) )
 			{
-				@opcache_reset();
+				@opcache_invalidate( "{$this->configuration['dir']}/{$this->container}/{$this->filename}" );
 			}
 
 			return $result;
@@ -376,29 +387,51 @@ class _FileSystem extends \IPS\File
 
 		if ( is_dir( $dir ) )
 		{
+			$files	= array();
+			$dirs	= array();
+
 			foreach ( new \DirectoryIterator( $dir ) as $f )
 			{
 				if ( !$f->isDot() )
 				{
 					if( $f->isDir() )
 					{
-						$this->deleteContainer( $container . '/' . $f->getFilename(), TRUE );
+						$dirs[]	= $container . '/' . $f->getFilename();
 					}
 					else
 					{
-						unlink( $f->getPathname() );
+						$files[]	= $f->getPathname();
 					}
 				}
 			}
 
+			/* If we have any directories to delete - delete them */
+			foreach( $dirs as $directory )
+			{
+				$this->deleteContainer( $directory, true );
+			}
+
+			/* And now delete the files */
+			foreach( $files as $file )
+			{
+				unlink( $file );
+
+				/* Clear zend opcache if enabled */
+				if( $skipClear === FALSE and function_exists( 'opcache_invalidate' ) )
+				{
+					@opcache_invalidate( $file );
+				}
+			}
+
+			/* And finally, remove the directory we're currently working with */
 			rmdir( $dir );
 		}
 
-		/* Clear zend opcache if enabled */
-		if( function_exists('opcache_reset') AND $skipClear === FALSE )
-		{
-			@opcache_reset();
-		}
+		/* Log deletion request */
+		$realContainer = $this->container;
+		$this->container = $container;
+		$this->log( "container_deletion", 'delete', $_REQUEST, 'log' );
+		$this->container = $realContainer;
 	}
 	
 	/**
@@ -425,9 +458,9 @@ class _FileSystem extends \IPS\File
 		@unlink( $chunk );
 
 		/* Clear zend opcache if enabled */
-		if( function_exists('opcache_reset') )
+		if ( function_exists( 'opcache_invalidate' ) )
 		{
-			@opcache_reset();
+			@opcache_invalidate( "{$this->configuration['dir']}/{$folder}/{$filename}" );
 		}
 
 		if ( $rename )

system/File/Ftp.php

@@ -2,9 +2,9 @@
 /**
  * @brief		File Handler: FTP
  * @author		<a href='http://www.invisionpower.com'>Invision Power Services, Inc.</a>
- * @copyright	(c) 2001 - SVN_YYYY Invision Power Services, Inc.
+ * @copyright	(c) 2001 - 2016 Invision Power Services, Inc.
  * @license		http://www.invisionpower.com/legal/standards/
- * @package		IPS Social Suite
+ * @package		IPS Community Suite
  * @since		06 May 2013
  * @version		SVN_VERSION_NUMBER
  */
@@ -138,7 +138,7 @@ class _Ftp extends \IPS\File
 	 */
 	public function baseUrl()
 	{
-		return $this->configuration['url'];
+		return rtrim( $this->configuration['url'], '/' );
 	}
 	
 	/**
@@ -181,7 +181,7 @@ class _Ftp extends \IPS\File
 		unlink( $tmpFileName );
 		
 		/* Set the URL */
-		$this->url = new \IPS\Http\Url( $this->fullyQualifiedUrl( "{$this->container}/{$this->filename}" ) );
+		$this->url = \IPS\Http\Url::createFromString( $this->fullyQualifiedUrl( "{$this->container}/{$this->filename}" ), FALSE );
 	}
 	
 	/**
@@ -193,6 +193,9 @@ class _Ftp extends \IPS\File
 	public function delete()
 	{
 		$this->ftp()->delete( $this->container . '/' . $this->filename );
+
+		/* Log deletion request */
+		$this->log( "file_deletion", 'delete', NULL, 'log' );
 	}
 		
 	/**
@@ -205,6 +208,12 @@ class _Ftp extends \IPS\File
 	public function deleteContainer( $container )
 	{
 		$this->ftp()->rmdir( $container, TRUE );
+
+		/* Log deletion request */
+		$realContainer = $this->container;
+		$this->container = $container;
+		$this->log( "container_deletion", 'delete', NULL, 'log' );
+		$this->container = $realContainer;
 	}
 	
 	/* !FTP Utility Methods */

system/File/Iterator.php

@@ -2,9 +2,9 @@
 /**
  * @brief		File IteratorIterator
  * @author		<a href='http://www.invisionpower.com'>Invision Power Services, Inc.</a>
- * @copyright	(c) 2001 - SVN_YYYY Invision Power Services, Inc.
+ * @copyright	(c) 2001 - 2016 Invision Power Services, Inc.
  * @license		http://www.invisionpower.com/legal/standards/
- * @package		IPS Social Suite
+ * @package		IPS Community Suite
  * @since		10 Oct 2013
  * @version		SVN_VERSION_NUMBER
  */
@@ -38,19 +38,26 @@ class _Iterator extends \IteratorIterator implements \Countable
 	 */
 	protected $fileUrlsOnly;
 	
+	/**
+	 * @brief	Used to restore 'real' names when filenames cleaned (eg adh1029_file.php back to fi^le.php)
+	 */
+	protected $replaceNameField;
+	
 	/**
 	 * Constructor
 	 *
 	 * @param	Traversable $iterator			The iterator
 	 * @param	string		$storageExtension	The storage extension
 	 * @param	string|NULL	$urlField			If passed a string, will look for an element with that key in the data returned from the iterator
+	 * @param	string|NULL	$replaceNameField	If passed a string, it will replace the originalFilename with the data in the array. Used to restore 'real' names when filenames cleaned (eg adh1029_file.php back to fi^le.php)
 	 * @return	void
 	 */
-	public function __construct( \Traversable $iterator, $storageExtension, $urlField=NULL, $fileUrlsOnly=FALSE )
+	public function __construct( \Traversable $iterator, $storageExtension, $urlField=NULL, $fileUrlsOnly=FALSE, $replaceNameField=NULL )
 	{
 		$this->storageExtension = $storageExtension;
 		$this->urlField = $urlField;
 		$this->fileUrlsOnly = $fileUrlsOnly;
+		$this->replaceNameField = $replaceNameField;
 		return parent::__construct( $iterator );
 	}
 	
@@ -64,7 +71,27 @@ class _Iterator extends \IteratorIterator implements \Countable
 		try
 		{
 			$data = $this->data();
-			$obj = \IPS\File::get( $this->storageExtension, $this->urlField ? $data[ $this->urlField ] : $data );
+			$urlField = NULL;
+			
+			if ( $this->urlField )
+			{
+				if ( !is_string( $this->urlField ) and is_callable( $this->urlField ) )
+				{
+					$urlField = call_user_func( $this->urlField, $data );
+				}
+				else
+				{
+					$urlField = $this->urlField;
+				}
+			}
+
+			$obj = \IPS\File::get( $this->storageExtension, $urlField ? $data[ $urlField ] : $data );
+			
+			if ( $this->replaceNameField and ! empty( $data[ $this->replaceNameField ] ) )
+			{
+				$obj->originalFilename = $data[ $this->replaceNameField ];
+			}
+			
 			return ( $this->fileUrlsOnly ) ? (string) $obj->url : $obj;
 		}
 		catch ( \Exception $e )