Version 4.5.0

system/File/Amazon.php

@@ -11,7 +11,7 @@
 namespace IPS\File;
 
 /* To prevent PHP errors (extending class does not exist) revealing path */
-if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
+if ( !\defined( '\IPS\SUITE_UNIQUE_KEY' ) )
 {
 	header( ( isset( $_SERVER['SERVER_PROTOCOL'] ) ? $_SERVER['SERVER_PROTOCOL'] : 'HTTP/1.0' ) . ' 403 Forbidden' );
 	exit;
@@ -22,7 +22,6 @@ if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
  */
 class _Amazon extends \IPS\File
 {
-	
 	/**
 	 * An array of ( configuration_id => array( ext, etx ) ) extensions that require gzip versions storing
 	 * Looks up $storageExtensions of extension classes
@@ -136,10 +135,11 @@ class _Amazon extends \IPS\File
 	public static function moveCheck( $configuration, $oldConfiguration )
 	{
 		foreach( array( 'bucket', 'bucket_path' ) as $field )
-		
-		if ( $configuration[ $field ] !== $oldConfiguration[ $field ] )
 		{
-			return TRUE;
+			if ( $configuration[ $field ] !== $oldConfiguration[ $field ] )
+			{
+				return TRUE;
+			}
 		}
 		
 		return FALSE;
@@ -157,6 +157,21 @@ class _Amazon extends \IPS\File
 	}
 	
 	/* !File Handling */
+	
+	/**
+	 * @brief	Does this storage method support chunked uploads?
+	 */
+	public static $supportsChunking = TRUE;
+	
+	/**
+ 	 * @brief	Min chunk size (in MB)
+ 	 */
+ 	public static $minChunkSize = 5;
+ 	
+	/**
+	 * @brief	Max chunk size (in MB)
+	 */
+	public static $maxChunkSize = 5000;
 
 	/**
 	 * Constructor
@@ -169,6 +184,24 @@ class _Amazon extends \IPS\File
 		$this->container = 'monthly_' . date( 'Y' ) . '_' . date( 'm' );
 		parent::__construct( $configuration );
 	}
+
+	/**
+	 * Get output for API
+	 *
+	 * @param	\IPS\Member|NULL	$authorizedMember	The member making the API request or NULL for API Key / client_credentials
+	 * @return	array
+	 * @apiresponse	string	name	The filename
+	 * @apiresponse	string	url		URL to where file is stored
+	 * @apiresponse	int		size	Filesize in bytes
+	 */
+	public function apiOutput( \IPS\Member $authorizedMember = NULL )
+	{
+		return array(
+			'name'	=> $this->originalFilename,
+			'url'	=> ( $this->isPrivate() AND $signedUrl = $this->generateTemporaryDownloadUrl() ) ? (string) $signedUrl : (string) $this->url,
+			'size'	=> $this->filesize()
+		);
+	}
 	
 	/**
 	 * Fetch the gzip extensions specific for $this->configurationId
@@ -186,7 +219,7 @@ class _Amazon extends \IPS\File
 				$bits     = explode( '_', $this->storageExtension );
 				$class    = '\IPS\\' . $bits[0] . '\extensions\core\FileStorage\\' . $bits[1];
 			
-				if ( isset( $class::$storeGzipExtensions ) and is_array( $class::$storeGzipExtensions ) and count( $class::$storeGzipExtensions ) )
+				if ( isset( $class::$storeGzipExtensions ) and \is_array( $class::$storeGzipExtensions ) and \count( $class::$storeGzipExtensions ) )
 				{
 					static::$gzipExtensions[ $this->storageExtension ] = $class::$storeGzipExtensions;
 				}
@@ -229,7 +262,7 @@ class _Amazon extends \IPS\File
 	public function needsGzipVersion()
 	{
 		/* Use filename and not originalFilename so only true .js and .css files are checked, and not renamed uploads */
-		return in_array(  mb_substr( $this->filename, mb_strrpos( $this->filename, '.' ) + 1 ), $this->getGzipExtensions() );
+		return \in_array(  mb_substr( $this->filename, mb_strrpos( $this->filename, '.' ) + 1 ), $this->getGzipExtensions() );
 	}
 	
 	/**
@@ -272,17 +305,7 @@ class _Amazon extends \IPS\File
 
 		if ( $response->httpResponseCode != 200 )
 		{
-			try
-			{
-				$xml = $response->decodeXml();
-				if ( isset( $xml->Code ) and isset( $xml->Message ) )
-				{
-					\IPS\Log::log( $xml->Code . ' ' . $xml->Message . ' ' . $path, 'Amazon' );
-				}
-			}
-			catch( \Exception $e ) { }
-			
-			throw new \IPS\File\Exception( $path, \IPS\File\Exception::CANNOT_WRITE );
+			throw new \IPS\File\Exception( $path, \IPS\File\Exception::CANNOT_WRITE, $this->originalFilename, $this->getExtraMessage( $response, $path ), $this->getErrorInformation( $response ) );
 		}
 		
 		/* Write the gzip version */
@@ -292,17 +315,7 @@ class _Amazon extends \IPS\File
 	
 			if ( $response->httpResponseCode != 200 )
 			{
-				try
-				{
-					$xml = $response->decodeXml();
-					if ( isset( $xml->Code ) and isset( $xml->Message ) )
-					{
-						\IPS\Log::log( $xml->Code . ' ' . $xml->Message . ' ' . $path, 'Amazon' );
-					}
-				}
-				catch( \Exception $e ) { }
-			
-				throw new \IPS\File\Exception( $path . '.gz', \IPS\File\Exception::CANNOT_WRITE );
+				throw new \IPS\File\Exception( $path . '.gz', \IPS\File\Exception::CANNOT_WRITE, $this->originalFilename, $this->getExtraMessage( $response, $path . '.gz' ), $this->getErrorInformation( $response ) );
 			}
 		}
 	}
@@ -320,11 +333,11 @@ class _Amazon extends \IPS\File
 			$response = static::makeRequest( $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}", 'GET', $this->configuration, $this->configurationId );
 			if ( $response->httpResponseCode == 404 )
 			{
-				throw new \IPS\File\Exception( $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}", \IPS\File\Exception::DOES_NOT_EXIST );
+				throw new \IPS\File\Exception( $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}", \IPS\File\Exception::DOES_NOT_EXIST, $this->originalFilename, $this->getExtraMessage( $response, $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}" ), $this->getErrorInformation( $response ) );
 			}
 			elseif( $response->httpResponseCode == 403 )
 			{
-				throw new \IPS\File\Exception( $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}", \IPS\File\Exception::CANNOT_COPY );
+				throw new \IPS\File\Exception( $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}", \IPS\File\Exception::CANNOT_COPY, $this->originalFilename, $this->getExtraMessage( $response, $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}" ), $this->getErrorInformation( $response ) );
 			}
 			else
 			{
@@ -346,7 +359,7 @@ class _Amazon extends \IPS\File
 		
 		$debug = array_map( function( $row ) {
 			return array_filter( $row, function( $key ) {
-				return in_array( $key, array( 'class', 'function', 'line' ) );
+				return \in_array( $key, array( 'class', 'function', 'line' ) );
 			}, ARRAY_FILTER_USE_KEY );
 		}, debug_backtrace( DEBUG_BACKTRACE_IGNORE_ARGS ) );
 		
@@ -388,46 +401,214 @@ class _Amazon extends \IPS\File
 	 * @return	void
 	 */
 	public function deleteContainer( $container )
+	{
+		/* Add it the s3 table */
+		try 
+		{
+			/* Is it already being deleted? */
+			\IPS\Db::i()->select( '*', 'core_s3_deletions', array( 's3_container=?', $container ) )->first();
+			
+			/* Update the added time so we delete everything < that time() */
+			\IPS\Db::i()->update( 'core_s3_deletions', array( 's3_added' => time(), 's3_last_cycle' => 0, 's3_marker' => '' ), array( 's3_container=?', $container ) );
+		}
+		catch( \UnderflowException $e )
+		{
+			/* It doesn't already exist, so add it */
+			\IPS\Db::i()->insert( 'core_s3_deletions', array(
+				's3_container'  => $container,
+				's3_added'	    => time(),
+				's3_last_cycle' => 0
+			) );
+			
+			/* Turn on task */
+			\IPS\Db::i()->update( 'core_tasks', array( 'enabled' => 1 ), array( '`key`=?', 's3_delete' ) );
+			
+			/* Log deletion request */
+			$realContainer = $this->container;
+			$this->container = $container;
+			$this->log( "container_deletion", 'delete', NULL, 'log' );
+			$this->container = $realContainer;
+		}
+	}
+	
+	/**
+	 * Return all keys that match the prefix
+	 * @note Future me, you're welcome: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html
+	 *
+	 * @param	string	$prefix		Prefix to match when fetching keys (such as css_built_)
+	 * @param	int		$limit		Number of keys to return in any one go
+	 * @param	string	$delimiter	Group keys by delimiter, so if you want to get all css_built_* containers: prefix=css_built_ delimiter=/ will return all folder names matching
+	 * @param	string	$marker		Pass in a marker to fetch the results from if result set truncated (this can be the last key fetched)
+	 * @return	array
+	 */
+	public function getContainerKeys( $prefix = '', $limit = 50, $delimiter = '', $marker='' )
 	{
 		$_strip = array( '_strip_querystring' => TRUE, 'bucket_path' => NULL );
 	
 		if ( $this->configuration['bucket_path'] )
 		{
-			$container = $this->configuration['bucket_path'] . '/' . $container;
+			$prefix = $this->configuration['bucket_path'] . '/' . $prefix;
 		}
 		
-		$response = static::makeRequest( "?prefix=" . urlencode( $container . "/" ), 'GET', array_merge( $this->configuration, $_strip ), $this->configurationId );
-		
+		if ( $marker and $this->configuration['bucket_path'] )
+		{
+			$marker = $this->configuration['bucket_path'] . '/' . $marker;
+		}
+
+		$response = static::makeRequest( '', 'GET', array_merge( $this->configuration, $_strip ), $this->configurationId, NULL, NULL, FALSE, FALSE, array( "prefix" => $prefix, 'max-keys' => $limit, 'delimiter' => $delimiter, 'marker' => $marker ) );
+
 		/* Parse XML document */
 		$document = \IPS\Xml\SimpleXML::loadString( $response );
-		
-		/* Loop over dom document */
-		foreach( $document->Contents as $result )
+		$keys = array();
+	
+		if ( $delimiter )
 		{
-			if ( $this->configuration['bucket_path'] )
+			/* Loop over dom document */
+			foreach( $document->CommonPrefixes as $result )
 			{
-				$result->Key = mb_substr( $result->Key, ( mb_strlen( $this->configuration['bucket_path'] ) + 1 ) );
+				if ( $this->configuration['bucket_path'] )
+				{
+					$keys[] =  trim( mb_substr( $result->Prefix, ( mb_strlen( $this->configuration['bucket_path'] ) + 1 ) ), '/' );
+				}
+				else
+				{
+					$keys[] = trim( $result->Prefix, '/' );
+				}
 			}
-			
-			static::makeRequest( $result->Key, 'DELETE', $this->configuration, $this->configurationId );
 		}
-
-		/* Log deletion request */
-		$realContainer = $this->container;
-		$this->container = $container;
-		$this->log( "container_deletion", 'delete', NULL, 'log' );
-		$this->container = $realContainer;
+		else
+		{
+			/* Loop over dom document */
+			foreach( $document->Contents as $result )
+			{
+				if ( $this->configuration['bucket_path'] )
+				{
+					$keys[ mb_substr( $result->Key, ( mb_strlen( $this->configuration['bucket_path'] ) + 1 ) ) ] = strtotime( (string) $result->LastModified );
+				}
+				else
+				{
+					$keys[ trim( $result->Key, '/' ) ] = strtotime( (string) $result->LastModified );
+				}
+			}
+		}
+		
+		return $keys;
 	}
-
+	
 	/**
-	 * @brief Cached filesize
+	 * Delete with a known key
+	 * Bit of a cheeky bypass right into AWS S3 guts
+	 *
+	 * @param	string	$key Key to delete, bucket_path added automatically if needed
+	 * @return	void
 	 */
-	protected $_cachedFilesize	= NULL;
+	public function deleteByKey( $key )
+	{
+		if ( $this->configuration['bucket_path'] )
+		{
+			$key = $this->configuration['bucket_path'] . '/' . $key;
+		}
+		
+		$_strip = array( '_strip_querystring' => TRUE, 'bucket_path' => NULL );
+		
+		try
+		{
+			static::makeRequest( $key, 'DELETE', array_merge( $this->configuration, $_strip ), $this->configurationId );
+			
+			\IPS\Db::i()->insert( 'core_file_logs', array (
+	          'log_action'           => 'delete',
+	          'log_type'             => 'log',
+	          'log_configuration_id' => $this->configurationId,
+	          'log_method'           => static::$storageConfigurations[ $this->configurationId ]['method'],
+	          'log_filename'         => $key,
+	          'log_url'              => '',
+	          'log_container'        => '',
+	          'log_msg'              => 'file_deletion',
+	          'log_date'             => time(),
+	          'log_data'             => NULL
+	        ) );
+		}
+		catch( \IPS\Http\Request\Exception $e )
+		{
+			/* If there was a problem deleting the file, don't stop code execution just because of that */
+			$this->log( 'HTTP_ERROR_DELETE_FILE', 'delete', array( $e->getCode(), $e->getMessage() ), 'error' );
+		}
+	}
+	
+	/**
+	 * Initiate a chunked upload
+	 *
+	 * @param	string		$filename	The desired filename
+	 * @param	string|null	$container	Key to identify container for storage
+	 * @param	bool		$obscure		Controls if an md5 hash should be added to the filename
+	 * @return	mixed					A reference to be passed to chunkProcess() and chunkFinish()
+	 * @throws	\RuntimeException
+	 */
+	public function chunkInit( $filename, $container = '', $obscure = TRUE )
+	{
+		$container = trim( $container, '/' );
+		$filename = $obscure ? static::obscureFilename( $filename ) : $filename;
+		$path = $this->container ? "{$this->container}/{$filename}" : "{$filename}";
+		
+		$r = static::makeRequest( $path, 'POST', $this->configuration, $this->configurationId, NULL, NULL, FALSE, FALSE, array( 'uploads' => '' ) );
+		$uploadId = (string) $r->decodeXml()->UploadId;
+		
+		return array(
+			'path'		=> $path,
+			'uploadId'	=> $uploadId
+		);
+	}
+	
+	/**
+	 * Append more contents in a chunked upload
+	 *
+	 * @param	mixed	$ref							The reference for this upload as returned by chunkInit() 
+	 * @param	string	$temporaryFileOrContents		The contents to write, or the path to the temporary file on disk with the contents
+	 * @param	int		$chunkNumber					Which chunk this is (0 for the first chunk, 1 for the second, etc)
+	 * @param	bool	$isContents					If TRUE, $temporaryFileOrContents is treated as raw contents. If FALSE, is path to file
+	 * @return	mixed								Updated reference fore future chunkProcess() calls and chunkFinish()
+	 * @throws	\RuntimeException
+	 */
+	public function chunkProcess( $ref, $temporaryFileOrContents, $chunkNumber, $isContents = FALSE )
+	{
+		$r = static::makeRequest( $ref['path'], 'PUT', $this->configuration, $this->configurationId, $isContents ? $temporaryFileOrContents : file_get_contents( $temporaryFileOrContents ), NULL, FALSE, NULL, array( 'partNumber' => ++$chunkNumber, 'uploadId' => $ref['uploadId'] ) );
+		
+		$ref['etags'][ $chunkNumber ] = trim( $r->httpHeaders['ETag'], '"' );
+		
+		if ( !$isContents )
+		{
+			@unlink( $temporaryFileOrContents );
+		}
+		
+		return $ref;
+	}
+	
+	/**
+	 * Finalize a chunked upload
+	 *
+	 * @param	mixed	$ref		The reference for this upload as returned by chunkInit() 
+	 * @return	string			The file's reference which can be passed to \IPS\File::get() as the URL
+	 * @throws	\RuntimeException
+	 */
+	public function chunkFinish( $ref )
+	{
+		$xml = \IPS\Xml\SimpleXML::create( 'CompleteMultipartUpload', 'http://s3.amazonaws.com/doc/2006-03-01/' );
+		foreach ( $ref['etags'] as $partNumber => $eTag )
+		{
+			$part = $xml->addChild( 'Part' );
+			$part->addChild( 'ETag', $eTag );
+			$part->addChild( 'PartNumber', $partNumber );
+		}
+						
+		static::makeRequest( $ref['path'], 'POST', $this->configuration, $this->configurationId, $xml->asXML(), NULL, FALSE, NULL, array( 'uploadId' => $ref['uploadId'] ) );
+
+		return $ref['path'];
+	}
 
 	/**
 	 * Get filesize (in bytes)
 	 *
-	 * @return	string
+	 * @return	string|bool
 	 */
 	public function filesize()
 	{
@@ -480,7 +661,7 @@ class _Amazon extends \IPS\File
 			$xml = $url->request()->get()->decodeXml();
 			if ( !isset( $xml->Region ) )
 			{
-				throw new \IPS\File\Exception( $fileUrl, \IPS\File\Exception::MISSING_REGION );
+				throw new \IPS\File\Exception( $fileUrl, \IPS\File\Exception::MISSING_REGION, $this->originalFilename );
 			}
 			
 			$this->configuration['region'] = (string) $xml->Region;
@@ -500,11 +681,14 @@ class _Amazon extends \IPS\File
 	 * @param	array 		$configuration		The configuration for this instance
 	 * @param	int			$configurationId	The configuration ID
 	 * @param	string|null	$content			The content to send
+	 * @param	string|null	$storageExtension	Storage extension
 	 * @param	bool		$skipExtraChecks	Skips the endpoint check (to prevent infinite looping)
+	 * @param	bool		$isPrivate			This can be set to true to access/store private files (i.e. that are not publicly readable)
+	 * @param	array 		$queryString		Array of key => value pairs
 	 * @return	\IPS\Http\Response
 	 * @throws	\IPS\Http\Request\Exception
 	 */
-	protected static function makeRequest( $uri, $verb, $configuration, $configurationId, $content=NULL, $storageExtension=NULL, $skipExtraChecks=FALSE, $isPrivate=false )
+	protected static function makeRequest( $uri, $verb, $configuration, $configurationId, $content=NULL, $storageExtension=NULL, $skipExtraChecks=FALSE, $isPrivate=false, $queryString=array() )
 	{
 		/* Amazon requires filename characters to be properly encoded - let's urlencode the filename here */
 		$uriPieces	= explode( '/', $uri );
@@ -512,7 +696,12 @@ class _Amazon extends \IPS\File
 		$uri		= ltrim( implode( '/', $uriPieces ) . '/' . rawurlencode( $filename ), '/' );
 		
 		/* Build a request */
-		$request = \IPS\Http\Url::external( static::buildBaseUrl( $configuration ) . $uri )->request( \IPS\LONG_REQUEST_TIMEOUT, NULL, FALSE ); # Amazon will send a 301 header code, but no Location header, if we need to try another endpoint
+		$url = \IPS\Http\Url::external( static::buildBaseUrl( $configuration ) . $uri );
+		if ( $queryString )
+		{
+			$url = $url->setQueryString( $queryString );
+		}
+		$request = $url->request( \IPS\VERY_LONG_REQUEST_TIMEOUT, NULL, FALSE ); # Amazon will send a 301 header code, but no Location header, if we need to try another endpoint
 		
 		/* When using virtual hosted–style buckets with SSL, the SSL wild card certificate only matches buckets that do not contain periods. To work around this, use HTTP or write your own certificate verification logic. @link http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html */
 		if ( \IPS\Request::i()->isSecure() and mb_strstr( $configuration['bucket'], '.' ) )
@@ -525,8 +714,11 @@ class _Amazon extends \IPS\File
 		$headers = array(
 			'Content-Type'	=> \IPS\File::getMimeType( $mimeUri ),
 			'Content-MD5'	=> base64_encode( md5( $content, TRUE ) ),
-			'X-Amz-Acl'		=> ( $isPrivate ? 'bucket-owner-read' : 'public-read' )
 		);
+		if ( $isPrivate !== NULL )
+		{
+			$headers['X-Amz-Acl'] = ( $isPrivate ? 'bucket-owner-read' : 'public-read' );
+		}
 		if ( $mimeUri !== $uri )
 		{
 			$headers['Content-Encoding'] = 'gzip';
@@ -562,7 +754,6 @@ class _Amazon extends \IPS\File
 		}
 		
 		/* Sign the request */
-		$queryString = array();
 		$authorization = static::signature( $configuration, $verb, $uri, $headers, $queryString, $content );
 		$headers['Authorization'] = $authorization;
 		unset( $headers['Host'] );
@@ -653,7 +844,7 @@ class _Amazon extends \IPS\File
 	 * @param	array 		$configuration					The configuration for this instance
 	 * @param	string		$verb							The HTTP verb that will be used in the request
 	 * @param	string		$uri							The URI (relative to the bucket)
-	 * @param	array		$header							The request headers as an array
+	 * @param	array		$headers						The request headers as an array
 	 * @param	array		$queryString					The query string as an array
 	 * @param	string|null	$content						The content to send
 	 * @param	bool		$signatureIsForQueryString		If true, signature will be generated for query string. If false, header.
@@ -668,7 +859,7 @@ class _Amazon extends \IPS\File
 		$contentSha256 = ( $signatureIsForQueryString and !$content ) ? 'UNSIGNED-PAYLOAD' : hash( 'sha256', $content );
 				
 		/* Figure out the canonical headers and query string */
-		$headers['Host'] = ( isset( $configuration['endpoint'] ) ? $configuration['endpoint'] : "s3.amazonaws.com" );
+		$headers['Host'] = $configuration['bucket'] . '.' . ( isset( $configuration['endpoint'] ) ? $configuration['endpoint'] : "s3.amazonaws.com" );
 		if ( $signatureIsForQueryString )
 		{
 			$queryString['X-Amz-Algorithm'] = 'AWS4-HMAC-SHA256';
@@ -693,13 +884,13 @@ class _Amazon extends \IPS\File
 		/* Task 1: Create a Canonical Request */
 		$canonicalRequest = implode( "\n", array(
 			mb_strtoupper( $verb ),
-			'/' . $configuration['bucket'] . static::bucketPath( $configuration ) . '/' . ltrim( $uri, '/' ),
+			static::bucketPath( $configuration ) . '/' . ltrim( $uri, '/' ),
 			http_build_query( $queryString, '', '&', PHP_QUERY_RFC3986 ),
 			$canonicalHeadersAsString,
 			implode( ';', array_map( 'mb_strtolower', array_keys( $headers ) ) ),
 			$contentSha256
 		) );
-						
+				
 		/* Task 2: Create a String to Sign */
 		$stringToSign = implode( "\n", array(
 			'AWS4-HMAC-SHA256',
@@ -734,9 +925,8 @@ class _Amazon extends \IPS\File
 	public static function buildBaseUrl( $configuration )
 	{
 		return (
-			\IPS\Request::i()->isSecure() ? "https" : "http" ) . "://"
+			\IPS\Request::i()->isSecure() ? "https" : "http" ) . "://{$configuration['bucket']}."
 			. ( isset( $configuration['endpoint'] ) ? $configuration['endpoint'] : "s3.amazonaws.com" )
-			. "/{$configuration['bucket']}"
 			. static::bucketPath( $configuration )
 			. '/';
 	}
@@ -829,4 +1019,74 @@ class _Amazon extends \IPS\File
 
 		return $results;
 	}
+
+	/**
+	 *  Retrieve any additional error information
+	 *
+	 * @param	\IPS\Http\Response	$response	The response object
+	 * @param	string				$path		File path if available
+	 * @return	string|null
+	 */
+	protected function getExtraMessage( $response, $path='' )
+	{
+		try
+		{
+			$xml = $response->decodeXml();
+			if ( isset( $xml->Code ) )
+			{
+				switch( $xml->Code )
+				{
+					case 'RequestTimeTooSkewed':
+						return 's3-RequestTimeTooSkewed';
+					break;
+
+					case 'AccessDenied':
+					case 'AccountProblem':
+					case 'AllAccessDisabled':
+					case 'InvalidPayer':
+					case 'NotSignedUp':
+						return 's3-AccountProblem';
+					break;
+
+					case 'EntityTooSmall':
+					case 'MissingRequestBodyError':
+						return 's3-FileSizeSmall';
+					break;
+
+					case 'EntityTooLarge':
+					case 'MaxMessageLengthExceeded':
+						return 's3-FileSizeLarge';
+					break;
+
+					default:
+						return 's3-GenericError';
+					break;
+				}
+			}
+		}
+		catch( \Exception $e ) { }
+
+		return null;
+	}
+
+	/**
+	 * Get the error code and message from Amazon to log
+	 *
+	 * @param	\IPS\Http\Response	$response	The response object
+	 * @return	string|null
+	 */
+	protected function getErrorInformation( $response )
+	{
+		try
+		{
+			$xml = $response->decodeXml();
+			if ( isset( $xml->Code ) and isset( $xml->Message ) )
+			{
+				return $xml->Code . ': ' . $xml->Message;
+			}
+		}
+		catch( \Exception $e ) { }
+
+		return null;
+	}
 }

system/File/Database.php

@@ -11,7 +11,7 @@
 namespace IPS\File;
 
 /* To prevent PHP errors (extending class does not exist) revealing path */
-if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
+if ( !\defined( '\IPS\SUITE_UNIQUE_KEY' ) )
 {
 	header( ( isset( $_SERVER['SERVER_PROTOCOL'] ) ? $_SERVER['SERVER_PROTOCOL'] : 'HTTP/1.0' ) . ' 403 Forbidden' );
 	exit;
@@ -91,7 +91,8 @@ class _Database extends \IPS\File
 	/**
 	 * Encode the file name for the fully qualified URL
 	 *
-	 * @return string
+	 * @param	string	$filename	The filename
+	 * @return	string
 	 */
 	public function encodeFileUrl( $filename )
 	{
@@ -125,7 +126,7 @@ class _Database extends \IPS\File
 			}
 			catch( \UnderflowException $ex )
 			{
-				throw new \IPS\File\Exception( $this->container . '/' . $this->filename, \IPS\File\Exception::DOES_NOT_EXIST );
+				throw new \IPS\File\Exception( $this->container . '/' . $this->filename, \IPS\File\Exception::DOES_NOT_EXIST, $this->originalFilename );
 			}
 		}
 	}
@@ -147,7 +148,7 @@ class _Database extends \IPS\File
 			}
 			catch( \UnderflowException $ex )
 			{
-				throw new \IPS\File\Exception( $this->container . '/' . $this->filename, \IPS\File\Exception::DOES_NOT_EXIST );
+				throw new \IPS\File\Exception( $this->container . '/' . $this->filename, \IPS\File\Exception::DOES_NOT_EXIST, $this->originalFilename );
 			}
 		}
 		return $this->contents;
@@ -199,7 +200,7 @@ class _Database extends \IPS\File
 		$immediateCaller = debug_backtrace( DEBUG_BACKTRACE_IGNORE_ARGS, 1 );
 		$debug = array_map( function( $row ) {
 			return array_filter( $row, function( $key ) {
-				return in_array( $key, array( 'class', 'function', 'line' ) );
+				return \in_array( $key, array( 'class', 'function', 'line' ) );
 			}, ARRAY_FILTER_USE_KEY );
 		}, debug_backtrace( DEBUG_BACKTRACE_IGNORE_ARGS ) );
 		$this->log( "file_deletion", 'delete', $debug, 'log' );

system/File/Exception.php

@@ -11,7 +11,7 @@
 namespace IPS\File;
 
 /* To prevent PHP errors (extending class does not exist) revealing path */
-if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
+if ( !\defined( '\IPS\SUITE_UNIQUE_KEY' ) )
 {
 	header( ( isset( $_SERVER['SERVER_PROTOCOL'] ) ? $_SERVER['SERVER_PROTOCOL'] : 'HTTP/1.0' ) . ' 403 Forbidden' );
 	exit;
@@ -23,14 +23,38 @@ if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
 class _Exception extends \RuntimeException
 {
 	/**
-	 * Exception types
+	 * @brief	Cannot open the file
 	 */
 	const CANNOT_OPEN		= 1;
+
+	/**
+	 * @brief	File does not exist
+	 */
 	const DOES_NOT_EXIST	= 2;
+
+	/**
+	 * @brief	Cannot write the file
+	 */
 	const CANNOT_WRITE		= 3;
+
+	/**
+	 * @brief	Cannot copy the file
+	 */
 	const CANNOT_COPY		= 4;
+
+	/**
+	 * @brief	Cannot move the file
+	 */
 	const CANNOT_MOVE		= 5;
+
+	/**
+	 * @brief	Cannot create a directory
+	 */
 	const CANNOT_MAKE_DIR	= 6;
+
+	/**
+	 * @brief	Region is missing (AWS)
+	 */
 	const MISSING_REGION	= 7;
 
 	/**
@@ -38,28 +62,63 @@ class _Exception extends \RuntimeException
 	 */
 	public $file = NULL;
 
+	/**
+	 * @brief	Original filename (used for friendlier errors)
+	 */
+	public $originalFilename = NULL;
+
+	/**
+	 * @brief	Additional error information
+	 */
+	public $errorMessage = NULL;
+
+	/**
+	 * @brief	Additional log information
+	 */
+	public $extraLog = NULL;
+
 	/**
 	 * Constructor
 	 *
-	 * @param	string		$file		File path
-	 * @param	int			$error		One of the defined exception constants
+	 * @param	string		$file			File path
+	 * @param	int			$error			One of the defined exception constants
+	 * @param	string|null	$originalFilename	The original filename (used for friendlier error messages)
+	 * @param	string|null	$errorMessage	Error message language string
+	 * @param	string|null	$extraLog		Error information to log
 	 * @return	void
 	 */
-	public function __construct( $file, $error )
+	public function __construct( $file, $error, $originalFilename = NULL, $errorMessage = NULL, $extraLog = NULL )
 	{
 		/* Store the file */
 		$this->file = $file;
+		$this->originalFilename = $originalFilename;
 
-		/* We use a prettier error message if we have a dispatcher instance */
-		if( \IPS\Dispatcher::hasInstance() )
-		{
-			$message = sprintf( \IPS\Member::loggedIn()->language()->get( 'files-' . $error ), $file );
-		}
-		else
-		{
-			$message = array_flip( ( new \ReflectionClass( __CLASS__ ) )->getConstants() )[ $error ];
-		}
+		/* Store additional debug info */
+		$this->errorMessage	= $errorMessage;
+		$this->extraLog		= $extraLog;
+
+		$message = array_flip( ( new \ReflectionClass( __CLASS__ ) )->getConstants() )[ $error ];
 
 		parent::__construct( $message, $error );
 	}
+
+	/**
+	 * Additional error message info
+	 *
+	 * @return	string
+	 */
+	public function extraErrorMessage()
+	{
+		return (string) $this->errorMessage;
+	}
+
+	/**
+	 * Additional log data?
+	 *
+	 * @return	string
+	 */
+	public function extraLogData()
+	{
+		return (string) $this->extraLog;
+	}
 }

system/File/File.php

@@ -11,7 +11,7 @@
 namespace IPS;
 
 /* To prevent PHP errors (extending class does not exist) revealing path */
-if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
+if ( !\defined( '\IPS\SUITE_UNIQUE_KEY' ) )
 {
 	header( ( isset( $_SERVER['SERVER_PROTOCOL'] ) ? $_SERVER['SERVER_PROTOCOL'] : 'HTTP/1.0' ) . ' 403 Forbidden' );
 	exit;
@@ -22,10 +22,36 @@ if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
  */
 abstract class _File
 {	
+	/**
+	 * Get available storage handlers
+	 *
+	 * @param	array|NULL	$current	The file storage configuration being edited
+	 * @return	array
+	 */
+	public static function storageHandlers( $current )
+	{
+		$return = array();
+		
+		if ( !\IPS\CIC OR ( $current and $current['method'] === 'FileSystem' ) )
+		{
+			$return['FileSystem'] = 'IPS\\File\\FileSystem';
+		}
+		
+		$return['Amazon'] = 'IPS\\File\\Amazon';
+		$return['Database'] = 'IPS\\File\\Database';
+		
+		if ( $current and $current['method'] === 'Ftp' )
+		{
+			$return['Ftp'] = 'IPS\\File\\Ftp';
+		}
+		
+		return $return;
+	}
+	
 	/**
 	 * @brief	File extensions considered safe. Not an exhaustive list but these are the files we're most interested in being recognised.
 	 */
-	public static $safeFileExtensions = array( 'js', 'css', 'txt', 'xml', 'ico', 'gif', 'jpg', 'jpe', 'jpeg', 'png', 'mp4', '3gp', 'mov', 'ogg', 'ogv', 'mp3', 'mpg', 'mpeg', 'ico', 'flv', 'webm', 'wmv', 'avi', 'm4v' );
+	public static $safeFileExtensions = array( 'js', 'css', 'txt', 'ico', 'gif', 'jpg', 'jpe', 'jpeg', 'png', 'mp4', '3gp', 'mov', 'ogg', 'ogv', 'mp3', 'mpg', 'mpeg', 'ico', 'flv', 'webm', 'wmv', 'avi', 'm4v' );
 	
 	/**
 	 * @brief	File extensions for HTML5 compatible videos
@@ -47,6 +73,11 @@ abstract class _File
 	 */
 	protected static $thumbnailDimensions = array();
 
+	/**
+	 * @brief Cached filesize
+	 */
+	protected $_cachedFilesize	= NULL;
+
 	/**
 	 * @brief	Ignore errors from uploaded files?
 	 */
@@ -63,14 +94,14 @@ abstract class _File
 	 * @param	string|int		$storageExtension	Storage extension or configuration ID
 	 * @param   bool			$tryOldFirst		Whether to try the old file storage config first or not
 	 * @throws 	\RuntimeException
-	 * @return	\IPS\File
+	 * @return	static
 	 */
 	public static function getClass( $storageExtension, $tryOldFirst=FALSE )
 	{
 		static::getConfigurations();
-
+		
 		$configurationId = NULL;
-		if ( is_int( $storageExtension ) ) 
+		if ( \is_int( $storageExtension ) ) 
 		{
 			$configurationId = $storageExtension;
 		}
@@ -78,13 +109,45 @@ abstract class _File
 		{
 			$settings = json_decode( \IPS\Settings::i()->upload_settings, TRUE );
 
+			/* If we are IN_DEV, make sure this is a valid file storage extension */
+			if ( \IPS\IN_DEV and !isset( $settings[ "filestorage__{$storageExtension}" ] ) )
+			{
+				/* Quick sanity check in case this is a new extension that hasn't been set yet. */
+				$isValid = FALSE;
+
+				foreach( \IPS\Application::allExtensions( 'core', 'FileStorage', FALSE ) as $extension )
+				{
+					$extensionNamespace = explode( '\\', \get_class( $extension ) );
+					$extensionName		= array_pop( $extensionNamespace );
+
+					if( $storageExtension == $extensionNamespace[1] . '_' . $extensionName )
+					{
+						$isValid = TRUE;
+						break;
+					}
+				}
+
+				if( !$isValid )
+				{
+					throw new \RuntimeException( 'NO_STORAGE_EXTENSION' );			
+				}
+			}
+			
 			/* If this storage extension hasn't been set yet, grab the first *valid* configuration and use that (and set it for future use) */
 			if ( !isset( $settings[ "filestorage__{$storageExtension}" ] ) )
 			{
 				foreach ( static::$storageConfigurations as $k => $data )
 				{
 					/* Test the storage config - the first one that works wins! */
-					$classname = 'IPS\File\\' . $data['method'];
+					$handlers = static::storageHandlers( $data );
+
+					/* Check the handler is still available */
+					if( !isset( $handlers[ $data['method'] ] ) )
+					{
+						continue;
+					}
+
+					$classname = $handlers[ $data['method'] ];
 					$class = new $classname( json_decode( $data['configuration'], TRUE ) );
 					$class->configurationId = $k;
 
@@ -112,7 +175,7 @@ abstract class _File
 			else
 			{
 				/* We have an array of IDs when a move is in progress, the first ID is the new storage method, the second ID is the old */
-				if ( is_array( $settings[ "filestorage__{$storageExtension}" ] ) )
+				if ( \is_array( $settings[ "filestorage__{$storageExtension}" ] ) )
 				{
 					/* Do we want to use the old storage config if available? */
 					if ( $tryOldFirst === TRUE )
@@ -154,8 +217,9 @@ abstract class _File
 			throw new \RuntimeException;
 		}		
 		
-		$classname = 'IPS\File\\' . static::$storageConfigurations[ $configurationId ]['method'];
-		$class = new $classname( json_decode( static::$storageConfigurations[ $configurationId ]['configuration'], TRUE ) );
+		$handlers	= static::storageHandlers( static::$storageConfigurations[ $configurationId ] );
+		$classname	= $handlers[ static::$storageConfigurations[ $configurationId ]['method'] ];
+		$class		= new $classname( json_decode( static::$storageConfigurations[ $configurationId ]['configuration'], TRUE ) );
 		$class->configurationId = $configurationId;
 		return $class;
 	}
@@ -165,7 +229,7 @@ abstract class _File
 	 *
 	 * @return	array
 	 */
-	public static function getConfigurations()
+	public static function getStore()
 	{
 		if ( static::$storageConfigurations === NULL )
 		{
@@ -175,19 +239,35 @@ abstract class _File
 			}
 			else
 			{
-				static::$storageConfigurations = iterator_to_array( \IPS\Db::i()->select( '*', 'core_file_storage' )->setKeyField('id') );
-				\IPS\Data\Store::i()->storageConfigurations = static::$storageConfigurations;
+				\IPS\Data\Store::i()->storageConfigurations = iterator_to_array( \IPS\Db::i()->select( '*', 'core_file_storage' )->setKeyField('id') );
+				static::$storageConfigurations = \IPS\Data\Store::i()->storageConfigurations;
 			}
 		}
 
 		return static::$storageConfigurations;
 	}
 
+	/**
+	 * Load storage configurations
+	 *
+	 * @return	array
+	 * @deprecated
+	 */
+	public static function getConfigurations()
+	{
+		return static::getStore();
+	}
+
 	/**
 	 * @brief	Copy files instead of moving them
 	 */
 	public static $copyFiles = FALSE;
 
+	/**
+	 * @brief	Temporarily stored EXIF data for an image
+	 */
+	public $exifData	= NULL;
+
 	/**
 	 * Create File
 	 *
@@ -211,7 +291,7 @@ abstract class _File
 		}
 
 		/* Init */
-		$class = static::getClass( $storageExtension );
+		$class = static::getClass( $storageExtension, TRUE );
 		if ( $container !== NULL )
 		{
 			$class->container = $container;
@@ -221,7 +301,7 @@ abstract class _File
 						
 		/* Image-specific stuff */
 		$ext = mb_substr( $filename, ( mb_strrpos( $filename, '.' ) + 1 ) );
-		if( in_array( mb_strtolower( $ext ), \IPS\Image::$imageExtensions ) and ( !$isSafe or \IPS\Image::exifSupported() ) )
+		if( \in_array( mb_strtolower( $ext ), \IPS\Image::$imageExtensions ) and ( !$isSafe or \IPS\Image::exifSupported() ) )
 		{
 			/* Get contents */
 			if( $data === NULL AND $filePath !== NULL )
@@ -250,6 +330,9 @@ abstract class _File
 			if ( \IPS\Image::exifSupported() )
 			{
 				$image = $image ?: \IPS\Image::create( $data );
+
+				$class->exifData = $image->parseExif();
+				
 				if ( $image->hasBeenRotated )
 				{
 					$data = (string) \IPS\Image::create( $data );
@@ -283,7 +366,7 @@ abstract class _File
 	 * @param	array|NULL	$allowedFileTypes	Array of allowed file extensions, or NULL to allow any extensions
 	 * @param	int|NULL	$maxFileSize		The maximum file size in MB, or NULL to allow any size
 	 * @param	int|NULL	$totalMaxSize		The maximum total size of all files in MB, or NULL for no limit
-	 * @param	int			$flags				\IPS\File::IGNORE_UPLOAD_ERRORS to skip over invalid files rather than throw exception
+	 * @param	int			$flags				`\IPS\File::IGNORE_UPLOAD_ERRORS` to skip over invalid files rather than throw exception
 	 * @param	array|NULL	$callback			Callback function to run against the file contents before creating the file (useful for resizing images, for instance)
 	 * @param	string|null	$container			Key to identify container for storage
 	 * @param	bool		$obscure			Controls if an md5 hash should be added to the filename
@@ -325,7 +408,7 @@ abstract class _File
 					$currentTotal += $file['size'];
 					if ( $currentTotal > ( $totalMaxSize * 1048576 ) )
 					{
-						throw new \DomainException( \IPS\Member::loggedIn()->language()->addToStack('uploaderr_total_size', FALSE, array( 'sprintf' => array( $totalMaxSize ) ) ) );
+						throw new \DomainException( \IPS\Member::loggedIn()->language()->addToStack('uploaderr_total_size', FALSE, array( 'sprintf' => array( \IPS\Output\Plugin\Filesize::humanReadableFilesize( $totalMaxSize * 1048576 ) ) ) ) );
 					}
 				}
 				
@@ -348,7 +431,7 @@ abstract class _File
 				if( $callback !== NULL )
 				{
 					$contents	= file_get_contents( $file['tmp_name'] );
-					$contents	= call_user_func( $callback, $contents, $file['name'] );
+					$contents	= $callback( $contents, $file['name'] );
 
 					$fileObjects[]	= static::create( $storageLocation, $file['name'], $contents, $container, FALSE, NULL, $obscure );
 				}
@@ -402,14 +485,14 @@ abstract class _File
 			}
 
 			/* Do we have $_FILES['field'] = array( 'name' => ..., 'size' => ... ) */
-			if( isset( $file['name'] ) AND !is_array( $file['name'] ) )
+			if( isset( $file['name'] ) AND !\is_array( $file['name'] ) )
 			{
 				$files[]	= $file;
 			}
 			/* Or do we have $_FILES['field'] = array( 'name' => array( 0 => ..., 1 => ... ), 'size' => array( 0 => ..., 1 => ... ) ) */
 			else
 			{
-				if( is_array( $file['name'] ) )
+				if( \is_array( $file['name'] ) )
 				{
 					foreach( $file as $fieldName => $fields )
 					{
@@ -491,19 +574,19 @@ abstract class _File
 
 		/* Check allowed types */
 		$ext = mb_substr( $file['name'], mb_strrpos( $file['name'], '.' ) + 1 );
-		if( $allowedFileTypes !== NULL and is_array( $allowedFileTypes ) and !empty( $allowedFileTypes ) )
+		if( $allowedFileTypes !== NULL and \is_array( $allowedFileTypes ) and !empty( $allowedFileTypes ) )
 		{
-			if( !in_array( mb_strtolower( $ext ), array_map( 'mb_strtolower', $allowedFileTypes ) ) )
+			if( !\in_array( mb_strtolower( $ext ), array_map( 'mb_strtolower', $allowedFileTypes ) ) )
 			{
 				throw new \DomainException( '-601', 3 );
 			}
 		}
 
 		/* If it's got an image extension, check it's actually a valid image */
-		if ( in_array( $ext, \IPS\Image::$imageExtensions ) AND !isset( $file['_skipUploadCheck'] ) )
+		if ( \in_array( $ext, \IPS\Image::$imageExtensions ) AND !isset( $file['_skipUploadCheck'] ) )
 		{
 			$imageAttributes = getimagesize( $file['tmp_name'] );
-			if( !in_array( $imageAttributes[2], array( IMAGETYPE_GIF, IMAGETYPE_JPEG, IMAGETYPE_PNG ) ) )
+			if( !\in_array( $imageAttributes[2], array( IMAGETYPE_GIF, IMAGETYPE_JPEG, IMAGETYPE_PNG ) ) )
 			{
 				throw new \DomainException( 'upload_error', 4 );
 			}
@@ -517,12 +600,13 @@ abstract class _File
 	 *
 	 * @param	string					$storageExtension	Storage extension
 	 * @param	string|\IPS\Http|Url	$url				URL to file
+	 * @param 	int						$cachedFilesize		Pre-cache filesize (bytes)
 	 * @return	\IPS\File
 	 * @throws	\OutOfRangeException
 	 */
-	public static function get( $storageExtension, $url )
+	public static function get( $storageExtension, $url, int $cachedFilesize=NULL )
 	{
-		if( mb_strpos( rawurldecode( $url ), '../' ) !== FALSE )
+		if( mb_strpos( rawurldecode( $url ), '../' ) !== FALSE OR mb_strpos( rawurldecode( $url ), '..\\' ) !== FALSE )
 		{
 			throw new \OutOfRangeException( 'INVALID_PATH' );
 		}
@@ -530,6 +614,7 @@ abstract class _File
 		$class = static::getClass( $storageExtension, TRUE );
 		$class->storageExtension = $storageExtension;
 		$class->url = $url;
+		$class->_cachedFilesize = $cachedFilesize;
 		$class->load();
 		return $class;
 	}
@@ -556,7 +641,7 @@ abstract class _File
 	public static function moveCheck( $configuration, $oldConfiguration )
 	{
 		$needsMove = FALSE;
-		if ( count( array_merge( array_diff( $configuration, $oldConfiguration ), array_diff( $oldConfiguration, $configuration ) ) ) )
+		if ( \count( array_merge( array_diff( $configuration, $oldConfiguration ), array_diff( $oldConfiguration, $configuration ) ) ) )
 		{
 			$needsMove = TRUE;
 		}
@@ -593,6 +678,7 @@ abstract class _File
 	 * This is primarily a utility function to convert old style full URLs (http://site.com/uploads/monthly_04_2015/file.txt) into the new style (monthly_04_2015/file.txt)
 	 *
 	 * @deprecated 4.1.0
+	 * @param	string	$url	The URL to repair
 	 * @return string|boolean	URL (string) if URL needed repairing, or FALSE if it did not.
 	 */
 	public static function repairUrl( $url )
@@ -667,12 +753,14 @@ abstract class _File
 	/**
 	 * Constructor
 	 *
-	 * @param	array	$configuration	Storage configuration
+	 * @param	array	$configuration		Storage configuration
+	 * @param 	int		$cachedFilesize		Pre-cached filesize
 	 * @return	void
 	 */
-	public function __construct( $configuration )
+	public function __construct( $configuration, int $cachedFilesize=null )
 	{
 		$this->configuration = $configuration;
+		$this->_cachedFilesize = $cachedFilesize;
 	}
 	
 	/**
@@ -742,7 +830,7 @@ abstract class _File
 			ob_end_clean();
 		}
 
-		if( $throttle === NULL AND $start === NULL AND function_exists('readfile') )
+		if( $throttle === NULL AND $start === NULL AND \function_exists('readfile') )
 		{
 			readfile( $file );
 		}
@@ -811,18 +899,25 @@ abstract class _File
 	/**
 	 * Get filesize (in bytes)
 	 *
-	 * @return	string
+	 * @return	string|bool
 	 */
 	public function filesize()
 	{
+		if( $this->_cachedFilesize !== NULL )
+		{
+			return $this->_cachedFilesize;
+		}
+
 		try
 		{
-			return \strlen( $this->contents() );
+			$this->_cachedFilesize = \strlen( $this->contents() );
 		}
 		catch( \RuntimeException $ex )
 		{
-			return FALSE;
-		}	
+			$this->_cachedFilesize = FALSE;
+		}
+
+		return $this->_cachedFilesize;
 	}
 
 	/**
@@ -881,7 +976,7 @@ abstract class _File
 	protected function obscureFilename( $filename )
 	{
 		$ext  = mb_substr( $filename, ( mb_strrpos( $filename, '.' ) + 1 ) );
-		$safe = in_array( mb_strtolower( $ext ), static::$safeFileExtensions );
+		$safe = \in_array( mb_strtolower( $ext ), static::$safeFileExtensions );
 
 		if ( ! $safe )
 		{
@@ -1040,6 +1135,7 @@ abstract class _File
 	/**
 	 * Return the fully qualified URL
 	 *
+	 * @param	string	$url	URL
 	 * @return string
 	 */
 	public function fullyQualifiedUrl( $url )
@@ -1055,6 +1151,7 @@ abstract class _File
 	/**
 	 * Encode the file name for the fully qualified URL
 	 *
+	 * @param	string	$filename	Filename in the URL
 	 * @return string
 	 */
 	public function encodeFileUrl( $filename )
@@ -1121,8 +1218,9 @@ abstract class _File
 	{
 		/* Load class */
 		static::getConfigurations();
-		$classname = '\IPS\File\\' . static::$storageConfigurations[ $storageConfiguration ]['method'];
-		$class = new $classname( json_decode( static::$storageConfigurations[ $storageConfiguration ]['configuration'], TRUE ) );
+		$handlers	= static::storageHandlers( static::$storageConfigurations[ $storageConfiguration ] );
+		$classname	= $handlers[ static::$storageConfigurations[ $storageConfiguration ]['method'] ];
+		$class		= new $classname( json_decode( static::$storageConfigurations[ $storageConfiguration ]['configuration'], TRUE ) );
 
 		/* Store it there */
 		if ( $this->container !== NULL )
@@ -1174,8 +1272,8 @@ abstract class _File
 	/**
 	 * Make into an attachment
 	 *
-	 * @param	string	$postKey	Post key
-	 * @param	\IPS\Member|NULL	Member who uploaded the attachment
+	 * @param	string				$postKey	Post key
+	 * @param	\IPS\Member|NULL	$member		Member who uploaded the attachment
 	 * @return	array
 	 * @throws	\DomainException
 	 */
@@ -1240,7 +1338,7 @@ abstract class _File
 	{
 		$ext = mb_substr( $this->originalFilename, mb_strrpos( $this->originalFilename, '.' ) + 1 );
 
-		if ( in_array( mb_strtolower( $ext ), \IPS\Image::$imageExtensions ) )
+		if ( \in_array( mb_strtolower( $ext ), \IPS\Image::$imageExtensions ) )
 		{
 			return TRUE;
 		}
@@ -1257,7 +1355,7 @@ abstract class _File
 	{
 		$ext = mb_substr( $this->originalFilename, mb_strrpos( $this->originalFilename, '.' ) + 1 );
 
-		if ( in_array( mb_strtolower( $ext ), \IPS\File::$videoExtensions ) )
+		if ( \in_array( mb_strtolower( $ext ), \IPS\File::$videoExtensions ) )
 		{
 			return TRUE;
 		}
@@ -1286,6 +1384,17 @@ abstract class _File
 		}
 	}
 
+	/**
+	 * Generate a temporary download URL the user can be redirected to
+	 *
+	 * @param	$validForSeconds	int	The number of seconds the link should be valid for
+	 * @return	\IPS\Http\Url
+	 */
+	public function generateTemporaryDownloadUrl( $validForSeconds = 1200 )
+	{
+		return NULL;
+	}
+
 	/**
 	 * If the file is an image, get the dimensions
 	 *
@@ -1318,7 +1427,7 @@ abstract class _File
 	 * @note	If you call this, it is your responsibility to call unclaimAttachments if/when the thing is deleted
 	 */
 	public static function claimAttachments( $autoSaveKey, $id1=NULL, $id2=NULL, $id3=NULL, $translatable=FALSE )
-	{
+	{		
 		if ( $translatable )
 		{
 			foreach ( \IPS\Lang::languages() as $lang )
@@ -1421,7 +1530,7 @@ abstract class _File
 		$thumbnailName = $this->thumbnailName ?: mb_substr( $this->originalFilename, 0, mb_strrpos( $this->originalFilename, '.' ) ) . '.thumb' . mb_substr( $this->originalFilename, mb_strrpos( $this->originalFilename, '.' ) );
 
 		/* Create and return */
-		return \IPS\File::create( $storageExtension, $thumbnailName, $image, $this->thumbnailContainer, FALSE, NULL, $this->thumbnailName ? FALSE : TRUE );
+		return \IPS\File::create( $storageExtension, $thumbnailName, (string) $image, $this->thumbnailContainer, FALSE, NULL, $this->thumbnailName ? FALSE : TRUE );
 	}
 
 	/**
@@ -1435,6 +1544,11 @@ abstract class _File
 	 */
 	protected function log( $message, $action, $data=NULL, $type='error' )
 	{
+		if ( $this->configurationId === NULL )
+		{
+			return;
+		}
+		
 		\IPS\Db::i()->insert( 'core_file_logs', array (
           'log_action'           => $action,
           'log_type'             => $type,
@@ -1445,7 +1559,7 @@ abstract class _File
           'log_container'        => $this->container,
           'log_msg'              => $message,
           'log_date'             => time(),
-          'log_data'             => is_array( $data ) ? json_encode( $data ) : NULL
+          'log_data'             => \is_array( $data ) ? json_encode( $data ) : NULL
         ) );
 	}
 	
@@ -1511,8 +1625,9 @@ abstract class _File
 	}
 	
 	/**
-	 * Get mime type
+	 * Get mime type based on file extension
 	 *
+	 * @param	string	$filename	Filename (with extension)
 	 * @return	string
 	 */
 	public static function getMimeType( $filename )
@@ -1531,6 +1646,7 @@ abstract class _File
 	 * Return a value pulled from php.ini in bytes
 	 *
 	 * @note	This function is intended to normalize values for things like post_max_size which could be -1, 0, 8383900, 8M, 1G, etc.
+	 * @param	string|int	$size	The size we want to convert to bytes (see note)
 	 * @return	float
 	 */
 	public static function returnBytes( $size )
@@ -1546,7 +1662,7 @@ abstract class _File
 		$last	= mb_strtolower( $size[ \strlen( $size ) - 1 ] );
 		
 		/* Convert $size to a number */
-		$size = intval( preg_replace( '/[^0-9]/', '', $size ) );
+		$size = \intval( preg_replace( '/[^0-9]/', '', $size ) );
 
 		/* Adjust value as necessary - note that we do not break intentionally */
 		switch( $last )
@@ -2491,4 +2607,4 @@ abstract class _File
         'zmm' => 'application/vnd.handheld-entertainment+xml'
 	);
 
-}
+}

system/File/FileSystem.php

@@ -11,7 +11,7 @@
 namespace IPS\File;
 
 /* To prevent PHP errors (extending class does not exist) revealing path */
-if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
+if ( !\defined( '\IPS\SUITE_UNIQUE_KEY' ) )
 {
 	header( ( isset( $_SERVER['SERVER_PROTOCOL'] ) ? $_SERVER['SERVER_PROTOCOL'] : 'HTTP/1.0' ) . ' 403 Forbidden' );
 	exit;
@@ -252,12 +252,12 @@ class _FileSystem extends \IPS\File
 				
 				if ( $this->contents === FALSE )
 				{
-					throw new \IPS\File\Exception( $this->container . '/' . $this->filename, \IPS\File\Exception::CANNOT_OPEN );
+					throw new \IPS\File\Exception( $this->container . '/' . $this->filename, \IPS\File\Exception::CANNOT_OPEN, $this->originalFilename );
 				}
 			}
 			else
 			{
-				throw new \IPS\File\Exception( $this->container . '/' . $this->filename, \IPS\File\Exception::DOES_NOT_EXIST );
+				throw new \IPS\File\Exception( $this->container . '/' . $this->filename, \IPS\File\Exception::DOES_NOT_EXIST, $this->originalFilename );
 			}
 		}
 
@@ -313,7 +313,7 @@ class _FileSystem extends \IPS\File
 				if( !@\copy( $this->temporaryFilePath, "{$folder}/{$this->filename}" ) )
 				{
 					\IPS\Log::log( "Could not copy file {$folder}/{$this->filename}" , 'FileSystem' );
-					throw new \IPS\File\Exception( "{$folder}/{$this->filename}", \IPS\File\Exception::CANNOT_COPY );
+					throw new \IPS\File\Exception( "{$folder}/{$this->filename}", \IPS\File\Exception::CANNOT_COPY, $this->originalFilename );
 				}
 			}
 			else
@@ -321,7 +321,7 @@ class _FileSystem extends \IPS\File
 				if( !@\rename( $this->temporaryFilePath, "{$folder}/{$this->filename}" ) )
 				{
 					\IPS\Log::log( "Could not move file {$folder}/{$this->filename}" , 'FileSystem' );
-					throw new \IPS\File\Exception( "{$folder}/{$this->filename}", \IPS\File\Exception::CANNOT_MOVE );
+					throw new \IPS\File\Exception( "{$folder}/{$this->filename}", \IPS\File\Exception::CANNOT_MOVE, $this->originalFilename );
 				}
 			}
 
@@ -334,7 +334,7 @@ class _FileSystem extends \IPS\File
 				if ( !@\file_put_contents( "{$folder}/{$this->filename}", $contents ) )
 				{					
 					\IPS\Log::log( "Could not write file {$folder}/{$this->filename}" , 'FileSystem' );
-					throw new \IPS\File\Exception( "{$folder}/{$this->filename}", \IPS\File\Exception::CANNOT_WRITE );
+					throw new \IPS\File\Exception( "{$folder}/{$this->filename}", \IPS\File\Exception::CANNOT_WRITE, $this->originalFilename );
 				}
 
 				@chmod( "{$folder}/{$this->filename}", \IPS\IPS_FILE_PERMISSION );
@@ -347,7 +347,7 @@ class _FileSystem extends \IPS\File
 		}
 		
 		/* Clear zend opcache if enabled */
-		if ( function_exists( 'opcache_invalidate' ) )
+		if ( \function_exists( 'opcache_invalidate' ) )
 		{
 			@opcache_invalidate( "{$folder}/{$this->filename}" );
 		}
@@ -367,7 +367,7 @@ class _FileSystem extends \IPS\File
 		$immediateCaller = debug_backtrace( DEBUG_BACKTRACE_IGNORE_ARGS, 1 );
 		$debug = array_map( function( $row ) {
 			return array_filter( $row, function( $key ) {
-				return in_array( $key, array( 'class', 'function', 'line' ) );
+				return \in_array( $key, array( 'class', 'function', 'line' ) );
 			}, ARRAY_FILTER_USE_KEY );
 		}, debug_backtrace( DEBUG_BACKTRACE_IGNORE_ARGS ) );
 		$this->log( "file_deletion", 'delete', $debug, 'log' );
@@ -377,7 +377,7 @@ class _FileSystem extends \IPS\File
 			$result = @unlink( "{$this->configuration['dir']}/{$this->container}/{$this->filename}" );
 
 			/* Clear zend opcache if enabled */
-			if ( function_exists( 'opcache_invalidate' ) )
+			if ( \function_exists( 'opcache_invalidate' ) )
 			{
 				@opcache_invalidate( "{$this->configuration['dir']}/{$this->container}/{$this->filename}" );
 			}
@@ -433,7 +433,7 @@ class _FileSystem extends \IPS\File
 				unlink( $file );
 
 				/* Clear zend opcache if enabled */
-				if( $skipClear === FALSE and function_exists( 'opcache_invalidate' ) )
+				if( $skipClear === FALSE and \function_exists( 'opcache_invalidate' ) )
 				{
 					@opcache_invalidate( $file );
 				}
@@ -451,44 +451,75 @@ class _FileSystem extends \IPS\File
 	}
 	
 	/**
-	 * Append more contents in a chunked upload
+	 * Initiate a chunked upload
 	 *
-	 * @param	string		$filename	The filename that was created
-	 * @param	string		$chunk		The temp filename for the new chunk to append
-	 * @param	string|NULL	$rename		If a value is provided, rename the file after appending (will be provided for the last chunk)
-	 * @param	string|NULL	$container  Key to identify container for storage
-	 * @return	string
+	 * @param	string		$filename	The desired filename
+	 * @param	string|null	$container	Key to identify container for storage
+	 * @param	bool		$obscure		Controls if an md5 hash should be added to the filename
+	 * @return	mixed					A reference to be passed to chunkProcess() and chunkFinish()
 	 * @throws	\RuntimeException
 	 */
-	public function chunkAppend( $filename, $chunk, $rename=NULL, $container=NULL )
+	public function chunkInit( $filename, $container = '', $obscure = TRUE )
 	{
+		/* If we don't have a chunk folder, create one */
+		$tempFolder = $this->getFolder( 'chunks' );
+
+		/* Get the final folder name */
 		$folder = $this->getFolder( $container );
-		
-		$file = fopen( "{$this->configuration['dir']}/{$folder}/{$filename}", 'ab' );
-		$_chunk = fopen( $chunk, 'rb' );
-		while ( $buffer = fread( $_chunk, 4096 ) )
-		{
-			\fwrite( $file, $buffer );
-		}
-		fclose( $file );
-		@unlink( $chunk );
 
-		/* Clear zend opcache if enabled */
-		if ( function_exists( 'opcache_invalidate' ) )
-		{
-			@opcache_invalidate( "{$this->configuration['dir']}/{$folder}/{$filename}" );
-		}
-
-		if ( $rename )
-		{
-			$newFileName = static::obscureFilename( $rename );
-			rename( "{$this->configuration['dir']}/{$folder}/{$filename}", "{$this->configuration['dir']}/{$folder}/{$newFileName}" );
-			return "{$folder}/{$newFileName}";
-		}
-
-		return "{$folder}/{$filename}";
+		$return = array( 'temp' => "{$this->configuration['dir']}/{$tempFolder}/chunk-" . md5( uniqid() ) . '.txt', 'final' => "{$folder}/" . ( $obscure ? static::obscureFilename( $filename ) : $filename ) );
+		touch( $return['temp'] );
+		return $return;
 	}
 	
+	/**
+	 * Append more contents in a chunked upload
+	 *
+	 * @param	mixed	$ref							The reference for this upload as returned by chunkInit() 
+	 * @param	string	$temporaryFileOrContents		The contents to write, or the path to the temporary file on disk with the contents
+	 * @param	int		$chunkNumber					Which chunk this is (0 for the first chunk, 1 for the second, etc)
+	 * @param	bool	$isContents					If TRUE, $temporaryFileOrContents is treated as raw contents. If FALSE, is path to file
+	 * @return	mixed								Updated reference fore future chunkProcess() calls and chunkFinish()
+	 * @throws	\RuntimeException
+	 */
+	public function chunkProcess( $ref, $temporaryFileOrContents, $chunkNumber, $isContents = FALSE )
+	{
+		$file = fopen( $ref['temp'], 'ab' );
+		if ( $isContents )
+		{
+			\fwrite( $file, $temporaryFileOrContents );
+		}
+		else
+		{
+			$_chunk = fopen( $temporaryFileOrContents, 'rb' );
+			while ( $buffer = fread( $_chunk, 4096 ) )
+			{
+				\fwrite( $file, $buffer );
+			}
+			@unlink( $temporaryFileOrContents );
+		}
+		fclose( $file );
+		
+		if ( \function_exists( 'opcache_invalidate' ) )
+		{
+			@opcache_invalidate( $ref['temp'] );
+		}
+		
+		return $ref;
+	}
+	
+	/**
+	 * Finalize a chunked upload
+	 *
+	 * @param	mixed	$ref		The reference for this upload as returned by chunkInit() 
+	 * @return	string			The file's reference which can be passed to \IPS\File::get() as the URL
+	 * @throws	\RuntimeException
+	 */
+	public function chunkFinish( $ref )
+	{
+		rename( $ref['temp'], "{$this->configuration['dir']}/{$ref['final']}" );
+		return $ref['final'];
+	}
 	
 	/* !File System Utility Methods */
 	
@@ -599,10 +630,16 @@ class _FileSystem extends \IPS\File
 	/**
 	 * Get filesize (in bytes)
 	 *
-	 * @return	string
+	 * @return	string|bool
 	 */
 	public function filesize()
 	{
-		return file_exists( $this->configuration['dir'] . '/' . $this->container . '/' . $this->filename ) ? @filesize( $this->configuration['dir'] . '/' . $this->container . '/' . $this->filename ) : NULL;
+		if( $this->_cachedFilesize !== NULL )
+		{
+			return $this->_cachedFilesize;
+		}
+
+		$this->_cachedFilesize = file_exists( $this->configuration['dir'] . '/' . $this->container . '/' . $this->filename ) ? @filesize( $this->configuration['dir'] . '/' . $this->container . '/' . $this->filename ) : FALSE;
+		return $this->_cachedFilesize;
 	}
 }

system/File/Ftp.php

@@ -11,7 +11,7 @@
 namespace IPS\File;
 
 /* To prevent PHP errors (extending class does not exist) revealing path */
-if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
+if ( !\defined( '\IPS\SUITE_UNIQUE_KEY' ) )
 {
 	header( ( isset( $_SERVER['SERVER_PROTOCOL'] ) ? $_SERVER['SERVER_PROTOCOL'] : 'HTTP/1.0' ) . ' 403 Forbidden' );
 	exit;
@@ -47,15 +47,15 @@ class _Ftp extends \IPS\File
 	 */
 	public static function testSettings( &$values )
 	{		
-		if ( !function_exists( 'ftp_connect' ) )
+		if ( !\function_exists( 'ftp_connect' ) )
 		{
 			throw new \BadFunctionCallException( 'ftp_err_no_ext' );
 		}
-		elseif ( $values['ftp_details']['protocol'] === 'ssl_ftp' and !function_exists( 'ftp_ssl_connect' ) )
+		elseif ( $values['ftp_details']['protocol'] === 'ssl_ftp' and !\function_exists( 'ftp_ssl_connect' ) )
 		{
 			throw new \BadFunctionCallException( 'ftp_err_no_ssl' );
 		}
-		elseif ( $values['ftp_details']['protocol'] === 'sftp' and !function_exists( 'ssh2_connect' ) )
+		elseif ( $values['ftp_details']['protocol'] === 'sftp' and !\function_exists( 'ssh2_connect' ) )
 		{
 			throw new \BadFunctionCallException( 'ftp_err_no_sftp' );
 		}
@@ -156,7 +156,7 @@ class _Ftp extends \IPS\File
 		\file_put_contents( $tmpFileName, $contents );
 				
 		/* Move into the correct folder */		
-		if( !in_array( $this->container, $this->ftp()->ls() ) )
+		if( !\in_array( $this->container, $this->ftp()->ls() ) )
 		{
 			/* Make dir */
 			$this->ftp()->mkdir( $this->container );
@@ -201,7 +201,7 @@ class _Ftp extends \IPS\File
 		$immediateCaller = debug_backtrace( DEBUG_BACKTRACE_IGNORE_ARGS, 1 );
 		$debug = array_map( function( $row ) {
 			return array_filter( $row, function( $key ) {
-				return in_array( $key, array( 'class', 'function', 'line' ) );
+				return \in_array( $key, array( 'class', 'function', 'line' ) );
 			}, ARRAY_FILTER_USE_KEY );
 		}, debug_backtrace( DEBUG_BACKTRACE_IGNORE_ARGS ) );
 		$this->log( "file_deletion", 'delete', $debug, 'log' );
@@ -242,8 +242,8 @@ class _Ftp extends \IPS\File
 
 		foreach( $pathBits as $piece )
 		{
-			$pathToCheck = count( $reconstructed ) ? implode( '/', $reconstructed ) : '.';
-			if( $this->ftp()->ls( $pathToCheck ) === FALSE OR !in_array( $piece, $this->ftp()->ls( $pathToCheck ) ) )
+			$pathToCheck = \count( $reconstructed ) ? implode( '/', $reconstructed ) : '.';
+			if( $this->ftp()->ls( $pathToCheck ) === FALSE OR !\in_array( $piece, $this->ftp()->ls( $pathToCheck ) ) )
 			{
 				return FALSE;
 			}
@@ -366,10 +366,15 @@ class _Ftp extends \IPS\File
 	/**
 	 * Get filesize (in bytes)
 	 *
-	 * @return	string
+	 * @return	string|bool
 	 */
 	public function filesize()
 	{
+		if( $this->_cachedFilesize !== null )
+		{
+			return $this->_cachedFilesize;
+		}
+
 		try
 		{
 			$this->ftp()->chdir( $this->configuration['ftp_details']['path'] );
@@ -383,12 +388,14 @@ class _Ftp extends \IPS\File
 
 			$this->ftp()->chdir( $this->configuration['ftp_details']['path'] );
 
-			return $size ?: parent::filesize();
+			$this->_cachedFilesize = $size ?: parent::filesize();
 		}
 		catch( \IPS\Ftp\Exception $e )
 		{
-			return FALSE;
+			$this->_cachedFilesize = FALSE;
 		}
+
+		return $this->_cachedFilesize;
 	}
 
 	/**

system/File/Iterator.php

@@ -11,7 +11,7 @@
 namespace IPS\File;
 
 /* To prevent PHP errors (extending class does not exist) revealing path */
-if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
+if ( !\defined( '\IPS\SUITE_UNIQUE_KEY' ) )
 {
 	header( ( isset( $_SERVER['SERVER_PROTOCOL'] ) ? $_SERVER['SERVER_PROTOCOL'] : 'HTTP/1.0' ) . ' 403 Forbidden' );
 	exit;
@@ -41,6 +41,12 @@ class _Iterator extends \IteratorIterator implements \Countable
 	 * @brief	Used to restore 'real' names when filenames cleaned (eg adh1029_file.php back to fi^le.php)
 	 */
 	protected $replaceNameField;
+
+	/**
+	 * @brief	Used to pre-cache the filesize value if available
+	 */
+	protected $fileSizeField;
+
 	
 	/**
 	 * Constructor
@@ -48,15 +54,18 @@ class _Iterator extends \IteratorIterator implements \Countable
 	 * @param	Traversable $iterator			The iterator
 	 * @param	string		$storageExtension	The storage extension
 	 * @param	string|NULL	$urlField			If passed a string, will look for an element with that key in the data returned from the iterator
+	 * @param	bool		$fileUrlsOnly		Only return the file URL instead of the file object
 	 * @param	string|NULL	$replaceNameField	If passed a string, it will replace the originalFilename with the data in the array. Used to restore 'real' names when filenames cleaned (eg adh1029_file.php back to fi^le.php)
+	 * @param 	string		$fileSizeField		Field to use to pre-cache the filesize
 	 * @return	void
 	 */
-	public function __construct( \Traversable $iterator, $storageExtension, $urlField=NULL, $fileUrlsOnly=FALSE, $replaceNameField=NULL )
+	public function __construct( \Traversable $iterator, $storageExtension, $urlField=NULL, $fileUrlsOnly=FALSE, $replaceNameField=NULL, $fileSizeField=NULL )
 	{
 		$this->storageExtension = $storageExtension;
 		$this->urlField = $urlField;
 		$this->fileUrlsOnly = $fileUrlsOnly;
 		$this->replaceNameField = $replaceNameField;
+		$this->fileSizeField = $fileSizeField;
 		return parent::__construct( $iterator );
 	}
 	
@@ -74,9 +83,10 @@ class _Iterator extends \IteratorIterator implements \Countable
 			
 			if ( $this->urlField )
 			{
-				if ( !is_string( $this->urlField ) and is_callable( $this->urlField ) )
+				if ( !\is_string( $this->urlField ) and \is_callable( $this->urlField ) )
 				{
-					$urlField = call_user_func( $this->urlField, $data );
+					$urlFieldCallback = $this->urlField;
+					$urlField = $urlFieldCallback( $data );
 				}
 				else
 				{
@@ -84,7 +94,8 @@ class _Iterator extends \IteratorIterator implements \Countable
 				}
 			}
 
-			$obj = \IPS\File::get( $this->storageExtension, $urlField ? $data[ $urlField ] : $data );
+			$fileSize = ( $this->fileSizeField AND !empty( $data[ $this->fileSizeField ] ) ) ? $data[ $this->fileSizeField ] : ( $this->fileSizeField === FALSE ? FALSE : NULL );
+			$obj = \IPS\File::get( $this->storageExtension, $urlField ? $data[ $urlField ] : $data, $fileSize );
 			
 			if ( $this->replaceNameField and ! empty( $data[ $this->replaceNameField ] ) )
 			{