sydney/IPB
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 4.5.0

Browse Source
This commit is contained in:
Neo committed 2025-12-19 05:51:58 -08:00
1 parent 1a0c7fd3c2
commit f79dcf067a
3791 files changed
+248032 -76372
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

system/File/Amazon.php
+323 -63
View File
@@ -11,7 +11,7 @@
namespace IPS\File;
/* To prevent PHP errors (extending class does not exist) revealing path */
if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
if ( !\defined( '\IPS\SUITE_UNIQUE_KEY' ) )
{
header( ( isset( $_SERVER['SERVER_PROTOCOL'] ) ? $_SERVER['SERVER_PROTOCOL'] : 'HTTP/1.0' ) . ' 403 Forbidden' );
exit;
@@ -22,7 +22,6 @@ if ( !defined( '\IPS\SUITE_UNIQUE_KEY' ) )
*/
class _Amazon extends \IPS\File
{
/**
* An array of ( configuration_id => array( ext, etx ) ) extensions that require gzip versions storing
* Looks up $storageExtensions of extension classes
@@ -136,10 +135,11 @@ class _Amazon extends \IPS\File
public static function moveCheck( $configuration, $oldConfiguration )
{
foreach( array( 'bucket', 'bucket_path' ) as $field )
if ( $configuration[ $field ] !== $oldConfiguration[ $field ] )
{
return TRUE;
if ( $configuration[ $field ] !== $oldConfiguration[ $field ] )
{
return TRUE;
}
}
return FALSE;
@@ -157,6 +157,21 @@ class _Amazon extends \IPS\File
}
/* !File Handling */
/**
* @brief Does this storage method support chunked uploads?
*/
public static $supportsChunking = TRUE;
/**
* @brief Min chunk size (in MB)
*/
public static $minChunkSize = 5;
/**
* @brief Max chunk size (in MB)
*/
public static $maxChunkSize = 5000;
/**
* Constructor
@@ -169,6 +184,24 @@ class _Amazon extends \IPS\File
$this->container = 'monthly_' . date( 'Y' ) . '_' . date( 'm' );
parent::__construct( $configuration );
}
/**
* Get output for API
*
* @param \IPS\Member|NULL $authorizedMember The member making the API request or NULL for API Key / client_credentials
* @return array
* @apiresponse string name The filename
* @apiresponse string url URL to where file is stored
* @apiresponse int size Filesize in bytes
*/
public function apiOutput( \IPS\Member $authorizedMember = NULL )
{
return array(
'name' => $this->originalFilename,
'url' => ( $this->isPrivate() AND $signedUrl = $this->generateTemporaryDownloadUrl() ) ? (string) $signedUrl : (string) $this->url,
'size' => $this->filesize()
);
}
/**
* Fetch the gzip extensions specific for $this->configurationId
@@ -186,7 +219,7 @@ class _Amazon extends \IPS\File
$bits = explode( '_', $this->storageExtension );
$class = '\IPS\\' . $bits[0] . '\extensions\core\FileStorage\\' . $bits[1];
if ( isset( $class::$storeGzipExtensions ) and is_array( $class::$storeGzipExtensions ) and count( $class::$storeGzipExtensions ) )
if ( isset( $class::$storeGzipExtensions ) and \is_array( $class::$storeGzipExtensions ) and \count( $class::$storeGzipExtensions ) )
{
static::$gzipExtensions[ $this->storageExtension ] = $class::$storeGzipExtensions;
}
@@ -229,7 +262,7 @@ class _Amazon extends \IPS\File
public function needsGzipVersion()
{
/* Use filename and not originalFilename so only true .js and .css files are checked, and not renamed uploads */
return in_array( mb_substr( $this->filename, mb_strrpos( $this->filename, '.' ) + 1 ), $this->getGzipExtensions() );
return \in_array( mb_substr( $this->filename, mb_strrpos( $this->filename, '.' ) + 1 ), $this->getGzipExtensions() );
}
/**
@@ -272,17 +305,7 @@ class _Amazon extends \IPS\File
if ( $response->httpResponseCode != 200 )
{
try
{
$xml = $response->decodeXml();
if ( isset( $xml->Code ) and isset( $xml->Message ) )
{
\IPS\Log::log( $xml->Code . ' ' . $xml->Message . ' ' . $path, 'Amazon' );
}
}
catch( \Exception $e ) { }
throw new \IPS\File\Exception( $path, \IPS\File\Exception::CANNOT_WRITE );
throw new \IPS\File\Exception( $path, \IPS\File\Exception::CANNOT_WRITE, $this->originalFilename, $this->getExtraMessage( $response, $path ), $this->getErrorInformation( $response ) );
}
/* Write the gzip version */
@@ -292,17 +315,7 @@ class _Amazon extends \IPS\File
if ( $response->httpResponseCode != 200 )
{
try
{
$xml = $response->decodeXml();
if ( isset( $xml->Code ) and isset( $xml->Message ) )
{
\IPS\Log::log( $xml->Code . ' ' . $xml->Message . ' ' . $path, 'Amazon' );
}
}
catch( \Exception $e ) { }
throw new \IPS\File\Exception( $path . '.gz', \IPS\File\Exception::CANNOT_WRITE );
throw new \IPS\File\Exception( $path . '.gz', \IPS\File\Exception::CANNOT_WRITE, $this->originalFilename, $this->getExtraMessage( $response, $path . '.gz' ), $this->getErrorInformation( $response ) );
}
}
}
@@ -320,11 +333,11 @@ class _Amazon extends \IPS\File
$response = static::makeRequest( $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}", 'GET', $this->configuration, $this->configurationId );
if ( $response->httpResponseCode == 404 )
{
throw new \IPS\File\Exception( $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}", \IPS\File\Exception::DOES_NOT_EXIST );
throw new \IPS\File\Exception( $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}", \IPS\File\Exception::DOES_NOT_EXIST, $this->originalFilename, $this->getExtraMessage( $response, $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}" ), $this->getErrorInformation( $response ) );
}
elseif( $response->httpResponseCode == 403 )
{
throw new \IPS\File\Exception( $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}", \IPS\File\Exception::CANNOT_COPY );
throw new \IPS\File\Exception( $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}", \IPS\File\Exception::CANNOT_COPY, $this->originalFilename, $this->getExtraMessage( $response, $this->container ? "{$this->container}/{$this->filename}" : "{$this->filename}" ), $this->getErrorInformation( $response ) );
}
else
{
@@ -346,7 +359,7 @@ class _Amazon extends \IPS\File
$debug = array_map( function( $row ) {
return array_filter( $row, function( $key ) {
return in_array( $key, array( 'class', 'function', 'line' ) );
return \in_array( $key, array( 'class', 'function', 'line' ) );
}, ARRAY_FILTER_USE_KEY );
}, debug_backtrace( DEBUG_BACKTRACE_IGNORE_ARGS ) );
@@ -388,46 +401,214 @@ class _Amazon extends \IPS\File
* @return void
*/
public function deleteContainer( $container )
{
/* Add it the s3 table */
try
{
/* Is it already being deleted? */
\IPS\Db::i()->select( '*', 'core_s3_deletions', array( 's3_container=?', $container ) )->first();
/* Update the added time so we delete everything < that time() */
\IPS\Db::i()->update( 'core_s3_deletions', array( 's3_added' => time(), 's3_last_cycle' => 0, 's3_marker' => '' ), array( 's3_container=?', $container ) );
}
catch( \UnderflowException $e )
{
/* It doesn't already exist, so add it */
\IPS\Db::i()->insert( 'core_s3_deletions', array(
's3_container' => $container,
's3_added' => time(),
's3_last_cycle' => 0
) );
/* Turn on task */
\IPS\Db::i()->update( 'core_tasks', array( 'enabled' => 1 ), array( '`key`=?', 's3_delete' ) );
/* Log deletion request */
$realContainer = $this->container;
$this->container = $container;
$this->log( "container_deletion", 'delete', NULL, 'log' );
$this->container = $realContainer;
}
}
/**
* Return all keys that match the prefix
* @note Future me, you're welcome: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html
*
* @param string $prefix Prefix to match when fetching keys (such as css_built_)
* @param int $limit Number of keys to return in any one go
* @param string $delimiter Group keys by delimiter, so if you want to get all css_built_* containers: prefix=css_built_ delimiter=/ will return all folder names matching
* @param string $marker Pass in a marker to fetch the results from if result set truncated (this can be the last key fetched)
* @return array
*/
public function getContainerKeys( $prefix = '', $limit = 50, $delimiter = '', $marker='' )
{
$_strip = array( '_strip_querystring' => TRUE, 'bucket_path' => NULL );
if ( $this->configuration['bucket_path'] )
{
$container = $this->configuration['bucket_path'] . '/' . $container;
$prefix = $this->configuration['bucket_path'] . '/' . $prefix;
}
$response = static::makeRequest( "?prefix=" . urlencode( $container . "/" ), 'GET', array_merge( $this->configuration, $_strip ), $this->configurationId );
if ( $marker and $this->configuration['bucket_path'] )
{
$marker = $this->configuration['bucket_path'] . '/' . $marker;
}
$response = static::makeRequest( '', 'GET', array_merge( $this->configuration, $_strip ), $this->configurationId, NULL, NULL, FALSE, FALSE, array( "prefix" => $prefix, 'max-keys' => $limit, 'delimiter' => $delimiter, 'marker' => $marker ) );
/* Parse XML document */
$document = \IPS\Xml\SimpleXML::loadString( $response );
/* Loop over dom document */
foreach( $document->Contents as $result )
$keys = array();
if ( $delimiter )
{
if ( $this->configuration['bucket_path'] )
/* Loop over dom document */
foreach( $document->CommonPrefixes as $result )
{
$result->Key = mb_substr( $result->Key, ( mb_strlen( $this->configuration['bucket_path'] ) + 1 ) );
if ( $this->configuration['bucket_path'] )
{
$keys[] = trim( mb_substr( $result->Prefix, ( mb_strlen( $this->configuration['bucket_path'] ) + 1 ) ), '/' );
}
else
{
$keys[] = trim( $result->Prefix, '/' );
}
}
static::makeRequest( $result->Key, 'DELETE', $this->configuration, $this->configurationId );
}
/* Log deletion request */
$realContainer = $this->container;
$this->container = $container;
$this->log( "container_deletion", 'delete', NULL, 'log' );
$this->container = $realContainer;
else
{
/* Loop over dom document */
foreach( $document->Contents as $result )
{
if ( $this->configuration['bucket_path'] )
{
$keys[ mb_substr( $result->Key, ( mb_strlen( $this->configuration['bucket_path'] ) + 1 ) ) ] = strtotime( (string) $result->LastModified );
}
else
{
$keys[ trim( $result->Key, '/' ) ] = strtotime( (string) $result->LastModified );
}
}
}
return $keys;
}
/**
* @brief Cached filesize
* Delete with a known key
* Bit of a cheeky bypass right into AWS S3 guts
*
* @param string $key Key to delete, bucket_path added automatically if needed
* @return void
*/
protected $_cachedFilesize = NULL;
public function deleteByKey( $key )
{
if ( $this->configuration['bucket_path'] )
{
$key = $this->configuration['bucket_path'] . '/' . $key;
}
$_strip = array( '_strip_querystring' => TRUE, 'bucket_path' => NULL );
try
{
static::makeRequest( $key, 'DELETE', array_merge( $this->configuration, $_strip ), $this->configurationId );
\IPS\Db::i()->insert( 'core_file_logs', array (
'log_action' => 'delete',
'log_type' => 'log',
'log_configuration_id' => $this->configurationId,
'log_method' => static::$storageConfigurations[ $this->configurationId ]['method'],
'log_filename' => $key,
'log_url' => '',
'log_container' => '',
'log_msg' => 'file_deletion',
'log_date' => time(),
'log_data' => NULL
) );
}
catch( \IPS\Http\Request\Exception $e )
{
/* If there was a problem deleting the file, don't stop code execution just because of that */
$this->log( 'HTTP_ERROR_DELETE_FILE', 'delete', array( $e->getCode(), $e->getMessage() ), 'error' );
}
}
/**
* Initiate a chunked upload
*
* @param string $filename The desired filename
* @param string|null $container Key to identify container for storage
* @param bool $obscure Controls if an md5 hash should be added to the filename
* @return mixed A reference to be passed to chunkProcess() and chunkFinish()
* @throws \RuntimeException
*/
public function chunkInit( $filename, $container = '', $obscure = TRUE )
{
$container = trim( $container, '/' );
$filename = $obscure ? static::obscureFilename( $filename ) : $filename;
$path = $this->container ? "{$this->container}/{$filename}" : "{$filename}";
$r = static::makeRequest( $path, 'POST', $this->configuration, $this->configurationId, NULL, NULL, FALSE, FALSE, array( 'uploads' => '' ) );
$uploadId = (string) $r->decodeXml()->UploadId;
return array(
'path' => $path,
'uploadId' => $uploadId
);
}
/**
* Append more contents in a chunked upload
*
* @param mixed $ref The reference for this upload as returned by chunkInit()
* @param string $temporaryFileOrContents The contents to write, or the path to the temporary file on disk with the contents
* @param int $chunkNumber Which chunk this is (0 for the first chunk, 1 for the second, etc)
* @param bool $isContents If TRUE, $temporaryFileOrContents is treated as raw contents. If FALSE, is path to file
* @return mixed Updated reference fore future chunkProcess() calls and chunkFinish()
* @throws \RuntimeException
*/
public function chunkProcess( $ref, $temporaryFileOrContents, $chunkNumber, $isContents = FALSE )
{
$r = static::makeRequest( $ref['path'], 'PUT', $this->configuration, $this->configurationId, $isContents ? $temporaryFileOrContents : file_get_contents( $temporaryFileOrContents ), NULL, FALSE, NULL, array( 'partNumber' => ++$chunkNumber, 'uploadId' => $ref['uploadId'] ) );
$ref['etags'][ $chunkNumber ] = trim( $r->httpHeaders['ETag'], '"' );
if ( !$isContents )
{
@unlink( $temporaryFileOrContents );
}
return $ref;
}
/**
* Finalize a chunked upload
*
* @param mixed $ref The reference for this upload as returned by chunkInit()
* @return string The file's reference which can be passed to \IPS\File::get() as the URL
* @throws \RuntimeException
*/
public function chunkFinish( $ref )
{
$xml = \IPS\Xml\SimpleXML::create( 'CompleteMultipartUpload', 'http://s3.amazonaws.com/doc/2006-03-01/' );
foreach ( $ref['etags'] as $partNumber => $eTag )
{
$part = $xml->addChild( 'Part' );
$part->addChild( 'ETag', $eTag );
$part->addChild( 'PartNumber', $partNumber );
}
static::makeRequest( $ref['path'], 'POST', $this->configuration, $this->configurationId, $xml->asXML(), NULL, FALSE, NULL, array( 'uploadId' => $ref['uploadId'] ) );
return $ref['path'];
}
/**
* Get filesize (in bytes)
*
* @return string
* @return string|bool
*/
public function filesize()
{
@@ -480,7 +661,7 @@ class _Amazon extends \IPS\File
$xml = $url->request()->get()->decodeXml();
if ( !isset( $xml->Region ) )
{
throw new \IPS\File\Exception( $fileUrl, \IPS\File\Exception::MISSING_REGION );
throw new \IPS\File\Exception( $fileUrl, \IPS\File\Exception::MISSING_REGION, $this->originalFilename );
}
$this->configuration['region'] = (string) $xml->Region;
@@ -500,11 +681,14 @@ class _Amazon extends \IPS\File
* @param array $configuration The configuration for this instance
* @param int $configurationId The configuration ID
* @param string|null $content The content to send
* @param string|null $storageExtension Storage extension
* @param bool $skipExtraChecks Skips the endpoint check (to prevent infinite looping)
* @param bool $isPrivate This can be set to true to access/store private files (i.e. that are not publicly readable)
* @param array $queryString Array of key => value pairs
* @return \IPS\Http\Response
* @throws \IPS\Http\Request\Exception
*/
protected static function makeRequest( $uri, $verb, $configuration, $configurationId, $content=NULL, $storageExtension=NULL, $skipExtraChecks=FALSE, $isPrivate=false )
protected static function makeRequest( $uri, $verb, $configuration, $configurationId, $content=NULL, $storageExtension=NULL, $skipExtraChecks=FALSE, $isPrivate=false, $queryString=array() )
{
/* Amazon requires filename characters to be properly encoded - let's urlencode the filename here */
$uriPieces = explode( '/', $uri );
@@ -512,7 +696,12 @@ class _Amazon extends \IPS\File
$uri = ltrim( implode( '/', $uriPieces ) . '/' . rawurlencode( $filename ), '/' );
/* Build a request */
$request = \IPS\Http\Url::external( static::buildBaseUrl( $configuration ) . $uri )->request( \IPS\LONG_REQUEST_TIMEOUT, NULL, FALSE ); # Amazon will send a 301 header code, but no Location header, if we need to try another endpoint
$url = \IPS\Http\Url::external( static::buildBaseUrl( $configuration ) . $uri );
if ( $queryString )
{
$url = $url->setQueryString( $queryString );
}
$request = $url->request( \IPS\VERY_LONG_REQUEST_TIMEOUT, NULL, FALSE ); # Amazon will send a 301 header code, but no Location header, if we need to try another endpoint
/* When using virtual hosted–style buckets with SSL, the SSL wild card certificate only matches buckets that do not contain periods. To work around this, use HTTP or write your own certificate verification logic. @link http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html */
if ( \IPS\Request::i()->isSecure() and mb_strstr( $configuration['bucket'], '.' ) )
@@ -525,8 +714,11 @@ class _Amazon extends \IPS\File
$headers = array(
'Content-Type' => \IPS\File::getMimeType( $mimeUri ),
'Content-MD5' => base64_encode( md5( $content, TRUE ) ),
'X-Amz-Acl' => ( $isPrivate ? 'bucket-owner-read' : 'public-read' )
);
if ( $isPrivate !== NULL )
{
$headers['X-Amz-Acl'] = ( $isPrivate ? 'bucket-owner-read' : 'public-read' );
}
if ( $mimeUri !== $uri )
{
$headers['Content-Encoding'] = 'gzip';
@@ -562,7 +754,6 @@ class _Amazon extends \IPS\File
}
/* Sign the request */
$queryString = array();
$authorization = static::signature( $configuration, $verb, $uri, $headers, $queryString, $content );
$headers['Authorization'] = $authorization;
unset( $headers['Host'] );
@@ -653,7 +844,7 @@ class _Amazon extends \IPS\File
* @param array $configuration The configuration for this instance
* @param string $verb The HTTP verb that will be used in the request
* @param string $uri The URI (relative to the bucket)
* @param array $header The request headers as an array
* @param array $headers The request headers as an array
* @param array $queryString The query string as an array
* @param string|null $content The content to send
* @param bool $signatureIsForQueryString If true, signature will be generated for query string. If false, header.
@@ -668,7 +859,7 @@ class _Amazon extends \IPS\File
$contentSha256 = ( $signatureIsForQueryString and !$content ) ? 'UNSIGNED-PAYLOAD' : hash( 'sha256', $content );
/* Figure out the canonical headers and query string */
$headers['Host'] = ( isset( $configuration['endpoint'] ) ? $configuration['endpoint'] : "s3.amazonaws.com" );
$headers['Host'] = $configuration['bucket'] . '.' . ( isset( $configuration['endpoint'] ) ? $configuration['endpoint'] : "s3.amazonaws.com" );
if ( $signatureIsForQueryString )
{
$queryString['X-Amz-Algorithm'] = 'AWS4-HMAC-SHA256';
@@ -693,13 +884,13 @@ class _Amazon extends \IPS\File
/* Task 1: Create a Canonical Request */
$canonicalRequest = implode( "\n", array(
mb_strtoupper( $verb ),
'/' . $configuration['bucket'] . static::bucketPath( $configuration ) . '/' . ltrim( $uri, '/' ),
static::bucketPath( $configuration ) . '/' . ltrim( $uri, '/' ),
http_build_query( $queryString, '', '&', PHP_QUERY_RFC3986 ),
$canonicalHeadersAsString,
implode( ';', array_map( 'mb_strtolower', array_keys( $headers ) ) ),
$contentSha256
) );
/* Task 2: Create a String to Sign */
$stringToSign = implode( "\n", array(
'AWS4-HMAC-SHA256',
@@ -734,9 +925,8 @@ class _Amazon extends \IPS\File
public static function buildBaseUrl( $configuration )
{
return (
\IPS\Request::i()->isSecure() ? "https" : "http" ) . "://"
\IPS\Request::i()->isSecure() ? "https" : "http" ) . "://{$configuration['bucket']}."
. ( isset( $configuration['endpoint'] ) ? $configuration['endpoint'] : "s3.amazonaws.com" )
. "/{$configuration['bucket']}"
. static::bucketPath( $configuration )
. '/';
}
@@ -829,4 +1019,74 @@ class _Amazon extends \IPS\File
return $results;
}
/**
* Retrieve any additional error information
*
* @param \IPS\Http\Response $response The response object
* @param string $path File path if available
* @return string|null
*/
protected function getExtraMessage( $response, $path='' )
{
try
{
$xml = $response->decodeXml();
if ( isset( $xml->Code ) )
{
switch( $xml->Code )
{
case 'RequestTimeTooSkewed':
return 's3-RequestTimeTooSkewed';
break;
case 'AccessDenied':
case 'AccountProblem':
case 'AllAccessDisabled':
case 'InvalidPayer':
case 'NotSignedUp':
return 's3-AccountProblem';
break;
case 'EntityTooSmall':
case 'MissingRequestBodyError':
return 's3-FileSizeSmall';
break;
case 'EntityTooLarge':
case 'MaxMessageLengthExceeded':
return 's3-FileSizeLarge';
break;
default:
return 's3-GenericError';
break;
}
}
}
catch( \Exception $e ) { }
return null;
}
/**
* Get the error code and message from Amazon to log
*
* @param \IPS\Http\Response $response The response object
* @return string|null
*/
protected function getErrorInformation( $response )
{
try
{
$xml = $response->decodeXml();
if ( isset( $xml->Code ) and isset( $xml->Message ) )
{
return $xml->Code . ': ' . $xml->Message;
}
}
catch( \Exception $e ) { }
return null;
}
}