Switch to s3 multipart uploads for backups

This commit is contained in:
Matthew Penner 2020-10-31 17:44:20 -06:00
parent 23d2352a9b
commit 85af073438
7 changed files with 137 additions and 41 deletions

View file

@ -0,0 +1,131 @@
<?php
namespace Pterodactyl\Http\Controllers\Api\Remote\Backups;
use Carbon\CarbonImmutable;
use Illuminate\Http\Request;
use Illuminate\Http\JsonResponse;
use League\Flysystem\AwsS3v3\AwsS3Adapter;
use Pterodactyl\Http\Controllers\Controller;
use Pterodactyl\Extensions\Backups\BackupManager;
use Pterodactyl\Repositories\Eloquent\BackupRepository;
class BackupRemoteUploadController extends Controller
{
// I would use 1024 but I'm unsure if AWS or other S3 servers,
// use SI gigabyte (base 10), or the proper IEC gibibyte (base 2).
// const PART_SIZE = 5 * 1000 * 1000 * 1000;
const PART_SIZE = 5 * 1024 * 1024 * 1024;
/**
* @var \Pterodactyl\Repositories\Eloquent\BackupRepository
*/
private $repository;
/**
* @var \Pterodactyl\Extensions\Backups\BackupManager
*/
private $backupManager;
/**
* BackupRemoteUploadController constructor.
*
* @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository
* @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager
*/
public function __construct(BackupRepository $repository, BackupManager $backupManager)
{
$this->repository = $repository;
$this->backupManager = $backupManager;
}
/**
* ?
*
* @param \Illuminate\Http\Request $request
* @param string $backup
*
* @return \Illuminate\Http\JsonResponse
*
* @throws \Pterodactyl\Exceptions\Repository\RecordNotFoundException
* @throws \Exception
*/
public function __invoke(Request $request, string $backup)
{
$size = $request->query('size', null);
if ($size === null) {
return new JsonResponse([], JsonResponse::HTTP_BAD_REQUEST);
}
/** @var \Pterodactyl\Models\Backup $model */
$model = $this->repository->findFirstWhere([[ 'uuid', '=', $backup ]]);
// Prevent backups that have already been completed from trying to
// be uploaded again.
if (! is_null($model->completed_at)) {
return new JsonResponse([], JsonResponse::HTTP_CONFLICT);
}
// Ensure we are using the S3 adapter.
$adapter = $this->backupManager->adapter();
if (! $adapter instanceof AwsS3Adapter) {
return new JsonResponse([], JsonResponse::HTTP);
}
$path = sprintf('%s/%s.tar.gz', $model->server->uuid, $model->uuid);
$client = $adapter->getClient();
$result = $client->execute($client->getCommand('CreateMultipartUpload', [
'Bucket' => $adapter->getBucket(),
'Key' => $path,
'ContentType' => 'application/x-gzip',
]));
$uploadId = $result->get('UploadId');
$completeMultipartUpload = $client->createPresignedRequest(
$client->getCommand('CompleteMultipartUpload', [
'Bucket' => $adapter->getBucket(),
'Key' => $path,
'ContentType' => 'application/x-gzip',
'UploadId' => $uploadId,
]),
CarbonImmutable::now()->addMinutes(30)
);
$abortMultipartUpload = $client->createPresignedRequest(
$client->getCommand('AbortMultipartUpload', [
'Bucket' => $adapter->getBucket(),
'Key' => $path,
'ContentType' => 'application/x-gzip',
'UploadId' => $uploadId,
]),
CarbonImmutable::now()->addMinutes(45)
);
$partCount = (int) $size / (self::PART_SIZE);
$parts = [];
for ($i = 0; $i < $partCount; $i++) {
$part = $client->createPresignedRequest(
$client->getCommand('UploadPart', [
'Bucket' => $adapter->getBucket(),
'Key' => $path,
'ContentType' => 'application/x-gzip',
'UploadId' => $uploadId,
'PartNumber' => $i + 1,
]),
CarbonImmutable::now()->addMinutes(30)
);
array_push($parts, $part->getUri()->__toString());
}
return new JsonResponse([
'CompleteMultipartUpload' => $completeMultipartUpload->getUri()->__toString(),
'AbortMultipartUpload' => $abortMultipartUpload->getUri()->__toString(),
'Parts' => $parts,
'PartSize' => self::PART_SIZE,
], JsonResponse::HTTP_OK);
}
}

View file

@ -2,12 +2,10 @@
namespace Pterodactyl\Http\Controllers\Api\Remote\Backups;
use Carbon\Carbon;
use Carbon\CarbonImmutable;
use Illuminate\Http\JsonResponse;
use Pterodactyl\Http\Controllers\Controller;
use Pterodactyl\Repositories\Eloquent\BackupRepository;
use Pterodactyl\Exceptions\Http\HttpForbiddenException;
use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
use Pterodactyl\Http\Requests\Api\Remote\ReportBackupCompleteRequest;
@ -42,7 +40,7 @@ class BackupStatusController extends Controller
/** @var \Pterodactyl\Models\Backup $model */
$model = $this->repository->findFirstWhere([[ 'uuid', '=', $backup ]]);
if (!is_null($model->completed_at)) {
if (! is_null($model->completed_at)) {
throw new BadRequestHttpException(
'Cannot update the status of a backup that is already marked as completed.'
);