STEMMechanics/Website
1
0
Fork 0
Code Issues 4 Pull Requests 1 Actions Packages Projects Releases 3 Wiki Activity

update secure media backend

Browse Source
This commit is contained in:
James Collins
2023-09-25 19:36:44 +10:00
parent 2913960bcb
commit 6cb7a8cb43
12 changed files with 189 additions and 431 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
app/Conductors/MediaConductor.php
View File

@@ -52,7 +52,7 @@ class MediaConductor extends Conductor
/** @var \App\Models\User */ /** @var \App\Models\User */
$user = auth()->user(); $user = auth()->user();
if ($user === null || $user->hasPermission('admin/media') === false) { if ($user === null || $user->hasPermission('admin/media') === false) {
$fields = arrayRemoveItem($fields, ['permission', 'storage']); $fields = arrayRemoveItem($fields, ['security', 'storage']);
} }
return $fields; return $fields;
@@ -68,9 +68,15 @@ class MediaConductor extends Conductor
{ {
$user = auth()->user(); $user = auth()->user();
if ($user === null) { if ($user === null) {
$builder->where('permission', ''); $builder->where('security', '');
} else { } else {
$builder->where('permission', '')->orWhereIn('permission', $user->permissions); $builder->where(function ($query) use ($user) {
$query->where('security', '')
->orWhere(function ($subquery) use ($user) {
$subquery->where('security', 'LIKE', 'permission:%')
->whereIn(DB::raw("SUBSTRING(security, 11)"), $user->permissions);
});
});
} }
} }

63
app/Console/Commands/MediaMigrate.php
View File

@@ -1,63 +0,0 @@
<?php
namespace App\Console\Commands;
use App\Jobs\StoreUploadedFileJob;
use Illuminate\Console\Command;
use App\Models\Media;
use File;
use Symfony\Component\Console\Input\InputOption;
class MediaMigrate extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'media:migrate';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Migrate the uploads folder to the CDN';
/**
* Configure the command options.
*/
protected function configure(): void
{
$this->addOption(
'replace',
null,
InputOption::VALUE_NONE,
'Replace existing files'
);
}
/**
* Execute the console command.
*/
public function handle(): void
{
$replace = $this->option('replace');
$files = File::allFiles(public_path('uploads'));
foreach ($files as $file) {
$filename = pathinfo($file, PATHINFO_BASENAME);
$medium = Media::where('name', $filename)->first();
if ($medium !== null) {
$medium->update(['status' => 'Processing media']);
StoreUploadedFileJob::dispatch($medium, $file, $replace)->onQueue('media');
} else {
unlink($file);
}
}
}
}

66
app/Console/Commands/MediaRebuild.php
View File

@@ -1,66 +0,0 @@
<?php
namespace App\Console\Commands;
use App\Jobs\StoreUploadedFileJob;
use Illuminate\Console\Command;
use App\Models\Media;
use Symfony\Component\Console\Input\InputOption;
class MediaRebuild extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'media:rebuild';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Rebuild the media table';
/**
* Configure the command options.
*/
protected function configure(): void
{
$this->addOption(
'replace',
null,
InputOption::VALUE_NONE,
'Replace existing files'
);
$this->addOption(
'all',
null,
InputOption::VALUE_NONE,
'Rebuild all variants'
);
}
/**
* Execute the console command.
*/
public function handle(): void
{
$replace = $this->option('replace');
$all = $this->option('replace');
$media = [];
if ($all === true) {
$media = Media::all();
} else {
$media = Media::where(['variants' => ''])->orWhere(['variants' => '[]'])->orWhere(['variants' => '{}'])->get();
}
foreach ($media as $medium) {
StoreUploadedFileJob::dispatch($medium, '', $replace)->onQueue('media');
}
}
}

21
app/Helpers/Array.php
View File

@@ -55,3 +55,24 @@ function arrayDefaultValue(string $key, array $arr, mixed $value): mixed
return $value; return $value;
} }
/**
* Return if an item exists in an array, case insensitive
*
* @param string $val The value to check.
* @param array $arr The array to check.
* @return bool
*/
function existsInArray(string $val, array $arr): bool
{
$exists = false;
foreach ($arr as $el) {
if (strcasecmp($val, $el) === 0) {
$exists = true;
break;
}
}
return $exists;
}

27
app/Http/Controllers/Api/MediaController.php
View File

@@ -154,14 +154,37 @@ class MediaController extends ApiController
} }
if ($file !== null) { if ($file !== null) {
$data['size'] = $request->has('chunk') === true ? 0 : $file->getSize(); $data['size'] = $request->has('chunk') === true ? intval($request->get('size', 0)) : $file->getSize();
$data['mime_type'] = $request->has('chunk') === true ? '' : $file->getMimeType(); $data['mime_type'] = $request->has('chunk') === true ? $request->get('mime_type', '') : $file->getMimeType();
} }
if ($request->has('storage') === true || $file !== null) { if ($request->has('storage') === true || $file !== null) {
$data['storage'] = $request->get('storage', ''); $data['storage'] = $request->get('storage', '');
} }
if ($request->has('security') === true || $file !== null) {
$data['security'] = $request->get('security', '');
}
if(array_key_exists('storage', $data) === true &&
array_key_exists('security', $data) === true &&
array_key_exists('mime_type', $data) === true &&
$data['mime_type'] !== "") {
$error = Media::verifyStorage($data['mime_type'], $data['security'], $data['storage']);
switch($error) {
case Media::STORAGE_VALID:
break;
case Media::STORAGE_MIME_MISSING:
return $this->respondWithErrors(['mime_type' => 'The file type is required.']);
case Media::STORAGE_NOT_FOUND:
return $this->respondWithErrors(['storage' => 'Storage was not found.']);
case Media::STORAGE_INVALID_SECURITY:
return $this->respondWithErrors(['storage' => 'Storage invalid for security value.']);
default:
return $this->respondWithErrors(['storage' => 'Storage verification error occurred.']);
}
}
if ($request->has('transform') === true) { if ($request->has('transform') === true) {
$transform = []; $transform = [];

51
app/Jobs/MediaWorkerJob.php
View File

@@ -85,6 +85,16 @@ class MediaWorkerJob implements ShouldQueue
$data['file'] = $jpgFileName; $data['file'] = $jpgFileName;
}//end if }//end if
// get security
$security = '';
if ($media === null) {
if (array_key_exists('security', $data) === true) {
$security = $data['security'];
}
} else {
$security = $media->security;
}
// get storage // get storage
$storage = ''; $storage = '';
if ($media === null) { if ($media === null) {
@@ -96,10 +106,14 @@ class MediaWorkerJob implements ShouldQueue
} }
if ($storage === '') { if ($storage === '') {
if (strpos($data['mime_type'], 'image/') === 0) { if(strlen($security) === 0) {
$storage = 'local'; if (strpos($data['mime_type'], 'image/') === 0) {
$storage = 'local';
} else {
$storage = 'cdn';
}
} else { } else {
$storage = 'cdn'; $storage = 'private';
} }
} }
@@ -131,7 +145,8 @@ class MediaWorkerJob implements ShouldQueue
'name' => $data['name'], 'name' => $data['name'],
'mime_type' => $data['mime_type'], 'mime_type' => $data['mime_type'],
'size' => $data['size'], 'size' => $data['size'],
'storage' => $storage 'security' => $data['security'],
'storage' => $storage,
]); ]);
}//end if }//end if
@@ -273,21 +288,6 @@ class MediaWorkerJob implements ShouldQueue
$media->changeStagingFile($tempFilePath); $media->changeStagingFile($tempFilePath);
} }
}//end if }//end if
// Move file
if (array_key_exists('move', $data['transform']) === true) {
if (array_key_exists('storage', $data['transform']['move']) === true) {
$newStorage = $data['transform']['move']['storage'];
if ($media->storage !== $newStorage) {
if (Storage::has($newStorage) === true) {
$media->createStagingFile();
$media->storage = $newStorage;
} else {
$this->throwMediaJobFailure("Cannot move file to '{$newStorage}' as it does not exist");
}
}
}
}
}//end if }//end if
// Update attributes // Update attributes
@@ -295,6 +295,19 @@ class MediaWorkerJob implements ShouldQueue
$media->title = $data['title']; $media->title = $data['title'];
} }
// Relocate file (if requested)
if (array_key_exists('security', $data) === true) {
$media->security = $data['security'];
}
if (array_key_exists('storage', $data) === true) {
if ($media->storage !== $data['storage']) {
$media->createStagingFile();
Storage::disk($media->storage)->delete($media->name);
$media->storage = $data['storage'];
}
}
// Finish media object // Finish media object
if ($media->hasStagingFile() === true) { if ($media->hasStagingFile() === true) {
$this->mediaJob->setStatusProcessing(0, 0, 'transferring to cdn'); $this->mediaJob->setStatusProcessing(0, 0, 'transferring to cdn');

84
app/Jobs/MoveMediaJob.php
View File

@@ -1,84 +0,0 @@
<?php
namespace App\Jobs;
use App\Models\Media;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Storage;
class MoveMediaJob implements ShouldQueue
{
use Dispatchable;
use InteractsWithQueue;
use Queueable;
use SerializesModels;
/**
* Media item
*
* @var Media
*/
public $media;
/**
* New storage ID
*
* @var string
*/
protected $newStorage;
/**
* Create a new job instance.
*
* @param Media $media The media model.
* @param string $newStorage The new storage ID.
* @return void
*/
public function __construct(Media $media, string $newStorage)
{
$this->media = $media;
$this->newStorage = $newStorage;
}
/**
* Execute the job.
*
* @return void
*/
public function handle(): void
{
// Don't continue if the media is already on the new storage disk
if ($this->media->storage === $this->newStorage) {
return;
}
$this->media->status = 'Moving file';
$this->media->save();
$files = ["/{$this->media->name}"];
if (empty($this->media->variants) === false) {
foreach ($this->media->variants as $variant => $name) {
$files[] = "/{$name}";
}
}
$this->media->invalidateCFCache();
// Move the files from the old storage disk to the new storage disk
foreach ($files as $file) {
Storage::disk($this->newStorage)->put($file, Storage::disk($this->media->storage)->get($file));
Storage::disk($this->media->storage)->delete($file);
}
// Update the media model with the new storage and save it to the database
$this->media->storage = $this->newStorage;
$this->media->status = 'OK';
$this->media->save();
}
}

160
app/Jobs/StoreUploadedFileJob.php
View File

@@ -1,160 +0,0 @@
<?php
namespace App\Jobs;
use App\Models\Media;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Container\BindingResolutionException;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Intervention\Image\Exception\NotWritableException;
use Intervention\Image\Exception\NotSupportedException;
use SplFileInfo;
use Symfony\Component\HttpFoundation\File\UploadedFile;
use Intervention\Image\Facades\Image;
use Spatie\ImageOptimizer\OptimizerChainFactory;
class StoreUploadedFileJob implements ShouldQueue
{
use Dispatchable;
use InteractsWithQueue;
use Queueable;
use SerializesModels;
/**
* Media item
*
* @var Media
*/
protected $media;
/**
* Uploaded file item
*
* @var string
*/
protected $uploadedFilePath;
/**
* Replace existing files
*
* @var string
*/
protected $replaceExisting;
/**
* Modifications to make on the Media
*
* @var array
*/
protected $modifications;
/**
* Create a new job instance.
*
* @param Media $media The media model.
* @param string $filePath The uploaded file.
* @param boolean $replaceExisting Replace existing files.
* @param array $modifications The modifications to make on the media.
* @return void
*/
public function __construct(Media $media, string $filePath, bool $replaceExisting = true, array $modifications = [])
{
$this->media = $media;
$this->uploadedFilePath = $filePath;
$this->replaceExisting = $replaceExisting;
$this->modifications = $modifications;
}
/**
* Execute the job.
*
* @return void
*/
public function handle(): void
{
$storageDisk = $this->media->storage;
$fileName = $this->media->name;
try {
$this->media->status = "Transferring to CDN";
$this->media->save();
// convert HEIC file
$fileExtension = File::extension($this->uploadedFilePath);
if ($fileExtension === 'heic') {
// Get the path without the file name
$uploadedFileDirectory = dirname($this->uploadedFilePath);
// Convert the HEIC file to JPG
$jpgFileName = pathinfo($this->uploadedFilePath, PATHINFO_FILENAME) . '.jpg';
$jpgFilePath = $uploadedFileDirectory . '/' . $jpgFileName;
Image::make($this->uploadedFilePath)->save($jpgFilePath);
// Update the uploaded file path and file name
$this->uploadedFilePath = $jpgFilePath;
$fileName = $jpgFileName;
$this->media->name = $fileName;
$this->media->save();
}
if (strlen($this->uploadedFilePath) > 0) {
if (Storage::disk($storageDisk)->exists($fileName) === false || $this->replaceExisting === true) {
/** @var Illuminate\Filesystem\FilesystemAdapter */
$fileSystem = Storage::disk($storageDisk);
$fileSystem->putFileAs('/', new SplFileInfo($this->uploadedFilePath), $fileName);
Log::info("uploading file {$storageDisk} / {$fileName} / {$this->uploadedFilePath}");
} else {
Log::info("file {$fileName} already exists in {$storageDisk} / " . // phpcs:ignore
"{$this->uploadedFilePath}. Not replacing file and using local {$fileName} for variants.");
}
} else {
if (Storage::disk($storageDisk)->exists($fileName) === true) {
Log::info("file {$fileName} already exists in {$storageDisk} / " . // phpcs:ignore
"{$this->uploadedFilePath}. No local {$fileName} for variants, downloading from CDN.");
$readStream = Storage::disk($storageDisk)->readStream($fileName);
$tempFilePath = tempnam(sys_get_temp_dir(), 'download-');
$writeStream = fopen($tempFilePath, 'w');
while (feof($readStream) !== true) {
fwrite($writeStream, fread($readStream, 8192));
}
fclose($readStream);
fclose($writeStream);
$this->uploadedFilePath = $tempFilePath;
} else {
$errorStr = "cannot upload file {$storageDisk} " . // phpcs:ignore
"/ {$fileName} / {$this->uploadedFilePath} as temp file is empty";
Log::info($errorStr);
throw new \Exception($errorStr);
}
}//end if
$this->media->status = "Optimizing image";
$this->media->save();
$this->media->generateVariants($this->uploadedFilePath);
$this->media->status = "Generating Thumbnail";
$this->media->save();
$this->media->generateThumbnail($this->uploadedFilePath);
if (strlen($this->uploadedFilePath) > 0) {
unlink($this->uploadedFilePath);
}
$this->media->status = 'OK';
$this->media->save();
} catch (\Exception $e) {
Log::error($e->getMessage());
$this->media->status = "Failed";
$this->media->save();
$this->fail($e);
}//end try
}
}

69
app/Models/Media.php
View File

@@ -30,11 +30,18 @@ class Media extends Model
use Uuids; use Uuids;
use DispatchesJobs; use DispatchesJobs;
public const NO_ERROR = 0;
public const INVALID_FILE_ERROR = 1; public const INVALID_FILE_ERROR = 1;
public const FILE_SIZE_EXCEEDED_ERROR = 2; public const FILE_SIZE_EXCEEDED_ERROR = 2;
public const FILE_NAME_EXISTS_ERROR = 3; public const FILE_NAME_EXISTS_ERROR = 3;
public const TEMP_FILE_ERROR = 4; public const TEMP_FILE_ERROR = 4;
public const STORAGE_VALID = 0;
public const STORAGE_MIME_MISSING = 10; // Mime type is missing and cannot verify
public const STORAGE_NOT_FOUND = 11; // Storage name is not found
public const STORAGE_INVALID_SECURITY = 12; // Security setting invalid for storage
/** /**
* The attributes that are mass assignable. * The attributes that are mass assignable.
* *
@@ -44,7 +51,7 @@ class Media extends Model
'title', 'title',
'user_id', 'user_id',
'mime_type', 'mime_type',
'permission', 'security',
'storage', 'storage',
'description', 'description',
'name', 'name',
@@ -70,7 +77,7 @@ class Media extends Model
'variants' => '[]', 'variants' => '[]',
'description' => '', 'description' => '',
'dimensions' => '', 'dimensions' => '',
'permission' => '', 'security' => '',
'thumbnail' => '', 'thumbnail' => '',
]; ];
@@ -120,21 +127,6 @@ class Media extends Model
static::updating(function ($media) use ($clearCache) { static::updating(function ($media) use ($clearCache) {
$clearCache($media); $clearCache($media);
if (array_key_exists('permission', $media->getChanges()) === true) {
$origPermission = $media->getOriginal()['permission'];
$newPermission = $media->permission;
$newPermissionLen = strlen($newPermission);
if ($newPermissionLen !== strlen($origPermission)) {
if ($newPermissionLen === 0) {
$this->moveToStorage('cdn');
} else {
$this->moveToStorage('private');
}
}
}
}); });
static::deleting(function ($media) use ($clearCache) { static::deleting(function ($media) use ($clearCache) {
@@ -360,21 +352,6 @@ class Media extends Model
return $this->belongsTo(User::class); return $this->belongsTo(User::class);
} }
/**
* Move files to new storage device.
*
* @param string $storage The storage ID to move to.
* @return void
*/
public function moveToStorage(string $storage): void
{
if ($storage !== $this->storage && Config::has("filesystems.disks.$storage") === true) {
// $this->status = "Processing media";
MoveMediaJob::dispatch($this, $storage)->onQueue('media');
$this->save();
}
}
/** /**
* Transform the media through the Media Job Queue * Transform the media through the Media Job Queue
* *
@@ -1053,4 +1030,32 @@ class Media extends Model
public function jobs(): HasMany { public function jobs(): HasMany {
return $this->hasMany(MediaJob::class, 'media_id'); return $this->hasMany(MediaJob::class, 'media_id');
} }
public static function verifyStorage($mime_type, $security, &$storage): int {
if($mime_type === '') {
return Media::STORAGE_MIME_MISSING;
}
if($storage === '') {
if($security === '') {
if (strpos($mime_type, 'image/') === 0) {
$storage = 'local';
} else {
$storage = 'cdn';
}
} else {
$storage = 'private';
}
} else {
if(Storage::has($storage) === false) {
return Media::STORAGE_NOT_FOUND;
}
if(strcasecmp($storage, 'private') !== 0) {
return Media::STORAGE_INVALID_SECURITY;
}
}
return Media::STORAGE_VALID;
}
} }

23
app/Models/MediaJob.php
View File

@@ -180,6 +180,29 @@ class MediaJob extends Model
$data['size'] = filesize($newFile); $data['size'] = filesize($newFile);
$data['mime_type'] = $mime; $data['mime_type'] = $mime;
if(array_key_exists('storage', $data) === true &&
array_key_exists('security', $data) === true &&
array_key_exists('mime_type', $data) === true &&
$data['mime_type'] !== "") {
$error = Media::verifyStorage($data['mime_type'], $data['security'], $data['storage']);
switch($error) {
case Media::STORAGE_VALID:
break;
case Media::STORAGE_MIME_MISSING:
$this->setStatusInvalid('The file type cannot be determined.');
return;
case Media::STORAGE_NOT_FOUND:
$this->setStatusInvalid('Storage was not found.');
return;
case Media::STORAGE_INVALID_SECURITY:
$this->setStatusInvalid('Storage invalid for security value.');
return;
default:
$this->setStatusInvalid('Storage verification error occurred.');
return;
}
}
$this->data = json_encode($data); $this->data = json_encode($data);
$this->setStatusQueued(); $this->setStatusQueued();
MediaWorkerJob::dispatch($this)->onQueue('media'); MediaWorkerJob::dispatch($this)->onQueue('media');

2
app/Models/Shortlink.php
View File

@@ -3,9 +3,7 @@
namespace App\Models; namespace App\Models;
use App\Enum\HttpResponseCodes; use App\Enum\HttpResponseCodes;
use App\Jobs\MoveMediaJob;
use App\Jobs\OptimizeMediaJob; use App\Jobs\OptimizeMediaJob;
use App\Jobs\StoreUploadedFileJob;
use App\Traits\Uuids; use App\Traits\Uuids;
use Illuminate\Contracts\Container\BindingResolutionException; use Illuminate\Contracts\Container\BindingResolutionException;
use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Factories\HasFactory;

42
database/migrations/2023_09_25_053320_rename_permission_column_in_media_table.php Normal file
View File

@@ -0,0 +1,42 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('media', function (Blueprint $table) {
$table->renameColumn('permission', 'security');
});
DB::table('media')
->where('security', '!=', '')
->update(['security' => DB::raw("CONCAT('permission:', security)")]);
}
/**
* Reverse the migrations.
*/
public function down(): void
{
DB::table('media')
->where(function ($query) {
$query->where('security', 'NOT LIKE', 'permission:%');
})
->update(['security' => '']);
DB::table('media')
->where('security', 'LIKE', 'permission:%')
->update(['security' => DB::raw("SUBSTRING(security, 11)")]);
Schema::table('media', function (Blueprint $table) {
$table->renameColumn('security', 'permission');
});
}
};