From 6cb7a8cb4370ece1b74f4cc076a21b15a7995cfb Mon Sep 17 00:00:00 2001 From: James Collins Date: Mon, 25 Sep 2023 19:36:44 +1000 Subject: [PATCH] update secure media backend --- app/Conductors/MediaConductor.php | 12 +- app/Console/Commands/MediaMigrate.php | 63 ------- app/Console/Commands/MediaRebuild.php | 66 -------- app/Helpers/Array.php | 21 +++ app/Http/Controllers/Api/MediaController.php | 27 ++- app/Jobs/MediaWorkerJob.php | 51 +++--- app/Jobs/MoveMediaJob.php | 84 --------- app/Jobs/StoreUploadedFileJob.php | 160 ------------------ app/Models/Media.php | 69 ++++---- app/Models/MediaJob.php | 23 +++ app/Models/Shortlink.php | 2 - ...ename_permission_column_in_media_table.php | 42 +++++ 12 files changed, 189 insertions(+), 431 deletions(-) delete mode 100644 app/Console/Commands/MediaMigrate.php delete mode 100644 app/Console/Commands/MediaRebuild.php delete mode 100644 app/Jobs/MoveMediaJob.php delete mode 100644 app/Jobs/StoreUploadedFileJob.php create mode 100644 database/migrations/2023_09_25_053320_rename_permission_column_in_media_table.php diff --git a/app/Conductors/MediaConductor.php b/app/Conductors/MediaConductor.php index ebabc90..547261f 100644 --- a/app/Conductors/MediaConductor.php +++ b/app/Conductors/MediaConductor.php @@ -52,7 +52,7 @@ class MediaConductor extends Conductor /** @var \App\Models\User */ $user = auth()->user(); if ($user === null || $user->hasPermission('admin/media') === false) { - $fields = arrayRemoveItem($fields, ['permission', 'storage']); + $fields = arrayRemoveItem($fields, ['security', 'storage']); } return $fields; @@ -68,9 +68,15 @@ class MediaConductor extends Conductor { $user = auth()->user(); if ($user === null) { - $builder->where('permission', ''); + $builder->where('security', ''); } else { - $builder->where('permission', '')->orWhereIn('permission', $user->permissions); + $builder->where(function ($query) use ($user) { + $query->where('security', '') + ->orWhere(function ($subquery) use ($user) { + $subquery->where('security', 'LIKE', 'permission:%') + ->whereIn(DB::raw("SUBSTRING(security, 11)"), $user->permissions); + }); + }); } } diff --git a/app/Console/Commands/MediaMigrate.php b/app/Console/Commands/MediaMigrate.php deleted file mode 100644 index 99103b1..0000000 --- a/app/Console/Commands/MediaMigrate.php +++ /dev/null @@ -1,63 +0,0 @@ -addOption( - 'replace', - null, - InputOption::VALUE_NONE, - 'Replace existing files' - ); - } - - /** - * Execute the console command. - */ - public function handle(): void - { - $replace = $this->option('replace'); - - $files = File::allFiles(public_path('uploads')); - - foreach ($files as $file) { - $filename = pathinfo($file, PATHINFO_BASENAME); - - $medium = Media::where('name', $filename)->first(); - - if ($medium !== null) { - $medium->update(['status' => 'Processing media']); - StoreUploadedFileJob::dispatch($medium, $file, $replace)->onQueue('media'); - } else { - unlink($file); - } - } - } -} diff --git a/app/Console/Commands/MediaRebuild.php b/app/Console/Commands/MediaRebuild.php deleted file mode 100644 index a37db86..0000000 --- a/app/Console/Commands/MediaRebuild.php +++ /dev/null @@ -1,66 +0,0 @@ -addOption( - 'replace', - null, - InputOption::VALUE_NONE, - 'Replace existing files' - ); - - $this->addOption( - 'all', - null, - InputOption::VALUE_NONE, - 'Rebuild all variants' - ); - } - - /** - * Execute the console command. - */ - public function handle(): void - { - $replace = $this->option('replace'); - $all = $this->option('replace'); - - $media = []; - if ($all === true) { - $media = Media::all(); - } else { - $media = Media::where(['variants' => ''])->orWhere(['variants' => '[]'])->orWhere(['variants' => '{}'])->get(); - } - - foreach ($media as $medium) { - StoreUploadedFileJob::dispatch($medium, '', $replace)->onQueue('media'); - } - } -} diff --git a/app/Helpers/Array.php b/app/Helpers/Array.php index 5c7ac8c..d56b794 100644 --- a/app/Helpers/Array.php +++ b/app/Helpers/Array.php @@ -55,3 +55,24 @@ function arrayDefaultValue(string $key, array $arr, mixed $value): mixed return $value; } + +/** + * Return if an item exists in an array, case insensitive + * + * @param string $val The value to check. + * @param array $arr The array to check. + * @return bool + */ +function existsInArray(string $val, array $arr): bool +{ + $exists = false; + + foreach ($arr as $el) { + if (strcasecmp($val, $el) === 0) { + $exists = true; + break; + } + } + + return $exists; +} \ No newline at end of file diff --git a/app/Http/Controllers/Api/MediaController.php b/app/Http/Controllers/Api/MediaController.php index 6fa0484..78fbcb8 100644 --- a/app/Http/Controllers/Api/MediaController.php +++ b/app/Http/Controllers/Api/MediaController.php @@ -154,14 +154,37 @@ class MediaController extends ApiController } if ($file !== null) { - $data['size'] = $request->has('chunk') === true ? 0 : $file->getSize(); - $data['mime_type'] = $request->has('chunk') === true ? '' : $file->getMimeType(); + $data['size'] = $request->has('chunk') === true ? intval($request->get('size', 0)) : $file->getSize(); + $data['mime_type'] = $request->has('chunk') === true ? $request->get('mime_type', '') : $file->getMimeType(); } if ($request->has('storage') === true || $file !== null) { $data['storage'] = $request->get('storage', ''); } + if ($request->has('security') === true || $file !== null) { + $data['security'] = $request->get('security', ''); + } + + if(array_key_exists('storage', $data) === true && + array_key_exists('security', $data) === true && + array_key_exists('mime_type', $data) === true && + $data['mime_type'] !== "") { + $error = Media::verifyStorage($data['mime_type'], $data['security'], $data['storage']); + switch($error) { + case Media::STORAGE_VALID: + break; + case Media::STORAGE_MIME_MISSING: + return $this->respondWithErrors(['mime_type' => 'The file type is required.']); + case Media::STORAGE_NOT_FOUND: + return $this->respondWithErrors(['storage' => 'Storage was not found.']); + case Media::STORAGE_INVALID_SECURITY: + return $this->respondWithErrors(['storage' => 'Storage invalid for security value.']); + default: + return $this->respondWithErrors(['storage' => 'Storage verification error occurred.']); + } + } + if ($request->has('transform') === true) { $transform = []; diff --git a/app/Jobs/MediaWorkerJob.php b/app/Jobs/MediaWorkerJob.php index 6d58771..f4a583b 100644 --- a/app/Jobs/MediaWorkerJob.php +++ b/app/Jobs/MediaWorkerJob.php @@ -85,6 +85,16 @@ class MediaWorkerJob implements ShouldQueue $data['file'] = $jpgFileName; }//end if + // get security + $security = ''; + if ($media === null) { + if (array_key_exists('security', $data) === true) { + $security = $data['security']; + } + } else { + $security = $media->security; + } + // get storage $storage = ''; if ($media === null) { @@ -96,10 +106,14 @@ class MediaWorkerJob implements ShouldQueue } if ($storage === '') { - if (strpos($data['mime_type'], 'image/') === 0) { - $storage = 'local'; + if(strlen($security) === 0) { + if (strpos($data['mime_type'], 'image/') === 0) { + $storage = 'local'; + } else { + $storage = 'cdn'; + } } else { - $storage = 'cdn'; + $storage = 'private'; } } @@ -131,7 +145,8 @@ class MediaWorkerJob implements ShouldQueue 'name' => $data['name'], 'mime_type' => $data['mime_type'], 'size' => $data['size'], - 'storage' => $storage + 'security' => $data['security'], + 'storage' => $storage, ]); }//end if @@ -273,21 +288,6 @@ class MediaWorkerJob implements ShouldQueue $media->changeStagingFile($tempFilePath); } }//end if - - // Move file - if (array_key_exists('move', $data['transform']) === true) { - if (array_key_exists('storage', $data['transform']['move']) === true) { - $newStorage = $data['transform']['move']['storage']; - if ($media->storage !== $newStorage) { - if (Storage::has($newStorage) === true) { - $media->createStagingFile(); - $media->storage = $newStorage; - } else { - $this->throwMediaJobFailure("Cannot move file to '{$newStorage}' as it does not exist"); - } - } - } - } }//end if // Update attributes @@ -295,6 +295,19 @@ class MediaWorkerJob implements ShouldQueue $media->title = $data['title']; } + // Relocate file (if requested) + if (array_key_exists('security', $data) === true) { + $media->security = $data['security']; + } + + if (array_key_exists('storage', $data) === true) { + if ($media->storage !== $data['storage']) { + $media->createStagingFile(); + Storage::disk($media->storage)->delete($media->name); + $media->storage = $data['storage']; + } + } + // Finish media object if ($media->hasStagingFile() === true) { $this->mediaJob->setStatusProcessing(0, 0, 'transferring to cdn'); diff --git a/app/Jobs/MoveMediaJob.php b/app/Jobs/MoveMediaJob.php deleted file mode 100644 index 9221cd7..0000000 --- a/app/Jobs/MoveMediaJob.php +++ /dev/null @@ -1,84 +0,0 @@ -media = $media; - $this->newStorage = $newStorage; - } - - /** - * Execute the job. - * - * @return void - */ - public function handle(): void - { - // Don't continue if the media is already on the new storage disk - if ($this->media->storage === $this->newStorage) { - return; - } - - $this->media->status = 'Moving file'; - $this->media->save(); - - $files = ["/{$this->media->name}"]; - if (empty($this->media->variants) === false) { - foreach ($this->media->variants as $variant => $name) { - $files[] = "/{$name}"; - } - } - - $this->media->invalidateCFCache(); - - // Move the files from the old storage disk to the new storage disk - foreach ($files as $file) { - Storage::disk($this->newStorage)->put($file, Storage::disk($this->media->storage)->get($file)); - Storage::disk($this->media->storage)->delete($file); - } - - // Update the media model with the new storage and save it to the database - $this->media->storage = $this->newStorage; - $this->media->status = 'OK'; - $this->media->save(); - } -} diff --git a/app/Jobs/StoreUploadedFileJob.php b/app/Jobs/StoreUploadedFileJob.php deleted file mode 100644 index ed21198..0000000 --- a/app/Jobs/StoreUploadedFileJob.php +++ /dev/null @@ -1,160 +0,0 @@ -media = $media; - $this->uploadedFilePath = $filePath; - $this->replaceExisting = $replaceExisting; - $this->modifications = $modifications; - } - - /** - * Execute the job. - * - * @return void - */ - public function handle(): void - { - $storageDisk = $this->media->storage; - $fileName = $this->media->name; - - try { - $this->media->status = "Transferring to CDN"; - $this->media->save(); - - // convert HEIC file - $fileExtension = File::extension($this->uploadedFilePath); - if ($fileExtension === 'heic') { - // Get the path without the file name - $uploadedFileDirectory = dirname($this->uploadedFilePath); - - // Convert the HEIC file to JPG - $jpgFileName = pathinfo($this->uploadedFilePath, PATHINFO_FILENAME) . '.jpg'; - $jpgFilePath = $uploadedFileDirectory . '/' . $jpgFileName; - Image::make($this->uploadedFilePath)->save($jpgFilePath); - - // Update the uploaded file path and file name - $this->uploadedFilePath = $jpgFilePath; - $fileName = $jpgFileName; - $this->media->name = $fileName; - $this->media->save(); - } - - if (strlen($this->uploadedFilePath) > 0) { - if (Storage::disk($storageDisk)->exists($fileName) === false || $this->replaceExisting === true) { - /** @var Illuminate\Filesystem\FilesystemAdapter */ - $fileSystem = Storage::disk($storageDisk); - $fileSystem->putFileAs('/', new SplFileInfo($this->uploadedFilePath), $fileName); - Log::info("uploading file {$storageDisk} / {$fileName} / {$this->uploadedFilePath}"); - } else { - Log::info("file {$fileName} already exists in {$storageDisk} / " . // phpcs:ignore - "{$this->uploadedFilePath}. Not replacing file and using local {$fileName} for variants."); - } - } else { - if (Storage::disk($storageDisk)->exists($fileName) === true) { - Log::info("file {$fileName} already exists in {$storageDisk} / " . // phpcs:ignore - "{$this->uploadedFilePath}. No local {$fileName} for variants, downloading from CDN."); - $readStream = Storage::disk($storageDisk)->readStream($fileName); - $tempFilePath = tempnam(sys_get_temp_dir(), 'download-'); - $writeStream = fopen($tempFilePath, 'w'); - while (feof($readStream) !== true) { - fwrite($writeStream, fread($readStream, 8192)); - } - fclose($readStream); - fclose($writeStream); - $this->uploadedFilePath = $tempFilePath; - } else { - $errorStr = "cannot upload file {$storageDisk} " . // phpcs:ignore - "/ {$fileName} / {$this->uploadedFilePath} as temp file is empty"; - Log::info($errorStr); - throw new \Exception($errorStr); - } - }//end if - - $this->media->status = "Optimizing image"; - $this->media->save(); - $this->media->generateVariants($this->uploadedFilePath); - - $this->media->status = "Generating Thumbnail"; - $this->media->save(); - $this->media->generateThumbnail($this->uploadedFilePath); - - if (strlen($this->uploadedFilePath) > 0) { - unlink($this->uploadedFilePath); - } - - $this->media->status = 'OK'; - $this->media->save(); - } catch (\Exception $e) { - Log::error($e->getMessage()); - $this->media->status = "Failed"; - $this->media->save(); - $this->fail($e); - }//end try - } -} diff --git a/app/Models/Media.php b/app/Models/Media.php index 82978af..b6f3fdf 100644 --- a/app/Models/Media.php +++ b/app/Models/Media.php @@ -30,11 +30,18 @@ class Media extends Model use Uuids; use DispatchesJobs; + public const NO_ERROR = 0; + public const INVALID_FILE_ERROR = 1; public const FILE_SIZE_EXCEEDED_ERROR = 2; public const FILE_NAME_EXISTS_ERROR = 3; public const TEMP_FILE_ERROR = 4; + public const STORAGE_VALID = 0; + public const STORAGE_MIME_MISSING = 10; // Mime type is missing and cannot verify + public const STORAGE_NOT_FOUND = 11; // Storage name is not found + public const STORAGE_INVALID_SECURITY = 12; // Security setting invalid for storage + /** * The attributes that are mass assignable. * @@ -44,7 +51,7 @@ class Media extends Model 'title', 'user_id', 'mime_type', - 'permission', + 'security', 'storage', 'description', 'name', @@ -70,7 +77,7 @@ class Media extends Model 'variants' => '[]', 'description' => '', 'dimensions' => '', - 'permission' => '', + 'security' => '', 'thumbnail' => '', ]; @@ -120,21 +127,6 @@ class Media extends Model static::updating(function ($media) use ($clearCache) { $clearCache($media); - - if (array_key_exists('permission', $media->getChanges()) === true) { - $origPermission = $media->getOriginal()['permission']; - $newPermission = $media->permission; - - $newPermissionLen = strlen($newPermission); - - if ($newPermissionLen !== strlen($origPermission)) { - if ($newPermissionLen === 0) { - $this->moveToStorage('cdn'); - } else { - $this->moveToStorage('private'); - } - } - } }); static::deleting(function ($media) use ($clearCache) { @@ -360,21 +352,6 @@ class Media extends Model return $this->belongsTo(User::class); } - /** - * Move files to new storage device. - * - * @param string $storage The storage ID to move to. - * @return void - */ - public function moveToStorage(string $storage): void - { - if ($storage !== $this->storage && Config::has("filesystems.disks.$storage") === true) { - // $this->status = "Processing media"; - MoveMediaJob::dispatch($this, $storage)->onQueue('media'); - $this->save(); - } - } - /** * Transform the media through the Media Job Queue * @@ -1053,4 +1030,32 @@ class Media extends Model public function jobs(): HasMany { return $this->hasMany(MediaJob::class, 'media_id'); } + + public static function verifyStorage($mime_type, $security, &$storage): int { + if($mime_type === '') { + return Media::STORAGE_MIME_MISSING; + } + + if($storage === '') { + if($security === '') { + if (strpos($mime_type, 'image/') === 0) { + $storage = 'local'; + } else { + $storage = 'cdn'; + } + } else { + $storage = 'private'; + } + } else { + if(Storage::has($storage) === false) { + return Media::STORAGE_NOT_FOUND; + } + + if(strcasecmp($storage, 'private') !== 0) { + return Media::STORAGE_INVALID_SECURITY; + } + } + + return Media::STORAGE_VALID; + } } diff --git a/app/Models/MediaJob.php b/app/Models/MediaJob.php index 65f9dcd..cbecf3b 100644 --- a/app/Models/MediaJob.php +++ b/app/Models/MediaJob.php @@ -180,6 +180,29 @@ class MediaJob extends Model $data['size'] = filesize($newFile); $data['mime_type'] = $mime; + if(array_key_exists('storage', $data) === true && + array_key_exists('security', $data) === true && + array_key_exists('mime_type', $data) === true && + $data['mime_type'] !== "") { + $error = Media::verifyStorage($data['mime_type'], $data['security'], $data['storage']); + switch($error) { + case Media::STORAGE_VALID: + break; + case Media::STORAGE_MIME_MISSING: + $this->setStatusInvalid('The file type cannot be determined.'); + return; + case Media::STORAGE_NOT_FOUND: + $this->setStatusInvalid('Storage was not found.'); + return; + case Media::STORAGE_INVALID_SECURITY: + $this->setStatusInvalid('Storage invalid for security value.'); + return; + default: + $this->setStatusInvalid('Storage verification error occurred.'); + return; + } + } + $this->data = json_encode($data); $this->setStatusQueued(); MediaWorkerJob::dispatch($this)->onQueue('media'); diff --git a/app/Models/Shortlink.php b/app/Models/Shortlink.php index 8eb3c53..e154129 100644 --- a/app/Models/Shortlink.php +++ b/app/Models/Shortlink.php @@ -3,9 +3,7 @@ namespace App\Models; use App\Enum\HttpResponseCodes; -use App\Jobs\MoveMediaJob; use App\Jobs\OptimizeMediaJob; -use App\Jobs\StoreUploadedFileJob; use App\Traits\Uuids; use Illuminate\Contracts\Container\BindingResolutionException; use Illuminate\Database\Eloquent\Factories\HasFactory; diff --git a/database/migrations/2023_09_25_053320_rename_permission_column_in_media_table.php b/database/migrations/2023_09_25_053320_rename_permission_column_in_media_table.php new file mode 100644 index 0000000..dff2ecf --- /dev/null +++ b/database/migrations/2023_09_25_053320_rename_permission_column_in_media_table.php @@ -0,0 +1,42 @@ +renameColumn('permission', 'security'); + }); + + DB::table('media') + ->where('security', '!=', '') + ->update(['security' => DB::raw("CONCAT('permission:', security)")]); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + DB::table('media') + ->where(function ($query) { + $query->where('security', 'NOT LIKE', 'permission:%'); + }) + ->update(['security' => '']); + + DB::table('media') + ->where('security', 'LIKE', 'permission:%') + ->update(['security' => DB::raw("SUBSTRING(security, 11)")]); + + Schema::table('media', function (Blueprint $table) { + $table->renameColumn('security', 'permission'); + }); + } +};