From 2178b616021e92997473d5457223180c85050d2a Mon Sep 17 00:00:00 2001 From: James Collins Date: Thu, 25 May 2023 19:35:20 +1000 Subject: [PATCH] updated analytics structure --- app/Conductors/AnalyticsConductor.php | 19 +-- app/Conductors/Conductor.php | 85 ++++++++----- .../Controllers/Api/AnalyticsController.php | 93 ++++---------- app/Http/Middleware/LogRequest.php | 10 +- app/Models/Analytics.php | 44 ------- app/Models/AnalyticsRequest.php | 54 ++++++++ app/Models/AnalyticsSession.php | 22 ++++ ...23_05_01_045630_update_analytics_table.php | 33 ----- ...023_05_25_024138_split_analytics_table.php | 116 ++++++++++++++++++ 9 files changed, 279 insertions(+), 197 deletions(-) delete mode 100644 app/Models/Analytics.php create mode 100644 app/Models/AnalyticsRequest.php create mode 100644 app/Models/AnalyticsSession.php create mode 100644 database/migrations/2023_05_25_024138_split_analytics_table.php diff --git a/app/Conductors/AnalyticsConductor.php b/app/Conductors/AnalyticsConductor.php index 19ed4d6..9b073da 100644 --- a/app/Conductors/AnalyticsConductor.php +++ b/app/Conductors/AnalyticsConductor.php @@ -2,16 +2,7 @@ namespace App\Conductors; -use App\Models\Media; -use App\Models\User; -use Carbon\Carbon; -use Illuminate\Contracts\Container\BindingResolutionException; -use Illuminate\Database\Eloquent\Builder; -use Illuminate\Database\Eloquent\InvalidCastException; -use Illuminate\Database\Eloquent\MissingAttributeException; use Illuminate\Database\Eloquent\Model; -use Illuminate\Http\Request; -use LogicException; class AnalyticsConductor extends Conductor { @@ -19,20 +10,14 @@ class AnalyticsConductor extends Conductor * The Model Class * @var string */ - protected $class = \App\Models\Analytics::class; - - /** - * The default sorting field - * @var string - */ - protected $sort = 'created_at'; + protected $class = \App\Models\AnalyticsSession::class; /** * The default includes to include in a request. * * @var array */ - protected $includes = ['duration']; + protected $includes = ['requests.type','requests.path']; /** diff --git a/app/Conductors/Conductor.php b/app/Conductors/Conductor.php index d7518ac..a887543 100644 --- a/app/Conductors/Conductor.php +++ b/app/Conductors/Conductor.php @@ -6,7 +6,6 @@ use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Model; use Illuminate\Http\Request; -use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; class Conductor @@ -143,6 +142,7 @@ class Conductor * * @param Request $request The user request. * @param array|null $limitFields A list of fields to limit the filter request to. + * @return void */ private function filter(Request $request, array|null $limitFields = null): void { @@ -156,6 +156,7 @@ class Conductor } $filterFields += $this->defaultFilters; + foreach ($filterFields as $field => $value) { if ( is_array($limitFields) === false || @@ -163,7 +164,7 @@ class Conductor ) { $value = trim($value); $operator = ''; - $join = 'OR'; + $join = 'AND'; // Check if value has a operator and remove it if it's a number if (preg_match('/^(!?=|[<>]=?|<>|!)([^=!<>].*)*$/', $value, $matches) > 0) { @@ -209,6 +210,8 @@ class Conductor /** * Apple the filter array to the collection. + * + * @return void */ final public function applyFilters(): void { @@ -217,6 +220,40 @@ class Conductor $result = null; $join = 'AND'; + $relationFilter = []; + + $buildWhereFunc = function ($query, $field, $operator, $value, $join) { + if ($join === 'OR') { + if ($operator === '<>') { + $separatorPos = strpos($value, '|'); + if ($separatorPos !== false) { + $query->orWhereBetween( + $field, + [substr($value, 0, $separatorPos), substr($value, ($separatorPos + 1))] + ); + } else { + $query->orWhere($field, '!=', $value); + } + } else { + $query->orWhere($field, $operator, $value); + } + } else { + if ($operator === '<>') { + $separatorPos = strpos($value, '|'); + if ($separatorPos !== false) { + $query->whereBetween( + $field, + [substr($value, 0, $separatorPos), substr($value, ($separatorPos + 1))] + ); + } else { + $query->where($field, '!=', $value); + } + } else { + $query->where($field, $operator, $value); + } + }//end if + }; + if (gettype($query) === 'array') { $item = $query; } @@ -288,35 +325,17 @@ class Conductor $operator = '='; } - if ($join === 'OR') { - if ($operator === '<>') { - $separatorPos = strpos($value, '|'); - if ($separatorPos !== false) { - $query->orWhereBetween( - $field, - [substr($value, 0, $separatorPos), substr($value, ($separatorPos + 1))] - ); - } else { - $query->orWhere($field, '!=', $value); - } - } else { - $query->orWhere($field, $operator, $value); + $relationSplit = strpos($field, '.'); + if ($relationSplit !== false) { + $relation = substr($field, 0, $relationSplit); + $field = substr($field, ($relationSplit + 1)); + + if (method_exists($this->class, $relation) === true) { + $relationFilter[$relation][] = [$field, $operator, $value, $join]; } } else { - if ($operator === '<>') { - $separatorPos = strpos($value, '|'); - if ($separatorPos !== false) { - $query->whereBetween( - $field, - [substr($value, 0, $separatorPos), substr($value, ($separatorPos + 1))] - ); - } else { - $query->where($field, '!=', $value); - } - } else { - $query->where($field, $operator, $value); - } - }//end if + $buildWhereFunc($query, $field, $operator, $value, $join); + } }//end if }//end if @@ -338,6 +357,14 @@ class Conductor }//end if }//end foreach + foreach ($relationFilter as $relation => $conditions) { + $query->whereHas($relation, function ($subQuery) use ($buildWhereFunc, $conditions) { + foreach ($conditions as $condition) { + $buildWhereFunc($subQuery, $condition[0], $condition[1], $condition[2], $condition[3]); + } + }); + } + return $result; }; diff --git a/app/Http/Controllers/Api/AnalyticsController.php b/app/Http/Controllers/Api/AnalyticsController.php index 541c971..fb015d9 100644 --- a/app/Http/Controllers/Api/AnalyticsController.php +++ b/app/Http/Controllers/Api/AnalyticsController.php @@ -6,13 +6,8 @@ use App\Conductors\AnalyticsConductor; use App\Conductors\Conductor; use App\Enum\HttpResponseCodes; use App\Http\Requests\AnalyticsRequest; -use App\Models\Media; use App\Models\Analytics; -use Illuminate\Http\JsonResponse; -use Carbon\Exceptions\InvalidFormatException; -use Illuminate\Contracts\Container\BindingResolutionException; -use Illuminate\Database\Eloquent\InvalidCastException; -use Illuminate\Database\Eloquent\MassAssignmentException; +use App\Models\AnalyticsSession; use Illuminate\Http\Request; class AnalyticsController extends ApiController @@ -39,27 +34,17 @@ class AnalyticsController extends ApiController public function index(Request $request) { if ($request->user() !== null && $request->user()?->hasPermission('admin/analytics') === true) { - $searchFields = ['attribute', 'type', 'useragent', 'ip']; + $request->rename([ + 'type' => 'requests.type', + 'path' => 'requests.path' + ]); - $queryRequest = new Request(); - $queryRequest->merge($request->only($searchFields)); - foreach ($searchFields as $field) { - unset($request[$field]); - } - - $query = Analytics::query() - ->selectRaw('session, - MIN(created_at) as created_at, - TIMESTAMPDIFF(MINUTE, MIN(created_at), MAX(created_at)) as duration'); - $query = Conductor::filterQuery($query, $queryRequest); - - list($collection, $total) = AnalyticsConductor::collection($request, $query - ->groupBy('session') - ->get()); + list($collection, $total) = AnalyticsConductor::request($request); return $this->respondAsResource( $collection, - ['isCollection' => true, + ['resourceName' => 'session', + 'isCollection' => true, 'appendData' => ['total' => $total] ] ); @@ -71,22 +56,25 @@ class AnalyticsController extends ApiController /** * Display the specified resource. * - * @param \Illuminate\Http\Request $request The endpoint request. - * @param \App\Models\Analytics $analytics The analyics model. + * @param \Illuminate\Http\Request $request The endpoint request. + * @param \App\Models\AnalyticsSession $session The analytics session. * @return \Illuminate\Http\Response */ - public function show(Request $request, int $session) + public function show(Request $request, AnalyticsSession $session) { if ($request->user() !== null && $request->user()?->hasPermission('admin/analytics') === true) { - list($collection, $total) = AnalyticsConductor::collection($request, Analytics::query() - ->where('session', $session) - ->get()); + $session->load(['requests' => function ($query) { + $query->select('session_id', 'type', 'path', 'created_at'); + } + ]); + + foreach ($session->requests as $requestItem) { + $requestItem->makeHidden('session_id'); + } return $this->respondAsResource( - $collection, - ['isCollection' => true, - 'appendData' => ['total' => $total] - ] + $session, + ['resourceName' => 'session'] ); } @@ -113,10 +101,10 @@ class AnalyticsController extends ApiController ]; if ($user !== null && $user->hasPermission('admin/analytics') === true && $request->has('session') === true) { - $data['session'] = $request->input('session'); - $analytics = Analytics::create($data); + $data['session_id'] = $request->input('session_id'); + $analytics = AnalyticsRequest::create($data); } else { - $analytics = Analytics::createWithSession($data); + $analytics = AnalyticsRequest::create($data); } return $this->respondAsResource( @@ -127,37 +115,4 @@ class AnalyticsController extends ApiController return $this->respondForbidden(); }//end if } - - /** - * Update the specified resource in storage. - * - * @param \App\Http\Requests\AnalyticsRequest $request The analytics update request. - * @param \App\Models\Analytics $analytics The specified analytics. - * @return \Illuminate\Http\Response - */ - public function update(AnalyticsRequest $request, Analytics $analytics) - { - if (AnalyticsConductor::updatable($analytics) === true) { - $analytics->update($request->all()); - return $this->respondAsResource(AnalyticsConductor::model($request, $analytics)); - } - - return $this->respondForbidden(); - } - - /** - * Remove the specified resource from storage. - * - * @param \App\Models\Analytics $analytics The specified analytics. - * @return \Illuminate\Http\Response - */ - public function destroy(Analytics $analytics) - { - if (AnalyticsConductor::destroyable($analytics) === true) { - $analytics->delete(); - return $this->respondNoContent(); - } else { - return $this->respondForbidden(); - } - } } diff --git a/app/Http/Middleware/LogRequest.php b/app/Http/Middleware/LogRequest.php index fdd5642..7697388 100644 --- a/app/Http/Middleware/LogRequest.php +++ b/app/Http/Middleware/LogRequest.php @@ -2,17 +2,19 @@ namespace App\Http\Middleware; +use App\Http\Requests\AnalyticsRequest; use Symfony\Component\HttpFoundation\Response; use Closure; use Illuminate\Http\Request; -use App\Models\Analytics; class LogRequest { /** * Handle an incoming request. * - * @param \Closure(\Illuminate\Http\Request): (\Illuminate\Http\Response|\Illuminate\Http\RedirectResponse) $next + * @param Illuminate\Http\Request $request HTTP Request. + * @param \Closure(\Illuminate\Http\Request): (\Illuminate\Http\Response|\Illuminate\Http\RedirectResponse) $next Closure. + * @return Symfony\Component\HttpFoundation\Response */ public function handle(Request $request, Closure $next): Response { @@ -20,11 +22,9 @@ class LogRequest $response = $next($request); try { - Analytics::createWithSession([ + AnalyticsRequest::create([ 'type' => 'apirequest', 'attribute' => $request->path(), - 'useragent' => $request->userAgent(), - 'ip' => $request->ip(), ]); return $response; diff --git a/app/Models/Analytics.php b/app/Models/Analytics.php deleted file mode 100644 index 988b181..0000000 --- a/app/Models/Analytics.php +++ /dev/null @@ -1,44 +0,0 @@ -where('ip', $attributes['ip']) - ->where('created_at', '>=', now()->subMinutes(30)) - ->whereNotNull('session') - ->orderBy('created_at', 'desc') - ->first(); - - if ($previousRow !== null) { - $attributes['session'] = $previousRow->session; - } else { - $lastSession = self::max('session'); - $attributes['session'] = ($lastSession + 1); - } - - return static::create($attributes); - } -} diff --git a/app/Models/AnalyticsRequest.php b/app/Models/AnalyticsRequest.php new file mode 100644 index 0000000..c4f9d1e --- /dev/null +++ b/app/Models/AnalyticsRequest.php @@ -0,0 +1,54 @@ +session_id) !== true) { + $request = request(); + if ($request !== null) { + $session = AnalyticsSession::where('ip', $request->ip()) + ->where('useragent', $request->userAgent()) + ->where('ended_at', '>=', now()->subMinutes(30)) + ->first(); + if ($session === null) { + $session = AnalyticsSession::create([ + 'ip' => $request->ip(), + 'useragent' => $request->userAgent(), + 'ended_at' => now() + ]); + } + + $analytics->session_id = $session->id; + } + } + }); + } + + /** + * Return the Analytics Session model. + * + * @return BelongsTo + */ + public function session(): BelongsTo + { + return $this->belongsTo(AnalyticsSession::class, 'id', 'session_id'); + } +} diff --git a/app/Models/AnalyticsSession.php b/app/Models/AnalyticsSession.php new file mode 100644 index 0000000..25457b4 --- /dev/null +++ b/app/Models/AnalyticsSession.php @@ -0,0 +1,22 @@ +hasMany(AnalyticsRequest::class, 'session_id', 'id'); + } + +} diff --git a/database/migrations/2023_05_01_045630_update_analytics_table.php b/database/migrations/2023_05_01_045630_update_analytics_table.php index cfcd5d7..502672f 100644 --- a/database/migrations/2023_05_01_045630_update_analytics_table.php +++ b/database/migrations/2023_05_01_045630_update_analytics_table.php @@ -58,39 +58,6 @@ return new class extends Migration $lastSessionUpdate = $sameSessionRows->first()->created_at; } while(true); } while(true); - - - // Loop through the rows and update `session` based on the logic you described - // foreach ($rows as $row) { - // // Check if this is the first row - // if ($row->created_at === $rows->first()->created_at) { - // DB::table('analytics') - // ->where('id', $row->id) - // ->update(['session' => $session]); - // } else { - // // Look for a previous row with the same useragent and ip within the last 30 minutes - // $previousRow = DB::table('analytics') - // ->where('useragent', $row->useragent) - // ->where('ip', $row->ip) - // ->where('created_at', '>=', date('Y-m-d H:i:s', strtotime('-30 minutes', strtotime($row->created_at)))) - // ->whereNotNull('session') - // ->orderBy('created_at', 'desc') - // ->first(); - - // if ($previousRow) { - // // If a previous row is found, set the session to the same value - // DB::table('analytics') - // ->where('id', $row->id) - // ->update(['session' => $previousRow->session]); - // } else { - // // If no previous row is found, increment the session value - // $session++; - // DB::table('analytics') - // ->where('id', $row->id) - // ->update(['session' => $session]); - // } - // } - // } } /** diff --git a/database/migrations/2023_05_25_024138_split_analytics_table.php b/database/migrations/2023_05_25_024138_split_analytics_table.php new file mode 100644 index 0000000..d5c9100 --- /dev/null +++ b/database/migrations/2023_05_25_024138_split_analytics_table.php @@ -0,0 +1,116 @@ +id(); + $table->text('useragent'); + $table->string('ip'); + $table->timestamps(); + $table->timestamp('ended_at')->nullable(); + }); + + Schema::create('analytics_requests', function (Blueprint $table) { + $table->id(); + $table->bigInteger('session_id')->unsigned(); + $table->string('type'); + $table->string('path'); + $table->timestamps(); + + $table->foreign('session_id')->references('id')->on('analytics_sessions')->onDelete('cascade'); + }); + + // Migrate old analytics table + $analytics = DB::table('analytics') + ->select( + 'session', + DB::raw('MAX(useragent) as useragent'), + DB::raw('MAX(ip) as ip'), + DB::raw('MIN(created_at) as created_at'), + DB::raw('MIN(updated_at) as updated_at')) + ->groupBy('session') + ->get(); + foreach ($analytics as $sessionItem) { + $ip = $sessionItem->ip; + $useragent = $sessionItem->useragent; + $session_id = $sessionItem->session; + $created_at = $sessionItem->created_at; + $updated_at = $sessionItem->updated_at; + + // Create a new row in analytics_sessions + $new_session_id = DB::table('analytics_sessions')->insertGetId([ + 'id' => $session_id, + 'useragent' => $useragent, + 'ip' => $ip, + 'created_at' => $created_at, + 'updated_at' => $updated_at + ]); + + $requests = DB::table('analytics')->where('session', $session_id)->select('type', 'attribute', 'created_at', 'updated_at')->get(); + $ended_at = $sessionItem->created_at; + + foreach($requests as $requestItem) { + if($ended_at < $requestItem->created_at) { + $ended_at = $requestItem->created_at; + } + + DB::table('analytics_requests')->insert([ + 'session_id' => $new_session_id, + 'type' => $requestItem->type, + 'path' => $requestItem->attribute, + 'created_at' => $requestItem->created_at, + 'updated_at' => $requestItem->updated_at, + ]); + } + + DB::table('analytics_sessions')->where('id', $new_session_id)->update(['ended_at' => $ended_at]); + } + + Schema::dropIfExists('analytics'); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::create('analytics', function (Blueprint $table) { + $table->id(); + $table->bigInteger('session')->nullable(false); + $table->string('type'); + $table->string('attribute')->default(''); + $table->text('useragent'); + $table->string('ip'); + $table->timestamps(); + }); + + $sessions = DB::table('analytics_sessions')->get(); + foreach ($sessions as $session) { + $requests = DB::table('analytics_requests')->where('session_id', $session->id)->get(); + foreach($requests as $request) { + DB::table('analytics')->insert([ + 'session' => $session->id, + 'type' => $request->type, + 'attribute' => $request->path, + 'ip' => $session->ip, + 'useragent' => $session->useragent, + 'created_at' => $request->created_at, + 'updated_at' => $request->updated_at, + ]); + } + } + + Schema::dropIfExists('analytics_requests'); + Schema::dropIfExists('analytics_sessions'); + } +};