From 565f099c90d0e5460feeb28a96a0e9d9f775aa13 Mon Sep 17 00:00:00 2001 From: Ro Date: Fri, 1 Sep 2023 15:22:57 -0700 Subject: [PATCH] Implemented internal blocklist sourcing Switched from using external CSV exports to having the system talk to instances directly to build and update the database. NOTE: Added a new table called Source with the corresponding model to access it --- app/Http/Controllers/FrontIndexController.php | 6 +- app/Http/Controllers/LocationController.php | 105 +++++++++++++----- app/Models/Source.php | 15 +++ composer.json | 3 +- composer.lock | 63 ++++++++++- 5 files changed, 162 insertions(+), 30 deletions(-) create mode 100644 app/Models/Source.php diff --git a/app/Http/Controllers/FrontIndexController.php b/app/Http/Controllers/FrontIndexController.php index ac8c498..073fd1c 100644 --- a/app/Http/Controllers/FrontIndexController.php +++ b/app/Http/Controllers/FrontIndexController.php @@ -15,7 +15,8 @@ class FrontIndexController extends Controller $locations = Location::where("active", true)->get(); $count = count($locations); $recent = Location::where("active", true) - ->limit(5)->orderByDesc('updated_at')->get(); + ->where('block_count', '>', 2) + ->limit(10)->orderByDesc('updated_at')->get(); //$result = DB::select("SELECT * FROM searchlocations('$terms')"); @@ -37,7 +38,8 @@ class FrontIndexController extends Controller $locations = Location::where("active", true)->get(); $count = count($locations); $recent = Location::where("active", true) - ->limit(5)->orderByDesc('updated_at')->get(); + ->where('block_count', '>', 2) + ->limit(10)->orderByDesc('updated_at')->get(); return view('front.index', [ 'count' => $count, diff --git a/app/Http/Controllers/LocationController.php b/app/Http/Controllers/LocationController.php index 3b056eb..b45c26b 100644 --- a/app/Http/Controllers/LocationController.php +++ b/app/Http/Controllers/LocationController.php @@ -7,6 +7,7 @@ use App\Models\Location; use Ramsey\Uuid\Uuid; use Illuminate\Support\Facades\Auth; use League\Csv\Reader; +use App\Models\Source; class LocationController extends Controller { @@ -63,22 +64,72 @@ class LocationController extends Controller //$recordCount = count($fresh); $duplicates = 0; $fresh = 0; - $denycount = array_map('str_getcsv', file($this->defed)); - $denylist = array_map('str_getcsv', file($this->three)); + // ['url' => "rage.love"], + //['url' => "indyapocalypse.social"], - foreach ($denylist as $item) { - $blockCount = 0; - //get block count for item - foreach ($denycount as $line) { - if ($line[0] == $item[0]) { - $blockcount = $line[1]; + $unified = []; + //$denycount = array_map('str_getcsv', file($this->defed)); + //$denylist = array_map('str_getcsv', file($this->three)); + $sources = Source::where("active", true)->get(); + foreach ($sources as $source) { + //parsing for mastodon + if ($source->type == 'mastodon') { + $result = []; + if ($source->token == null) { + $result = \Mastodon::domain('https://' . $source->url) + ->get('/instance/domain_blocks'); + } else { + $result = \Mastodon::domain('https://' . $source->url) + ->token($source->token) + ->get('/instance/domain_blocks'); + } + + foreach ($result as $item) { + $index = array_search($item['domain'], array_column($unified, 'url')); + if ($index) { + //if there is a match, update the count + ++$unified[$index]['count']; + } else { + array_push($unified, [ + 'name' => $item['domain'], + 'url' => $item['domain'], + 'rating' => $item['severity'], + 'comment' => $item['comment'], + 'count' => 1, + ]); + } } } - $location = Location::where("url", $item[0])->first(); + //parsing for custom csv + if ($source->type == 'custom' && $source->format == 'csv') { + $denylist = array_map('str_getcsv', file($source->url)); + foreach ($denylist as $item) { + $index = array_search($item[0], array_column($unified, 'url')); + if ($index) { + //if there is a match, update the count + ++$unified[$index]['count']; + } else { + array_push($unified, [ + 'name' => $item[0], + 'url' => $item[0], + 'rating' => $item[1], + 'comment' => $item[2], + 'count' => 1, + ]); + } + } + } + } + + //once the unified list is created, update current entries or create fresh ones + + foreach ($unified as $item) { + $location = Location::where("url", $item['url'])->first(); if ($location) { ++$duplicates; //update block count for existing item - $location->block_count = $blockcount; + + $location->block_count = $item['count']; //replace null with empty array if ($location->images == null) { @@ -87,25 +138,27 @@ class LocationController extends Controller $location->save(); } else { // make new entries for instances not present - if ($item[0] != 'domain') { - ++$fresh; - $images = []; - $new = Location::create([ - 'uuid' => Uuid::uuid4(), - 'name' => $item[0], - 'url' => $item[0], - 'description' => 'no description', - 'active' => true, - 'rating' => $item[1], - 'added_by' => 1, - 'tags' => 'poor moderation, hate speech', - 'images' => json_encode($images), - 'block_count' => $blockcount, - ]); - } + ++$fresh; + $images = []; + + $new = Location::create([ + 'uuid' => Uuid::uuid4(), + 'name' => $item['url'], + 'url' => $item['url'], + 'description' => ($item['comment'] != null) ? $item['comment'] : "no description", + 'active' => true, + 'rating' => $item['rating'], + 'added_by' => 1, + 'tags' => 'poor moderation, hate speech', + 'images' => json_encode($images), + 'block_count' => $item['count'], + ]); } } + //$lookfor = '0sint.social'; + //$index = array_search($lookfor, array_column($unified, 'url')); + //return back()->with('message', 'TOTAL: ' . count($unified) . " - " . $unified[$index]['count'] . " COUNT"); return back()->with('message', $duplicates . ' UPDATED - ' . $fresh . ' CREATED'); //$domain = $csv[1000][0]; diff --git a/app/Models/Source.php b/app/Models/Source.php new file mode 100644 index 0000000..7e455b6 --- /dev/null +++ b/app/Models/Source.php @@ -0,0 +1,15 @@ +