Implemented internal blocklist sourcing

Switched from using external CSV exports to having the system talk to
instances directly to build and update the database.

NOTE: Added a new table called Source with the corresponding model to
access it

app/Http/Controllers/FrontIndexController.php

@@ -15,7 +15,8 @@ class FrontIndexController extends Controller
         $locations = Location::where("active", true)->get();
         $count     = count($locations);
         $recent    = Location::where("active", true)
-            ->limit(5)->orderByDesc('updated_at')->get();
+            ->where('block_count', '>', 2)
+            ->limit(10)->orderByDesc('updated_at')->get();
 
         //$result = DB::select("SELECT * FROM searchlocations('$terms')");
 
@@ -37,7 +38,8 @@ class FrontIndexController extends Controller
         $locations = Location::where("active", true)->get();
         $count     = count($locations);
         $recent    = Location::where("active", true)
-            ->limit(5)->orderByDesc('updated_at')->get();
+            ->where('block_count', '>', 2)
+            ->limit(10)->orderByDesc('updated_at')->get();
 
         return view('front.index', [
             'count'   => $count,

app/Http/Controllers/LocationController.php

@@ -7,6 +7,7 @@ use App\Models\Location;
 use Ramsey\Uuid\Uuid;
 use Illuminate\Support\Facades\Auth;
 use League\Csv\Reader;
+use App\Models\Source;
 
 class LocationController extends Controller
 {
@@ -63,22 +64,72 @@ class LocationController extends Controller
         //$recordCount = count($fresh);
         $duplicates = 0;
         $fresh      = 0;
-        $denycount  = array_map('str_getcsv', file($this->defed));
+        // ['url' => "rage.love"],
-        $denylist   = array_map('str_getcsv', file($this->three));
+        //['url' => "indyapocalypse.social"],
 
-        foreach ($denylist as $item) {
+        $unified = [];
-            $blockCount = 0;
+        //$denycount  = array_map('str_getcsv', file($this->defed));
-            //get block count for item
+        //$denylist   = array_map('str_getcsv', file($this->three));
-            foreach ($denycount as $line) {
+        $sources = Source::where("active", true)->get();
-                if ($line[0] == $item[0]) {
+        foreach ($sources as $source) {
-                    $blockcount = $line[1];
+            //parsing for mastodon
+            if ($source->type == 'mastodon') {
+                $result = [];
+                if ($source->token == null) {
+                    $result = \Mastodon::domain('https://' . $source->url)
+                        ->get('/instance/domain_blocks');
+                } else {
+                    $result = \Mastodon::domain('https://' . $source->url)
+                        ->token($source->token)
+                        ->get('/instance/domain_blocks');
+                }
+
+                foreach ($result as $item) {
+                    $index = array_search($item['domain'], array_column($unified, 'url'));
+                    if ($index) {
+                        //if there is a match, update the count
+                        ++$unified[$index]['count'];
+                    } else {
+                        array_push($unified, [
+                            'name'    => $item['domain'],
+                            'url'     => $item['domain'],
+                            'rating'  => $item['severity'],
+                            'comment' => $item['comment'],
+                            'count'   => 1,
+                        ]);
+                    }
                 }
             }
-            $location = Location::where("url", $item[0])->first();
+            //parsing for custom csv
+            if ($source->type == 'custom' && $source->format == 'csv') {
+                $denylist = array_map('str_getcsv', file($source->url));
+                foreach ($denylist as $item) {
+                    $index = array_search($item[0], array_column($unified, 'url'));
+                    if ($index) {
+                        //if there is a match, update the count
+                        ++$unified[$index]['count'];
+                    } else {
+                        array_push($unified, [
+                            'name'    => $item[0],
+                            'url'     => $item[0],
+                            'rating'  => $item[1],
+                            'comment' => $item[2],
+                            'count'   => 1,
+                        ]);
+                    }
+                }
+            }
+        }
+
+        //once the unified list is created, update current entries or create fresh ones
+
+        foreach ($unified as $item) {
+            $location = Location::where("url", $item['url'])->first();
             if ($location) {
                 ++$duplicates;
                 //update block count for existing item
-                $location->block_count = $blockcount;
+
+                $location->block_count = $item['count'];
 
                 //replace null with empty array
                 if ($location->images == null) {
@@ -87,25 +138,27 @@ class LocationController extends Controller
                 $location->save();
             } else {
                 // make new entries for instances not present
-                if ($item[0] != 'domain') {
+                ++$fresh;
-                    ++$fresh;
+                $images = [];
-                    $images = [];
+
-                    $new    = Location::create([
+                $new = Location::create([
-                        'uuid'        => Uuid::uuid4(),
+                    'uuid'        => Uuid::uuid4(),
-                        'name'        => $item[0],
+                    'name'        => $item['url'],
-                        'url'         => $item[0],
+                    'url'         => $item['url'],
-                        'description' => 'no description',
+                    'description' => ($item['comment'] != null) ? $item['comment'] : "no description",
-                        'active'      => true,
+                    'active'      => true,
-                        'rating'      => $item[1],
+                    'rating'      => $item['rating'],
-                        'added_by'    => 1,
+                    'added_by'    => 1,
-                        'tags'        => 'poor moderation, hate speech',
+                    'tags'        => 'poor moderation, hate speech',
-                        'images'      => json_encode($images),
+                    'images'      => json_encode($images),
-                        'block_count' => $blockcount,
+                    'block_count' => $item['count'],
-                    ]);
+                ]);
-                }
             }
         }
 
+        //$lookfor = '0sint.social';
+        //$index   = array_search($lookfor, array_column($unified, 'url'));
+        //return back()->with('message', 'TOTAL: ' . count($unified) . " - " . $unified[$index]['count'] . " COUNT");
         return back()->with('message', $duplicates . ' UPDATED - ' . $fresh . ' CREATED');
 
         //$domain = $csv[1000][0];

app/Models/Source.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+class Source extends Model
+{
+    use HasFactory;
+
+    protected $table    = "source";
+    public $timestamps  = false;
+    protected $fillable = ["url", "type", "active", "format", "token"];
+}

composer.json

@@ -9,7 +9,8 @@
         "guzzlehttp/guzzle": "^7.2",
         "laravel/framework": "^10.10",
         "laravel/sanctum": "^3.2",
-        "laravel/tinker": "^2.8"
+        "laravel/tinker": "^2.8",
+        "revolution/laravel-mastodon-api": "^3.0"
     },
     "require-dev": {
         "fakerphp/faker": "^1.9.1",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "aa322c53454393ed775cfe4807d54a50",
+    "content-hash": "f22eb6ce893e1d9f9dd058367f9352cd",
     "packages": [
         {
             "name": "brick/math",
@@ -3091,6 +3091,67 @@
             ],
             "time": "2023-04-15T23:01:58+00:00"
         },
+        {
+            "name": "revolution/laravel-mastodon-api",
+            "version": "3.0.3",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/kawax/laravel-mastodon-api.git",
+                "reference": "b093f818d63f8da6a64114ce8ad2bcb636c40791"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/kawax/laravel-mastodon-api/zipball/b093f818d63f8da6a64114ce8ad2bcb636c40791",
+                "reference": "b093f818d63f8da6a64114ce8ad2bcb636c40791",
+                "shasum": ""
+            },
+            "require": {
+                "ext-json": "*",
+                "guzzlehttp/guzzle": "^7.5",
+                "illuminate/support": "^9.0||^10.0",
+                "php": "^8.0"
+            },
+            "require-dev": {
+                "mockery/mockery": "^1.6",
+                "orchestra/testbench": "^7.0||^8.0",
+                "phpunit/phpunit": "^9.5"
+            },
+            "type": "library",
+            "extra": {
+                "laravel": {
+                    "providers": [
+                        "Revolution\\Mastodon\\Providers\\MastodonServiceProvider"
+                    ],
+                    "aliases": {
+                        "Mastodon": "Revolution\\Mastodon\\Facades\\Mastodon"
+                    }
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Revolution\\Mastodon\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "kawax",
+                    "email": "kawaxbiz@gmail.com"
+                }
+            ],
+            "description": "Mastodon API for Laravel",
+            "keywords": [
+                "laravel",
+                "mastodon"
+            ],
+            "support": {
+                "source": "https://github.com/kawax/laravel-mastodon-api/tree/3.0.3"
+            },
+            "time": "2023-06-06T04:27:19+00:00"
+        },
         {
             "name": "symfony/console",
             "version": "v6.3.4",