Implemented internal blocklist sourcing
Switched from using external CSV exports to having the system talk to instances directly to build and update the database. NOTE: Added a new table called Source with the corresponding model to access it
This commit is contained in:
Ro
parent
973b02af61
commit
565f099c90
5 changed files with 162 additions and 30 deletions
|
|
@ -15,7 +15,8 @@ class FrontIndexController extends Controller
|
|||
$locations = Location::where("active", true)->get();
|
||||
$count = count($locations);
|
||||
$recent = Location::where("active", true)
|
||||
->limit(5)->orderByDesc('updated_at')->get();
|
||||
->where('block_count', '>', 2)
|
||||
->limit(10)->orderByDesc('updated_at')->get();
|
||||
|
||||
//$result = DB::select("SELECT * FROM searchlocations('$terms')");
|
||||
|
||||
|
|
@ -37,7 +38,8 @@ class FrontIndexController extends Controller
|
|||
$locations = Location::where("active", true)->get();
|
||||
$count = count($locations);
|
||||
$recent = Location::where("active", true)
|
||||
->limit(5)->orderByDesc('updated_at')->get();
|
||||
->where('block_count', '>', 2)
|
||||
->limit(10)->orderByDesc('updated_at')->get();
|
||||
|
||||
return view('front.index', [
|
||||
'count' => $count,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use App\Models\Location;
|
|||
use Ramsey\Uuid\Uuid;
|
||||
use Illuminate\Support\Facades\Auth;
|
||||
use League\Csv\Reader;
|
||||
use App\Models\Source;
|
||||
|
||||
class LocationController extends Controller
|
||||
{
|
||||
|
|
@ -63,22 +64,72 @@ class LocationController extends Controller
|
|||
//$recordCount = count($fresh);
|
||||
$duplicates = 0;
|
||||
$fresh = 0;
|
||||
$denycount = array_map('str_getcsv', file($this->defed));
|
||||
$denylist = array_map('str_getcsv', file($this->three));
|
||||
// ['url' => "rage.love"],
|
||||
//['url' => "indyapocalypse.social"],
|
||||
|
||||
foreach ($denylist as $item) {
|
||||
$blockCount = 0;
|
||||
//get block count for item
|
||||
foreach ($denycount as $line) {
|
||||
if ($line[0] == $item[0]) {
|
||||
$blockcount = $line[1];
|
||||
$unified = [];
|
||||
//$denycount = array_map('str_getcsv', file($this->defed));
|
||||
//$denylist = array_map('str_getcsv', file($this->three));
|
||||
$sources = Source::where("active", true)->get();
|
||||
foreach ($sources as $source) {
|
||||
//parsing for mastodon
|
||||
if ($source->type == 'mastodon') {
|
||||
$result = [];
|
||||
if ($source->token == null) {
|
||||
$result = \Mastodon::domain('https://' . $source->url)
|
||||
->get('/instance/domain_blocks');
|
||||
} else {
|
||||
$result = \Mastodon::domain('https://' . $source->url)
|
||||
->token($source->token)
|
||||
->get('/instance/domain_blocks');
|
||||
}
|
||||
|
||||
foreach ($result as $item) {
|
||||
$index = array_search($item['domain'], array_column($unified, 'url'));
|
||||
if ($index) {
|
||||
//if there is a match, update the count
|
||||
++$unified[$index]['count'];
|
||||
} else {
|
||||
array_push($unified, [
|
||||
'name' => $item['domain'],
|
||||
'url' => $item['domain'],
|
||||
'rating' => $item['severity'],
|
||||
'comment' => $item['comment'],
|
||||
'count' => 1,
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
$location = Location::where("url", $item[0])->first();
|
||||
//parsing for custom csv
|
||||
if ($source->type == 'custom' && $source->format == 'csv') {
|
||||
$denylist = array_map('str_getcsv', file($source->url));
|
||||
foreach ($denylist as $item) {
|
||||
$index = array_search($item[0], array_column($unified, 'url'));
|
||||
if ($index) {
|
||||
//if there is a match, update the count
|
||||
++$unified[$index]['count'];
|
||||
} else {
|
||||
array_push($unified, [
|
||||
'name' => $item[0],
|
||||
'url' => $item[0],
|
||||
'rating' => $item[1],
|
||||
'comment' => $item[2],
|
||||
'count' => 1,
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//once the unified list is created, update current entries or create fresh ones
|
||||
|
||||
foreach ($unified as $item) {
|
||||
$location = Location::where("url", $item['url'])->first();
|
||||
if ($location) {
|
||||
++$duplicates;
|
||||
//update block count for existing item
|
||||
$location->block_count = $blockcount;
|
||||
|
||||
$location->block_count = $item['count'];
|
||||
|
||||
//replace null with empty array
|
||||
if ($location->images == null) {
|
||||
|
|
@ -87,25 +138,27 @@ class LocationController extends Controller
|
|||
$location->save();
|
||||
} else {
|
||||
// make new entries for instances not present
|
||||
if ($item[0] != 'domain') {
|
||||
++$fresh;
|
||||
$images = [];
|
||||
$new = Location::create([
|
||||
'uuid' => Uuid::uuid4(),
|
||||
'name' => $item[0],
|
||||
'url' => $item[0],
|
||||
'description' => 'no description',
|
||||
'active' => true,
|
||||
'rating' => $item[1],
|
||||
'added_by' => 1,
|
||||
'tags' => 'poor moderation, hate speech',
|
||||
'images' => json_encode($images),
|
||||
'block_count' => $blockcount,
|
||||
]);
|
||||
}
|
||||
++$fresh;
|
||||
$images = [];
|
||||
|
||||
$new = Location::create([
|
||||
'uuid' => Uuid::uuid4(),
|
||||
'name' => $item['url'],
|
||||
'url' => $item['url'],
|
||||
'description' => ($item['comment'] != null) ? $item['comment'] : "no description",
|
||||
'active' => true,
|
||||
'rating' => $item['rating'],
|
||||
'added_by' => 1,
|
||||
'tags' => 'poor moderation, hate speech',
|
||||
'images' => json_encode($images),
|
||||
'block_count' => $item['count'],
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
//$lookfor = '0sint.social';
|
||||
//$index = array_search($lookfor, array_column($unified, 'url'));
|
||||
//return back()->with('message', 'TOTAL: ' . count($unified) . " - " . $unified[$index]['count'] . " COUNT");
|
||||
return back()->with('message', $duplicates . ' UPDATED - ' . $fresh . ' CREATED');
|
||||
|
||||
//$domain = $csv[1000][0];
|
||||
|
|
|
|||
15
app/Models/Source.php
Normal file
15
app/Models/Source.php
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Factories\HasFactory;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
|
||||
class Source extends Model
|
||||
{
|
||||
use HasFactory;
|
||||
|
||||
protected $table = "source";
|
||||
public $timestamps = false;
|
||||
protected $fillable = ["url", "type", "active", "format", "token"];
|
||||
}
|
||||
|
|
@ -9,7 +9,8 @@
|
|||
"guzzlehttp/guzzle": "^7.2",
|
||||
"laravel/framework": "^10.10",
|
||||
"laravel/sanctum": "^3.2",
|
||||
"laravel/tinker": "^2.8"
|
||||
"laravel/tinker": "^2.8",
|
||||
"revolution/laravel-mastodon-api": "^3.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"fakerphp/faker": "^1.9.1",
|
||||
|
|
|
|||
63
composer.lock
generated
63
composer.lock
generated
|
|
@ -4,7 +4,7 @@
|
|||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "aa322c53454393ed775cfe4807d54a50",
|
||||
"content-hash": "f22eb6ce893e1d9f9dd058367f9352cd",
|
||||
"packages": [
|
||||
{
|
||||
"name": "brick/math",
|
||||
|
|
@ -3091,6 +3091,67 @@
|
|||
],
|
||||
"time": "2023-04-15T23:01:58+00:00"
|
||||
},
|
||||
{
|
||||
"name": "revolution/laravel-mastodon-api",
|
||||
"version": "3.0.3",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/kawax/laravel-mastodon-api.git",
|
||||
"reference": "b093f818d63f8da6a64114ce8ad2bcb636c40791"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/kawax/laravel-mastodon-api/zipball/b093f818d63f8da6a64114ce8ad2bcb636c40791",
|
||||
"reference": "b093f818d63f8da6a64114ce8ad2bcb636c40791",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-json": "*",
|
||||
"guzzlehttp/guzzle": "^7.5",
|
||||
"illuminate/support": "^9.0||^10.0",
|
||||
"php": "^8.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"mockery/mockery": "^1.6",
|
||||
"orchestra/testbench": "^7.0||^8.0",
|
||||
"phpunit/phpunit": "^9.5"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"laravel": {
|
||||
"providers": [
|
||||
"Revolution\\Mastodon\\Providers\\MastodonServiceProvider"
|
||||
],
|
||||
"aliases": {
|
||||
"Mastodon": "Revolution\\Mastodon\\Facades\\Mastodon"
|
||||
}
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Revolution\\Mastodon\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "kawax",
|
||||
"email": "kawaxbiz@gmail.com"
|
||||
}
|
||||
],
|
||||
"description": "Mastodon API for Laravel",
|
||||
"keywords": [
|
||||
"laravel",
|
||||
"mastodon"
|
||||
],
|
||||
"support": {
|
||||
"source": "https://github.com/kawax/laravel-mastodon-api/tree/3.0.3"
|
||||
},
|
||||
"time": "2023-06-06T04:27:19+00:00"
|
||||
},
|
||||
{
|
||||
"name": "symfony/console",
|
||||
"version": "v6.3.4",
|
||||
|
|
|
|||
Loading…
Reference in a new issue