565f099c90
Switched from using external CSV exports to having the system talk to instances directly to build and update the database. NOTE: Added a new table called Source with the corresponding model to access it
181 lines
6.8 KiB
PHP
181 lines
6.8 KiB
PHP
<?php
|
|
|
|
namespace App\Http\Controllers;
|
|
|
|
use Illuminate\Http\Request;
|
|
use App\Models\Location;
|
|
use Ramsey\Uuid\Uuid;
|
|
use Illuminate\Support\Facades\Auth;
|
|
use League\Csv\Reader;
|
|
use App\Models\Source;
|
|
|
|
class LocationController extends Controller
|
|
{
|
|
//url to oli's unified tier 3 list
|
|
private $three = 'https://codeberg.org/oliphant/blocklists/raw/branch/main/blocklists/_unified_tier3_blocklist.csv';
|
|
|
|
//url to oli's domain audit containin block counts per domain
|
|
private $defed = 'https://codeberg.org/oliphant/blocklists/raw/branch/main/blocklists/other/domain_audit_file.csv';
|
|
|
|
public function addLocation(Request $request)
|
|
{
|
|
$fields = $request->validate([
|
|
'name' => ['required'],
|
|
'url' => ['required'],
|
|
'description' => ['required'],
|
|
'tags' => ['required'],
|
|
]);
|
|
|
|
if ($fields) {
|
|
$examples = [];
|
|
$files = $request->files->get("loc_examples");
|
|
if ($request->hasfile('loc_examples')) {
|
|
foreach ($request->file('loc_examples') as $file) {
|
|
$path = $file->store('reference');
|
|
array_push($examples, ["path" => $path]);
|
|
}
|
|
}
|
|
$request->merge(['active' => true]);
|
|
$request->merge(['uuid' => Uuid::uuid4()]);
|
|
$request->merge(['images' => json_encode($examples)]);
|
|
$request->merge(['added_by' => Auth::user()->id]);
|
|
//NOTE: Laravel gets funky if sequencing isn't explicitly set
|
|
$new = Location::create($request->all());
|
|
if ($new) {
|
|
return back()->with('message', 'New Location Added. Take a break!');
|
|
} else {
|
|
return back()->withErrors([
|
|
'error' => 'Uh oh. There was an inssue',
|
|
]);
|
|
}
|
|
} else {
|
|
return back()->withErrors([
|
|
'error' => 'All fields are required',
|
|
]);
|
|
}
|
|
}
|
|
|
|
public function updateLocations()
|
|
{
|
|
//$fresh = file($this->three);
|
|
//$deny = Reader::createFromPath($fresh, "r");
|
|
//$deny->setHeaderOffset(0);
|
|
//$list = $deny->getRecords();
|
|
//$recordCount = count($fresh);
|
|
$duplicates = 0;
|
|
$fresh = 0;
|
|
// ['url' => "rage.love"],
|
|
//['url' => "indyapocalypse.social"],
|
|
|
|
$unified = [];
|
|
//$denycount = array_map('str_getcsv', file($this->defed));
|
|
//$denylist = array_map('str_getcsv', file($this->three));
|
|
$sources = Source::where("active", true)->get();
|
|
foreach ($sources as $source) {
|
|
//parsing for mastodon
|
|
if ($source->type == 'mastodon') {
|
|
$result = [];
|
|
if ($source->token == null) {
|
|
$result = \Mastodon::domain('https://' . $source->url)
|
|
->get('/instance/domain_blocks');
|
|
} else {
|
|
$result = \Mastodon::domain('https://' . $source->url)
|
|
->token($source->token)
|
|
->get('/instance/domain_blocks');
|
|
}
|
|
|
|
foreach ($result as $item) {
|
|
$index = array_search($item['domain'], array_column($unified, 'url'));
|
|
if ($index) {
|
|
//if there is a match, update the count
|
|
++$unified[$index]['count'];
|
|
} else {
|
|
array_push($unified, [
|
|
'name' => $item['domain'],
|
|
'url' => $item['domain'],
|
|
'rating' => $item['severity'],
|
|
'comment' => $item['comment'],
|
|
'count' => 1,
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
//parsing for custom csv
|
|
if ($source->type == 'custom' && $source->format == 'csv') {
|
|
$denylist = array_map('str_getcsv', file($source->url));
|
|
foreach ($denylist as $item) {
|
|
$index = array_search($item[0], array_column($unified, 'url'));
|
|
if ($index) {
|
|
//if there is a match, update the count
|
|
++$unified[$index]['count'];
|
|
} else {
|
|
array_push($unified, [
|
|
'name' => $item[0],
|
|
'url' => $item[0],
|
|
'rating' => $item[1],
|
|
'comment' => $item[2],
|
|
'count' => 1,
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
//once the unified list is created, update current entries or create fresh ones
|
|
|
|
foreach ($unified as $item) {
|
|
$location = Location::where("url", $item['url'])->first();
|
|
if ($location) {
|
|
++$duplicates;
|
|
//update block count for existing item
|
|
|
|
$location->block_count = $item['count'];
|
|
|
|
//replace null with empty array
|
|
if ($location->images == null) {
|
|
$location->images = [];
|
|
};
|
|
$location->save();
|
|
} else {
|
|
// make new entries for instances not present
|
|
++$fresh;
|
|
$images = [];
|
|
|
|
$new = Location::create([
|
|
'uuid' => Uuid::uuid4(),
|
|
'name' => $item['url'],
|
|
'url' => $item['url'],
|
|
'description' => ($item['comment'] != null) ? $item['comment'] : "no description",
|
|
'active' => true,
|
|
'rating' => $item['rating'],
|
|
'added_by' => 1,
|
|
'tags' => 'poor moderation, hate speech',
|
|
'images' => json_encode($images),
|
|
'block_count' => $item['count'],
|
|
]);
|
|
}
|
|
}
|
|
|
|
//$lookfor = '0sint.social';
|
|
//$index = array_search($lookfor, array_column($unified, 'url'));
|
|
//return back()->with('message', 'TOTAL: ' . count($unified) . " - " . $unified[$index]['count'] . " COUNT");
|
|
return back()->with('message', $duplicates . ' UPDATED - ' . $fresh . ' CREATED');
|
|
|
|
//$domain = $csv[1000][0];
|
|
//$record = null;
|
|
|
|
/*
|
|
foreach ($blockcount as $line) {
|
|
if ($line[0] == $domain) {
|
|
$record = $line;
|
|
}
|
|
}
|
|
if ($record != null) {
|
|
return back()->with('message', $domain . ' has ' . $record[1] . ' blocks.');
|
|
} else {
|
|
return back()->with('message', 'NO MATCHES');
|
|
}
|
|
*/
|
|
}
|
|
}
|