ro
3edd6e5521
finally found the correct exception that captures a connect error when a source url cannot be loaded, so it logs the error and just keep executing the script. Now automation can be enabled since the system knows how to handle errors. Fuck. Yes.
196 lines
7.6 KiB
PHP
196 lines
7.6 KiB
PHP
<?php
|
|
|
|
namespace App\Services;
|
|
|
|
use App\Models\Location;
|
|
use App\Repositories\LocationRepository;
|
|
use App\Models\Source;
|
|
use Ramsey\Uuid\Uuid;
|
|
use Carbon\Carbon;
|
|
use GuzzleHttp\Exception\ConnectException;
|
|
|
|
class UpdateService
|
|
{
|
|
private $limit = 15;
|
|
protected $model;
|
|
protected $locationRepository;
|
|
|
|
public function __construct(LocationRepository $locationRepository)
|
|
{
|
|
$this->locationRepository = $locationRepository;
|
|
}
|
|
|
|
public function data()
|
|
{
|
|
$sources = Source::where("active", true)->get();
|
|
$missing = [];
|
|
$checked = [];
|
|
//checks source url to make sure they valid
|
|
foreach ($sources as $source) {
|
|
$result = [];
|
|
if ($source['type'] == 'mastodon') {
|
|
if ($source['token'] == null) {
|
|
try {
|
|
$result = \Mastodon::domain('https://' . $source['url'])
|
|
->get('/instance/domain_blocks');
|
|
array_push($checked, ['source' => $source->url]);
|
|
} catch (ConnectException $e) {
|
|
array_push($missing, ['source' => $source->url]);
|
|
}
|
|
} else {
|
|
try {
|
|
$result = \Mastodon::domain('https://' . $source['url'])
|
|
->token($source['token'])
|
|
->get('/instance/domain_blocks');
|
|
array_push($checked, ['source' => $source->url]);
|
|
} catch (ConnectException $e) {
|
|
}
|
|
}
|
|
} elseif ($source['type'] == 'custom' && $source['format'] == 'csv') {
|
|
try {
|
|
$denylist = array_map('str_getcsv', file('https://' . $source['url']));
|
|
foreach ($denylist as $item) {
|
|
array_push($result, [
|
|
'domain' => $item[0],
|
|
'severity' => $item[1],
|
|
'comment' => $item[2]]);
|
|
}
|
|
array_push($checked, ['source' => $source->url]);
|
|
} catch (Exception $e) {
|
|
array_push($missing, ['source' => $source->url]);
|
|
}
|
|
}
|
|
|
|
$source->list_data = json_encode($result);
|
|
$source->last_updated = Carbon::now();
|
|
$source->save();
|
|
}
|
|
return count($checked) . ' SOURCES UPDATED - ' . count($missing) . ' SOURCES NOT CHECKED';
|
|
}
|
|
|
|
public function list()
|
|
{
|
|
$duplicates = 0;
|
|
$fresh = 0;
|
|
$unified = [];
|
|
|
|
$sources = Source::where("active", true)->get();
|
|
|
|
foreach ($sources as $source) {
|
|
//$listData = json_decode();
|
|
foreach (json_decode($source->list_data) as $item) {
|
|
$index = array_search($item->domain, array_column($unified, 'url'));
|
|
if ($index) {
|
|
//if there is a match, update the count
|
|
if ($item->severity == "suspend" || $item->severity == "defederate") {
|
|
++$unified[$index]['block_count'];
|
|
array_push($unified[$index]['block_vote'], $source->url);
|
|
} else {
|
|
++$unified[$index]['silence_count'];
|
|
array_push($unified[$index]['silence_vote'], $source->url);
|
|
}
|
|
} else {
|
|
$silence = 0;
|
|
$suspend = 0;
|
|
$block_vote = [];
|
|
$silence_vote = [];
|
|
if ($item->severity == "suspend" || $item->severity == "defederate") {
|
|
++$suspend;
|
|
array_push($block_vote, $source->url);
|
|
} else {
|
|
++$silence;
|
|
array_push($silence_vote, $source->url);
|
|
}
|
|
array_push($unified, [
|
|
'name' => $item->domain,
|
|
'url' => $item->domain,
|
|
'rating' => $item->severity,
|
|
'comment' => $item->comment,
|
|
'block_count' => $suspend,
|
|
'silence_count' => $silence,
|
|
'block_vote' => $block_vote,
|
|
'silence_vote' => $silence_vote,
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach ($unified as $item) {
|
|
$location = $this->locationRepository->getLocation($item['url']);
|
|
if ($location) {
|
|
++$duplicates;
|
|
//update block count for existing item
|
|
|
|
$location->block_count = $item['block_count'];
|
|
$location->block_vote = [];
|
|
$location->block_vote = $item['block_vote'];
|
|
$location->silence_count = $item['silence_count'];
|
|
$location->silence_vote = [];
|
|
$location->silence_vote = $item['silence_vote'];
|
|
|
|
$location->actions_count = $item['block_count'] + $item['silence_count'];
|
|
|
|
if (($item['block_count'] + $item['silence_count']) < 2) {
|
|
$location->active = false;
|
|
} else {
|
|
$location->active = true;
|
|
}
|
|
|
|
//replace null with empty array
|
|
if ($location->images == null) {
|
|
$location->images = [];
|
|
};
|
|
$location->save();
|
|
} else {
|
|
// make new entries for instances not present
|
|
++$fresh;
|
|
$images = [];
|
|
$rating = ($item['rating'] == 'defederate') ? 'suspend' : $item['rating'];
|
|
|
|
$status = true;
|
|
if (($item['block_count'] + $item['silence_count']) < 2) {
|
|
$status = false;
|
|
}
|
|
|
|
$new = Location::create([
|
|
'uuid' => Uuid::uuid4(),
|
|
'name' => $item['url'],
|
|
'url' => $item['url'],
|
|
'description' => ($item['comment'] != null) ? $item['comment'] : "no description",
|
|
'active' => $status,
|
|
'rating' => $rating,
|
|
'added_by' => 1,
|
|
'tags' => 'poor moderation, hate speech',
|
|
'images' => json_encode($images),
|
|
'block_count' => $item['block_count'],
|
|
'block_vote' => $item['block_vote'],
|
|
'silence_count' => $item['silence_count'],
|
|
'silence_vote' => $item['silence_vote'],
|
|
'actions_cont' => $item['block_count'] + $item['silence_count']
|
|
]);
|
|
}
|
|
}
|
|
//TODO: Send update post to TBS social account
|
|
return $duplicates . ' LOCATIONS UPDATED | ' . $fresh . ' NEW LOCATIONS CREATED';
|
|
}
|
|
|
|
public function urlExists($url)
|
|
{
|
|
// Remove all illegal characters from a url
|
|
$url = filter_var($url, FILTER_SANITIZE_URL);
|
|
// Validate URI
|
|
if (
|
|
filter_var($url, FILTER_VALIDATE_URL) === false || // check only for http/https schemes.
|
|
!in_array(
|
|
strtolower(parse_url($url, PHP_URL_SCHEME)),
|
|
["http", "https"],
|
|
true
|
|
)
|
|
) {
|
|
return false;
|
|
} // Check that URL exists
|
|
$file_headers = @get_headers($url);
|
|
return !(!$file_headers || $file_headers[0] === "HTTP/1.1 404 Not Found");
|
|
}
|
|
}
|