locationRepository = $locationRepository; } public function locations() { $duplicates = 0; $fresh = 0; $missing = []; $unified = []; $cleanSources = []; $sources = Source::where("active", true)->get(); //checks source url to make sure they valid foreach ($sources as $source) { if ($source->type == 'mastodon') { $url = 'https://' . $source->url; } else { $url = $source->url; } if ($this->urlExists($url)) { array_push($cleanSources, [ 'url' => $source->url, 'token' => $source->token, 'type' => $source->type, 'format' => $source->format]); } else { var_dump($url); array_push($missing, ['source' => $url]); } } //valid source url get compiled for unified foreach ($cleanSources as $source) { //check url to make sure it's cool //parsing for mastodon if ($source['type'] == 'mastodon') { $result = []; if ($source['token'] == null) { $result = \Mastodon::domain('https://' . $source['url']) ->get('/instance/domain_blocks'); } else { $result = \Mastodon::domain('https://' . $source['url']) ->token($source['token']) ->get('/instance/domain_blocks'); } foreach ($result as $item) { $index = array_search($item['domain'], array_column($unified, 'url')); if ($index) { //if there is a match, update the count if ($item['severity'] == "suspend" || $item['severity'] == "defederate") { ++$unified[$index]['block_count']; } else { ++$unified[$index]['silence_count']; } } else { $silence = 0; $suspend = 0; if ($item['severity'] == "suspend" || $item['severity'] == "defederate") { ++$suspend; } else { ++$silence; } array_push($unified, [ 'name' => $item['domain'], 'url' => $item['domain'], 'rating' => $item['severity'], 'comment' => $item['comment'], 'block_count' => $suspend, 'silence_count' => $silence, ]); } } } //parsing for custom csv if ($source['type'] == 'custom' && $source['format'] == 'csv') { $denylist = array_map('str_getcsv', file($source['url'])); foreach ($denylist as $item) { $index = array_search($item[0], array_column($unified, 'url')); if ($index) { //if there is a match, update the count if ($item[1] == "suspend" || $item['severity'] == "defederate") { ++$unified[$index]['block_count']; } else { ++$unified[$index]['silence_count']; } } else { $silence = 0; $suspend = 0; if ($item[1] == "suspend" || $item[1] == "defederate") { ++$suspend; } else { ++$silence; } array_push($unified, [ 'name' => $item[0], 'url' => $item[0], 'rating' => $item[1], 'comment' => $item[2], 'block_count' => $suspend, 'silence_count' => $silence, ]); } } } } //once the unified list is created, update current entries or create fresh ones foreach ($unified as $item) { $location = $this->locationRepository->getLocation($item['url']); if ($location) { ++$duplicates; //update block count for existing item $location->block_count = $item['block_count']; $location->silence_count = $item['silence_count']; $location->actions_count = $item['block_count'] + $item['silence_count']; if (($item['block_count'] + $item['silence_count']) < 2) { $location->active = false; } //replace null with empty array if ($location->images == null) { $location->images = []; }; $location->save(); } else { // make new entries for instances not present ++$fresh; $images = []; $rating = ($item['rating'] == 'defederate') ? 'suspend' : $item['rating']; $status = true; if (($item['block_count'] + $item['silence_count']) < 2) { $status = false; } $new = Location::create([ 'uuid' => Uuid::uuid4(), 'name' => $item['url'], 'url' => $item['url'], 'description' => ($item['comment'] != null) ? $item['comment'] : "no description", 'active' => $status, 'rating' => $rating, 'added_by' => 1, 'tags' => 'poor moderation, hate speech', 'images' => json_encode($images), 'block_count' => $item['block_count'], 'silence_count' => $item['silence_count'], 'actions_cont' => $item['block_count'] + $item['silence_count'] ]); } } //TODO: Send update post to TBS social account return ['duplicates' => $duplicates, 'fresh' => $fresh, 'missing' => $missing]; } public function urlExists($url) { // Remove all illegal characters from a url $url = filter_var($url, FILTER_SANITIZE_URL); // Validate URI if ( filter_var($url, FILTER_VALIDATE_URL) === false || // check only for http/https schemes. !in_array( strtolower(parse_url($url, PHP_URL_SCHEME)), ["http", "https"], true ) ) { return false; } // Check that URL exists $file_headers = @get_headers($url); return !(!$file_headers || $file_headers[0] === "HTTP/1.1 404 Not Found"); } }