sources url checking

addded some preliminary url validation to make sure a source url is
still active.

the consolidation process is starting to get top heavy, so it's a stop
gap until a more effecient process can be created.

app/Http/Controllers/LocationController.php

@@ -52,19 +52,45 @@ class LocationController extends Controller
     {
         $duplicates = 0;
         $fresh      = 0;
+        $missing    = [];
 
-        $unified = [];
+        $unified      = [];
-        $sources = Source::where("active", true)->get();
+        $cleanSources = [];
+        $sources      = Source::where("active", true)->get();
+
+        //checks source url to make sure they valid
         foreach ($sources as $source) {
-            //parsing for mastodon
             if ($source->type == 'mastodon') {
+                $url = 'https://' . $source->url;
+            } else {
+                $url = $source->url;
+            }
+
+            if ($this->urlExists($url)) {
+                array_push($cleanSources, [
+                    'url'    => $source->url,
+                    'token'  => $source->token,
+                    'type'   => $source->type,
+                    'format' => $source->format]);
+            } else {
+                var_dump($url);
+                array_push($missing, ['source' => $url]);
+            }
+        }
+
+        //valid source url get compiled for unified
+        foreach ($cleanSources as $source) {
+            //check url to make sure it's cool
+
+            //parsing for mastodon
+            if ($source['type'] == 'mastodon') {
                 $result = [];
-                if ($source->token == null) {
+                if ($source['token'] == null) {
-                    $result = \Mastodon::domain('https://' . $source->url)
+                    $result = \Mastodon::domain('https://' . $source['url'])
                         ->get('/instance/domain_blocks');
                 } else {
-                    $result = \Mastodon::domain('https://' . $source->url)
+                    $result = \Mastodon::domain('https://' . $source['url'])
-                        ->token($source->token)
+                        ->token($source['token'])
                         ->get('/instance/domain_blocks');
                 }
 
@@ -97,8 +123,8 @@ class LocationController extends Controller
                 }
             }
             //parsing for custom csv
-            if ($source->type == 'custom' && $source->format == 'csv') {
+            if ($source['type'] == 'custom' && $source['format'] == 'csv') {
-                $denylist = array_map('str_getcsv', file($source->url));
+                $denylist = array_map('str_getcsv', file($source['url']));
                 foreach ($denylist as $item) {
                     $index = array_search($item[0], array_column($unified, 'url'));
                     if ($index) {
@@ -212,6 +238,26 @@ class LocationController extends Controller
             }
         }
         //TODO: Send update post to TBS social account
-        return back()->with('message', $duplicates . ' UPDATED - ' . $fresh . ' CREATED');
+
+        return back()->with('message', $duplicates . ' UPDATED - ' . $fresh . ' CREATED - ' . count($missing) . ' SOURCE(S) NOT CHECKED');
+    }
+
+    public function urlExists($url)
+    {
+        // Remove all illegal characters from a url
+        $url = filter_var($url, FILTER_SANITIZE_URL);
+        // Validate URI
+        if (
+            filter_var($url, FILTER_VALIDATE_URL) === false || // check only for http/https schemes.
+            !in_array(
+                strtolower(parse_url($url, PHP_URL_SCHEME)),
+                ["http", "https"],
+                true
+            )
+        ) {
+            return false;
+        } // Check that URL exists
+        $file_headers = @get_headers($url);
+        return !(!$file_headers || $file_headers[0] === "HTTP/1.1 404 Not Found");
     }
 }