More provider validation (iamcal#679)

- removed providers that don't follow the API, since they should only use discovery
- removed/disabled providers that need wildcard domains
- removed wildcards from API URLs
- fixed example URLs to be oEmbed URLs, not regular page URLs
- adding missing schemes in example URLs
- fixed typo in API URL (.com.com)
- dev and staging domains don't need to be in a public catalog
- stronger tests for API URL and example URLs

Author: iamcal

providers/didacte.yml renamed to providers-disabled/didacte.yml

@@ -6,10 +6,10 @@
     - https://*.wiredrive.com/*
     url: http://*.wiredrive.com/present-oembed/
     example_urls:
-    - deckers.wiredrive.com/present-oembed?url=deckers.wiredrive.com/363254/851732644105a5a8b3fa14fe164452df
+    - http://deckers.wiredrive.com/present-oembed?url=deckers.wiredrive.com/363254/851732644105a5a8b3fa14fe164452df
     formats:
     - json
     discovery: true
     notes:
     - 'URL Scheme: `http://*.wiredrive.com/*` (where wiredrive.com = Wiredrive customer domain)'
-...
+...

providers/medialab.yml renamed to providers-disabled/medialab.yml

@@ -6,8 +6,8 @@
     - https://core.hash.ai/@*
     url: https://api.hash.ai/oembed
     example_urls:
-    - https://core.hash.ai/@hash/wildfires-regrowth/stable
-    - https://core.hash.ai/@hash/warehouse-logistics/stable
-    - https://core.hash.ai/@hash/traffic-intersection/stable
+    - https://api.hash.ai/oembed?url=https://core.hash.ai/@hash/wildfires-regrowth/stable
+    - https://api.hash.ai/oembed?url=https://core.hash.ai/@hash/warehouse-logistics/stable
+    - https://api.hash.ai/oembed?url=https://core.hash.ai/@hash/traffic-intersection/stable
     discovery: false
 ...

providers/slateapp.yml renamed to providers-disabled/slateapp.yml

@@ -6,9 +6,7 @@
     - https://app.avocode.com/view/*
     url: https://stage-embed.avocode.com/api/oembed
     example_urls:
-    - https://app.avocode.com/view/a851fc49602446be8c1ef6e5dc3845d8
-    - https://app.avocode.com/view/a851fc49602446be8c1ef6e5dc3845d8/_/comments/
-    - https://app.avocode.com/view/a851fc49602446be8c1ef6e5dc3845d8/33250729/comments/
+    - https://stage-embed.avocode.com/api/oembed?url=https://app.avocode.com/view/a851fc49602446be8c1ef6e5dc3845d8
     formats:
     - json
 ...

providers/wiredrive.yml renamed to providers-disabled/wiredrive.yml

@@ -5,7 +5,7 @@
     - https://buttondown.email/*
     url: https://buttondown.email/embed
     example_urls:
-    - https://buttondown.email/jmduke
+    - https://buttondown.email/embed?url=https://buttondown.email/jmduke
     formats:
     - json
     discovery: true

providers/HASH.yml

@@ -4,7 +4,7 @@
   endpoints:
   - schemes:
     - https://codehs.com/editor/share_abacus/*
-    url: https://codehs.com/api/sharedprogram/*/oembed/
+    url: https://codehs.com/api/sharedprogram/1/oembed/
     docs_url: http://www.codehs.com
     example_urls:
     - https://codehs.com/api/sharedprogram/1/oembed/?url=https://codehs.com/editor/share_abacus/xtOPDik2wNFjSDUoyl2T

providers/adways.yml

@@ -11,6 +11,5 @@
     example_urls:
     - https://codepoints.net/api/v1/oembed?url=https%3A%2F%2Fcodepoints.net%2FU%2B00E4&format=json
     - https://codepoints.net/api/v1/oembed?url=https%3A%2F%2Fcodepoints.net%2FA&format=xml&maxwidth=64
-    - https://codepoints.net/U+2665
     discovery: true
 ...

providers/apester.yml

@@ -7,7 +7,6 @@
     - https://www.playbuzz.com/*
     url: https://oembed.ex.co/item
     example_urls:
-    - https://oembed.ex.co/item?id=8fb2343f-fa5d-48d4-8723-f8b5d51cc1a9
     - https://oembed.ex.co/item?url=https://app.ex.co/stories/item/8fb2343f-fa5d-48d4-8723-f8b5d51cc1a9
     - https://oembed.ex.co/item?url=https://app.ex.co/stories/paolagarcia10/remembering-september-11th-flash-back-to-images-we-will-never-forget
     - https://oembed.ex.co/item?url=https://www.playbuzz.com/paolagarcia10/remembering-september-11th-flash-back-to-images-we-will-never-forget

providers/avocode.yml

@@ -6,9 +6,9 @@
     - https://knowledgepad.co/#/knowledge/*
     url: https://api.spoonacular.com/knowledge/oembed
     example_urls:
-    - https://knowledgepad.co/#/knowledge/1
-    - https://knowledgepad.co/#/knowledge/0
-    - https://knowledgepad.co/#/knowledge/0/183
+    - https://api.spoonacular.com/knowledge/oembed?url=https://knowledgepad.co/#/knowledge/1
+    - https://api.spoonacular.com/knowledge/oembed?url=https://knowledgepad.co/#/knowledge/0
+    - https://api.spoonacular.com/knowledge/oembed?url=https://knowledgepad.co/#/knowledge/0/183
     formats:
     - json
 ...

providers/buttondown.yml

@@ -3,13 +3,9 @@
   provider_url: https://www.pitchhub.com/
   endpoints:
   - schemes:
-    - https://*.pitchhub.com/en/public/player/*
-    url: https://*.pitchhub.com.com/en/public/oembed
+    - https://player.pitchhub.com/en/public/player/*
+    url: https://player.pitchhub.com/en/public/oembed
     discovery: true
     example_urls:
-    - https://player-dev.pitchhub.com/en/public/oembed?url=https://player-dev.pitchhub.com/en/public/player/9330fd8734deb58345c80dd124203b57&format=json
-    - https://player-staging.pitchhub.com/en/public/oembed?url=https://player-staging.pitchhub.com/en/public/player/296599c9ff2a945eb33d403929e92f35&format=json
     - https://player.pitchhub.com/en/public/oembed?url=https://player.pitchhub.com/en/public/player/296599c9ff2a945eb33d403929e92f35&format=json
-    notes:
-      - 'URL Scheme: `https://*.pitchhub.com.com/*` (where subdomain can be `player`, `player-staging`, `player-dev` )'
 ...

providers/codehs.yml

@@ -10,8 +10,8 @@
     url: https://animatron.com/oembed
     docs_url: https://rcvis.readthedocs.io/en/latest/index.html?highlight=oembed#oembed
     example_urls:
-    - https://rcvis.com/v/macomb-multiwinner-surplusjson-3
-    - https://rcvis.com/v/opavote2json
-    - https://rcvis.com/v/2018_maine_gov_primary_demjson
+    - https://rcvis.com/oembed?url=https%3A%2F%2Frcvis.com%2Fvo%2Fmacomb-multiwinner-surplusjson-3%2Fbarchart-interactive
+    - https://rcvis.com/oembed?url=https%3A%2F%2Frcvis.com%2Fvo%2Fopavote2json%2Fbarchart-interactive
+    - https://rcvis.com/oembed?url=https%3A%2F%2Frcvis.com%2Fvo%2F2018_maine_gov_primary_demjson%2Fbarchart-interactive
     discovery: true
 ...

providers/codepoints.yml

@@ -7,5 +7,5 @@
     url: https://www.tumblr.com/oembed/1.0
     docs_url: https://www.tumblr.com/docs/en/api/v2
     example_urls:
-    - https://staff.tumblr.com/post/619193383820410880/have-a-post-youre-particularly-proud-of
+    - https://www.tumblr.com/oembed/1.0?url=https://staff.tumblr.com/post/619193383820410880/have-a-post-youre-particularly-proud-of
 ...

providers/exco.yml

@@ -8,7 +8,6 @@
     discovery: true
     example_urls:
     - https://data.voxsnap.com/oembed?url=https%3A%2F%2Farticle.voxsnap.com%2Fnirandfar%2Fthe-psychology-of-a-billion-dollar-enterprise-app
-    - https://article.voxsnap.com/nirandfar/the-psychology-of-a-billion-dollar-enterprise-app
     formats:
     - json
 ...

providers/knowledgepad.yml

@@ -89,6 +89,28 @@
 						exit(1);
 					}
 
+					if (preg_match('!\*!', $endpoint['url'])){
+						echo "Endpoint URL contains a wildcard in provider file providers/$file\n";
+						print_r($endpoint);
+						exit(1);
+					}
+
+					if (isset($endpoint['example_urls']))
+					foreach ($endpoint['example_urls'] as $example){
+
+						if (!preg_match('!^https?://!', $example)){
+							echo "Endpoint example URL does not start with http:// or https:// in provider file providers/$file\n";
+							print_r($endpoint);
+							exit(1);
+						}
+
+						if (!preg_match('!url=!', $example)){
+							echo "Endpoint example URL does not contain url= param in provider file providers/$file\n";
+							print_r($endpoint);
+							exit(1);
+						}
+					}
+
 					if (isset($endpoint['schemes'])){
 					foreach ($endpoint['schemes'] as $scheme){