initial commit of code to scrape pastebin

Author: imnotjames

pastebin-mirror/__main__.py

@@ -0,0 +1,49 @@
+from scraper import PastebinComScraper
+from storage import SQLite3Storage
+import time
+import sys
+import os
+
+
+def usage():
+    print("pastebin-mirror [sqlite-file]")
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        usage()
+        sys.exit(1)
+
+    pastebin_sqlite_database = sys.argv[1]
+
+    if os.path.isdir(pastebin_sqlite_database):
+        usage()
+        sys.exit(1)
+
+    scraper = PastebinComScraper()
+    storage = SQLite3Storage()
+
+    storage.initialize_tables()
+
+    while True:
+        recent_pastes = scraper.get_recent_pastes()
+
+        for paste in recent_pastes:
+            key = paste['key']
+
+            storage.save_paste_reference(
+                key,
+                paste['date'],
+                paste['size'],
+                paste['expire'],
+                paste['title'],
+                paste['syntax'],
+                paste['user']
+            )
+
+            if not storage.has_paste_content(key):
+                print("Fetching paste content for %s" % key)
+
+                storage.save_paste_content(key, scraper.get_paste_content(key))
+
+        time.sleep(1)

pastebin-mirror/scraper.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+import requests
+
+
+class PastebinComScraper:
+    def __init__(self):
+        self.__RAW_URL__ = 'https://pastebin.com/raw/'
+        self.__ITEM_URL__ = 'https://pastebin.com/api_scrape_item.php'
+        self.__METADATA_URL__ = 'https://pastebin.com/api_scrape_item_meta.php'
+        self.__LIST_URL__ = 'https://pastebin.com/api_scraping.php'
+        self.__ERROR_TEXT__ = 'Error, we cannot find this paste.'
+
+    def get_paste_content(self, key):
+        result = requests.get(self.__RAW_URL__ + key)
+
+        if not result.ok:
+            return None
+
+        return result.content
+
+    def get_paste_metadata(self, key):
+        paste = requests.get(self.__METADATA_URL__, params={'i': key})
+
+        if not paste.ok:
+            return None
+
+        if paste.text == self.__ERROR_TEXT__:
+            return None
+
+        return paste.json()[0]
+
+    def get_recent_pastes(self, limit=250):
+        paste_list = requests.get(self.__LIST_URL__, {limit: min(250, limit)})
+
+        if not paste_list.ok:
+            return []
+
+        return paste_list.json()

pastebin-mirror/storage.py

@@ -0,0 +1,73 @@
+import sqlite3
+
+class SQLite3Storage:
+    def __init__(self, location='pastebin.db'):
+        self.connection = sqlite3.connect(location)
+
+    def initialize_tables(self):
+        self.connection.execute(
+            '''
+            CREATE TABLE IF NOT EXISTS paste (
+                paste_key CHAR(8) PRIMARY KEY,
+                timestamp TIMESTAMP,
+                size INT,
+                expires TIMESTAMP,
+                title TEXT,
+                syntax TEXT,
+                user TEXT NULL
+            );
+            '''
+        )
+
+        self.connection.execute(
+            '''
+            CREATE TABLE IF NOT EXISTS paste_content (
+                paste_key CHAR(8) PRIMARY KEY,
+                raw_content TEXT
+            );
+            '''
+        )
+
+    def has_paste_content(self, key):
+        cursor = self.connection.cursor()
+
+        cursor.execute('SELECT COUNT(*) FROM paste_content WHERE paste_key = ?', (key,))
+
+        paste_content_count = cursor.fetchone()[0]
+
+        return paste_content_count > 0
+
+    def save_paste_reference(self, key, size, timestamp, expires, title, syntax, user):
+        self.connection.execute(
+            '''
+            INSERT OR REPLACE INTO paste
+              (paste_key, timestamp, size, expires, title, syntax, user)
+            VALUES
+              (?, ?, ?, ?, ?, ?, ?)
+            ''',
+            (
+                key,
+                timestamp,
+                size,
+                expires,
+                title,
+                syntax,
+                user,
+            )
+        )
+
+        self.connection.commit()
+
+    def save_paste_content(self, key, content):
+        self.connection.execute(
+            '''
+            INSERT OR REPLACE INTO paste_content
+              (paste_key, raw_content)
+            VALUES
+              (?, ?)
+            ''',
+            (
+                key,
+                content,
+            )
+        )

setup.py

@@ -0,0 +1,18 @@
+from setuptools import setup
+
+setup(
+    name='pastebin-mirror',
+    version='0.0.1',
+    description='Mirror Pastebin to an SQLite DB',
+    url='http://github.com/imnotjames/pastebin-mirror',
+    author='James Ward',
+    author_email='[email protected]',
+    license='MIT',
+    packages=[
+        'pastebin-mirror'
+    ],
+    install_requires=[
+      'requests',
+    ],
+    zip_safe=False
+)