added yelp cron

Author: lorenseanstewart

2_yelp_scraper_code/20_ZZZZZcron_launcher/.gitignore

@@ -0,0 +1,6 @@
+# package directories
+node_modules
+jspm_packages
+
+# Serverless directories
+.serverless

2_yelp_scraper_code/20_ZZZZZcron_launcher/handler.js

@@ -0,0 +1,39 @@
+"use strict";
+const {
+  getPage,
+  parsePage,
+  saveRatingsToDB,
+  deployScrapers
+} = require("./utils");
+
+module.exports.scrape = (event, context, callback) => {
+  // 1. fetch yelp page
+  getPage(event)
+    // 2. parse the page
+    .then(page => parsePage(page))
+    // 3. save ratings data to our db
+    .then(yelpData => saveRatingsToDB(yelpData, event))
+    .then(() =>
+      callback(null, {
+        statusCode: 200,
+        body: JSON.stringify({
+          message: `Scraped ${event}`
+        })
+      })
+    )
+    .catch(error =>
+      callback(new Error(`Error scraping ${event}: ${JSON.stringify(error)}`))
+    );
+};
+
+module.exports.launch_scrapers = (event, context, callback) => {
+  const fakeDBResults = [
+    "urban-light-at-lacma-los-angeles",
+    "the-museum-of-contemporary-art-los-angeles",
+    "the-last-bookstore-los-angeles"
+  ];
+
+  fakeDBResults.forEach(businessName => {
+    deployScrapers(businessName);
+  });
+};

2_yelp_scraper_code/20_ZZZZZcron_launcher/launch.js

@@ -0,0 +1,39 @@
+const request = require("request-promise");
+const AWS = require("aws-sdk");
+
+const list = [
+  "urban-light-at-lacma-los-angeles",
+  "the-museum-of-contemporary-art-los-angeles",
+  "the-last-bookstore-los-angeles"
+];
+
+function deployScraper(businessName) {
+  const lambda = new AWS.Lambda({
+    region: "us-west-2"
+  });
+
+  const params = {
+    FunctionName: "yelp-scraper-dev-scrape",
+    InvocationType: "RequestResponse",
+    LogType: "Tail",
+    Payload: JSON.stringify(businessName)
+  };
+
+  return lambda.invoke(params, function(error, data) {
+    if (error) {
+      console.error(JSON.stringify(error));
+      return new Error(`Error scraping: ${JSON.stringify(error)}`);
+    } else if (data) {
+      console.log(data);
+      return JSON.stringify(data);
+    }
+  });
+}
+
+function swarm(arr) {
+  arr.forEach(businessName => {
+    deployScraper(businessName);
+  });
+}
+
+swarm(list);

2_yelp_scraper_code/20_ZZZZZcron_launcher/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "cron-launcher",
+  "version": "1.0.0",
+  "description": "",
+  "main": "handler.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "aws-sdk": "^2.92.0",
+    "cheerio": "^1.0.0-rc.2",
+    "request": "^2.81.0",
+    "request-promise": "^4.2.1",
+    "uuid": "^3.1.0"
+  },
+  "devDependencies": {
+    "serverless-offline-scheduler": "^0.3.3"
+  }
+}

2_yelp_scraper_code/20_ZZZZZcron_launcher/serverless.yml

@@ -0,0 +1,40 @@
+service: yelp-scraper
+
+plugins:
+  - serverless-offline-scheduler
+
+provider:
+  name: aws
+  runtime: nodejs6.10
+  region: us-west-2
+  stage: dev
+  environment:
+    DYNAMODB_TABLE: yelp-ratings
+  iamRoleStatements:
+    - Effect: "Allow"
+      Action:
+        - "lambda:InvokeFunction"
+      Resource: "*"
+    - Effect: Allow
+      Action:
+        - dynamodb:Query
+        - dynamodb:Scan
+        - dynamodb:GetItem
+        - dynamodb:PutItem
+        - dynamodb:UpdateItem
+        - dynamodb:DeleteItem
+      Resource: "arn:aws:dynamodb:${opt:region, self:provider.region}:*:table/${self:provider.environment.DYNAMODB_TABLE}"
+package:
+  include:
+    - utils/**
+
+
+
+functions:
+  scrape:
+    handler: handler.scrape
+
+  launch_scrapers:
+    handler: handler.launch_scrapers
+    events:
+      - schedule: rate(1 minute)

…20_cron_launcher/utils/deployScrapers.js …ZZZcron_launcher/utils/deployScrapers.js2_yelp_scraper_code/20_cron_launcher/utils/deployScrapers.js renamed to 2_yelp_scraper_code/20_ZZZZZcron_launcher/utils/deployScrapers.js 2_yelp_scraper_code/20_cron_launcher/utils/deployScrapers.js renamed to 2_yelp_scraper_code/20_ZZZZZcron_launcher/utils/deployScrapers.js

@@ -0,0 +1,7 @@
+const request = require("request-promise");
+
+module.exports = businessName => {
+  // https://www.yelp.com/biz/the-last-bookstore-los-angeles
+  const url = `https://www.yelp.com/biz/${businessName}`;
+  return request({ method: "GET", url: url });
+};

2_yelp_scraper_code/20_ZZZZZcron_launcher/utils/getPage.js

@@ -0,0 +1,6 @@
+module.exports = {
+  getPage: require("./getPage"),
+  parsePage: require("./parsePage"),
+  saveRatingsToDB: require("./saveRatingsToDB"),
+  deployScrapers: require("./deployScrapers")
+};

2_yelp_scraper_code/20_ZZZZZcron_launcher/utils/index.js

@@ -0,0 +1,24 @@
+const cheerio = require("cheerio");
+
+module.exports = page => {
+  try {
+    const $ = cheerio.load(page);
+    const rating = $(".rating-info .i-stars")
+      .attr("title")
+      .trim()
+      .split(" ")[0];
+    const reviewCount = $(".rating-info .review-count")
+      .text()
+      .trim()
+      .split(" ")[0];
+
+    const yelpData = {
+      rating,
+      reviewCount
+    };
+
+    return Promise.resolve(yelpData);
+  } catch (error) {
+    return Promise.reject(`Error parsing page: ${JSON.stringify(error)}`);
+  }
+};

2_yelp_scraper_code/20_ZZZZZcron_launcher/utils/parsePage.js

@@ -0,0 +1,29 @@
+const uuid = require("uuid");
+const AWS = require("aws-sdk");
+
+const dynamoDb = new AWS.DynamoDB.DocumentClient();
+
+module.exports = (yelpData, businessName) => {
+  const date = JSON.stringify(new Date());
+  const params = {
+    TableName: process.env.DYNAMODB_TABLE,
+    Item: {
+      id: uuid.v1(),
+      businessName: businessName,
+      reviewCount: yelpData.reviewCount,
+      rating: yelpData.rating,
+      scrapedAt: date
+    }
+  };
+
+  dynamoDb.put(params, error => {
+    if (error) {
+      console.error(`Error saving data to DynamoDB: ${JSON.stringify(error)}`);
+      return Promise.reject(
+        `Error saving data to DynamoDB: ${JSON.stringify(error)}`
+      );
+    } else {
+      return Promise.resolve(params.Item);
+    }
+  });
+};