Fix getUserQuestions(), use OAuth API instead of parsing HTML

Author: jirutka

README.md

@@ -54,7 +54,7 @@ Working endpoints as of 9/21/18
 [Like](#like)					|`.like(user_id, callback)` 			|✅|	
 [Unlike](#unlike)				|`.unlike(user_id, callback)` 			|✅|	
 [Get User's Profile](#get-users-profile)	|`.getUserProfile(username, callback)` 		|❌|	Always returns html, newline if okc_api
-[Get User's Questions](#get-users-questions)	|`.getUserQuestions(username, low, callback)` 	|❌| 	always returns html
+[Get User's Questions](#get-users-questions)	|`.getUserQuestions(username, options, callback)` 	|✅|
 [Get Visitors](#get-visitors)			|`.getVisitors(callback)` 			|❌| 	get JSON OKC developer/recruiting message
 [Send Message](#send-message)			|`.sendMessage(user_id, message_body, callback)`|✅| 	
 [Get Recent Messages](#get-recent-messages)	|`.getRecentMessages(callback)`			|❌| 	route changed
@@ -117,11 +117,13 @@ Returns a json of the user profile. Contains all the information as you would se
 
 ---
 ### Get User's Questions
-`.getUserQuestions(username, low, callback)`
+`.getUserQuestions(username, options, callback)`
 
-Returns a json of user question data, beginning with the "low" question.
+Returns a json of user question/answer data.
 
-The OkCupid API enforces pagination and won't return more than 10 questions per request, so to fetch all question data for a user, you need to make multiple calls and increment the "low" value. The index of a user's first answered question is 1. (Passing a value of 0 returns nothing.) For example:
+The results come back with a page hash. To get the next page results, this method can be called again with an additional `after` field set in the options object. The value should be set to the value stored in `body.paging.after`. This process can iterated to produce all search results.
+
+The same process can also be applied in reverse with `before`. 
 
 ```
 okc.getUserQuestions(username, 1, cb) // First 10 questions

custom_modules/constants.js

@@ -4,8 +4,6 @@ module.exports = {
     rate: 'http://www.okcupid.com/quickmatch',
     visit_user: 'http://www.okcupid.com/profile/{username}',
     user_profile: 'http://www.okcupid.com/profile/{username}?okc_api=1',
-    user_questions: 'http://www.okcupid.com/profile/{username}/questions?okc_api=1&low={low}',
-    user_questions_no_api: 'http://www.okcupid.com/profile/{username}/questions?low={low}',
     get_visitors: 'http://www.okcupid.com/visitors?okc_api=1',
     quickmatch: 'http://www.okcupid.com/quickmatch?okc_api=1',
 
@@ -19,14 +17,7 @@ module.exports = {
     search: 'https://www.okcupid.com/1/apitun/match/search',
     edit_profile: 'https://www.okcupid.com/1/apitun/profile/save_essay',
     connections: 'https://www.okcupid.com/1/apitun/connections/outgoing',
+    user_questions: 'http://www.okcupid.com/1/apitun/profile/{username}/answers',
   },
-  SELECTORS: {
-    current_page_selector : ".count > .curpage",
-    total_pages_selector : ".count > .last",
-    question_not_answered_selector : "div.question.not_answered",
-    question_answered_selector : "div.question.public",
-    question_text_selector : ".qtext > p",
-    question_answer_selector : "p.answer.target"
-  }
 }
 

lib/client.js

@@ -1,46 +1,13 @@
 var constants = require('../custom_modules/constants')
 var headers = require('../custom_modules/headers')
 var Requester = require('./requester')
-var cheerio = require('cheerio')
 
 var URLS = constants.URLS
-var SELECTORS = constants.SELECTORS
 
 function getProxyString(proxy){
     return `http://${(proxy.username && proxy.password) ? `${proxy.username}:${proxy.password}@` : ''}${proxy.ip}:${proxy.port}`
 }
 
-function parseQuestions(html, low, maxPageCount = null) {
-    var $ = cheerio.load(html);
-    var questionsData = {
-      questions: {
-        public: [],
-        hidden: []
-      },
-      pagination: {
-        low: low,
-        next_page_low: low + 10,
-        done: false,
-        current_page: $(SELECTORS.current_page_selector).text(),
-        total_num_pages: $(SELECTORS.total_pages_selector).text()
-      }
-    };  
-    questionsData.pagination.done = questionsData.pagination.current_page == questionsData.pagination.total_num_pages || questionsData.pagination.current_page == maxPageCount;
-    const answeredQuestions = $(SELECTORS.question_answered_selector);
-    const notAnsweredQuestions = $(SELECTORS.question_not_answered_selector);
-    answeredQuestions.each(function(i, elem) {    
-      const question = $(this).find(SELECTORS.question_text_selector).text().trim();
-      const answer = $(this).find(SELECTORS.question_answer_selector).text().trim();
-      questionsData.questions.public.push({ question, answer })
-    });
-    notAnsweredQuestions.each(function(i, elem) {    
-      const question = $(this).find(SELECTORS.question_text_selector).text().trim();
-      const answer = $(this).find(SELECTORS.question_answer_selector).text().trim();
-      questionsData.questions.hidden.push({ question, answer })
-    });
-    return questionsData;
-}
-
 var OKCupid = function(proxy = null){
     this.requester = new Requester(proxy)
     this.proxy = proxy;
@@ -109,36 +76,10 @@ OKCupid.prototype.getUserProfile = function(username, callback){
     var user_profile_url = URLS.user_profile.replace('{username}', username)
     this.requester.getRequest(user_profile_url, callback)
 }
-/*
-OKCupid.prototype.getUserQuestions = function(username, low, callback){
-    var user_questions_url = URLS.user_questions.replace('{username}', username).replace('{low}', low)
-    this.requester.getRequest(user_questions_url, callback)
-}*/
-
-OKCupid.prototype.getUserQuestions = function(username, low, callback, maxPageCount = null){
-    var user_questions_url = URLS.user_questions_no_api.replace('{username}', username).replace('{low}', low)
-    this.requester.getRequestHtml(user_questions_url, function(err, res, body){
-        if (err) callback(err, res, body)
-        else {
-            callback(err, res, parseQuestions(body, low, maxPageCount))
-        }
-    })
-}
-
-OKCupid.prototype.getAllUserQuestions = function(username, callback, maxPageCount = null){
-    this.getUserQuestionsUntilDone(username, 1, callback, maxPageCount, [])
-}
 
-OKCupid.prototype.getUserQuestionsUntilDone = function(username, low, callback, maxPageCount, questionsSoFar) {
-    var doAgain = (err, res, body) => {
-        if (err) callback(err, res, questions)
-        questionsSoFar = questionsSoFar.concat(body.questions)
-        if(body.pagination.done) callback(err, res, questionsSoFar)
-        else {
-            this.getUserQuestionsUntilDone(username, body.pagination.next_page_low, callback, maxPageCount, questionsSoFar)
-        }
-    }
-    this.getUserQuestions(username, low, doAgain, maxPageCount)
+OKCupid.prototype.getUserQuestions = function(username, options, callback){
+    var user_questions_url = URLS.user_questions.replace('{username}', username)
+    this.requester.postJsonRequest(user_questions_url, options, callback)
 }
 
 OKCupid.prototype.getVisitors = function(callback){

package.json

@@ -25,7 +25,6 @@
         "node": "*"
     },
     "dependencies": {
-        "cheerio": "^1.0.0-rc.2",
         "request": "~2.83.0"
     }
 }