fix cmc scrape + remove price_btc+day_volume_btc from monitored_pairs

Author: galeone

README.md

@@ -1,5 +1,4 @@
 # OpenAT: Open Source Algorithmic Trading Library
-[![Build Status](https://travis-ci.org/galeone/openat.svg?branch=master)](https://travis-ci.org/galeone/openat)
 
 OpenAT provides an easy to use C++ interface for working with (crypto-)currencies markets and exchanges.
 
@@ -230,8 +229,7 @@ typedef struct {
     std::string name;
     currency_pair_t pair;
     long long int day_volume_usd;
-    double day_volume_btc;
-    double price_usd, price_btc;
+    double price_usd;
     float percent_volume;
     std::time_t last_updated;
 } cm_market_t;

include/at/types.hpp

@@ -493,8 +493,7 @@ typedef struct {
     std::string name;
     currency_pair_t pair;
     long long int day_volume_usd;
-    double day_volume_btc;
-    double price_usd, price_btc;
+    double price_usd;
     float percent_volume;
     std::time_t last_updated;
 } cm_market_t;
@@ -504,8 +503,6 @@ inline void to_json(json& j, const cm_market_t& t)
     j = json{{"name", t.name},
              {"pair", t.pair},
              {"day_volume_usd", t.day_volume_usd},
-             {"day_volume_btc", t.day_volume_btc},
-             {"price_btc", t.price_btc},
              {"price_usd", t.price_usd},
              {"percent_volume", t.percent_volume},
              {"last_updated", t.last_updated}};
@@ -517,8 +514,6 @@ inline void from_json(const json& j, cm_market_t& t)
     t.pair = j.at("pair").get<currency_pair_t>();
     t.day_volume_usd =
         static_cast<long long int>(j.at("day_volume_usd").get<double>());
-    t.day_volume_btc = j.at("day_volume_btc").get<double>();
-    t.price_btc = j.at("price").get<double>();
     t.percent_volume = j.at("percent_volume").get<float>();
     t.last_updated = j.at("last_updated").get<std::time_t>();
 }

src/at/coinmarketcap.cc

@@ -59,33 +59,40 @@ std::vector<cm_market_t> CoinMarketCap::markets(std::string currency_symbol)
     toupper(currency_symbol);
     auto id = _symbol_to_id.find(currency_symbol);
     currency_symbol = id != _symbol_to_id.end() ? id->second : currency_symbol;
-    std::string page =
-        req.getHTML(_reverse_host + "currencies/" + currency_symbol + "/");
+    std::string page = req.getHTML(_reverse_host + "currencies/" +
+                                   currency_symbol + "/markets/");
 
     CDocument doc;
     doc.parse(page.c_str());
-    CSelection table = doc.find("#markets-table tbody");
+    CSelection table = doc.find("tbody");
     if (table.nodeNum() == 0) {
-        throw std::runtime_error(
-            "Unable to find table with ID markets-table, on " + _reverse_host +
-            "currencies/" + currency_symbol);
+        throw std::runtime_error("Unable to find a table on " + _reverse_host +
+                                 "currencies/" + currency_symbol + "/markets/");
     }
 
     std::vector<cm_market_t> ret;
     CSelection rows = table.nodeAt(0).find("tr");
     auto now =
         std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
+
+    auto _to_number_string = [](std::string text) -> std::string {
+        text.erase(std::remove(text.begin(), text.end(), ','), text.end());
+        text.erase(std::remove(text.begin(), text.end(), '$'), text.end());
+        text.erase(std::remove(text.begin(), text.end(), ' '), text.end());
+        return text;
+    };
     for (size_t i = 0; i < rows.nodeNum(); ++i) {
         CNode row = rows.nodeAt(i);
         CSelection fields = row.find("td");
-        if (fields.nodeNum() != 9) {
-            throw std::runtime_error("CMC markets: expected 9 columns, got " +
+        if (fields.nodeNum() != 10) {
+            throw std::runtime_error("CMC markets: expected 10 columns, got " +
                                      std::to_string(fields.nodeNum()));
         }
 
         // Skip markets not updated recently
         // 8: updated
-        std::string updated_string = fields.nodeAt(8).text();
+        std::string updated_string =
+            fields.nodeAt(9).find("div").nodeAt(0).text();
         at::tolower(updated_string);
         if (updated_string != "recently") {
             continue;
@@ -101,31 +108,39 @@ std::vector<cm_market_t> CoinMarketCap::markets(std::string currency_symbol)
         auto first = pair_string.substr(0, split_pos);
         auto second = pair_string.substr(split_pos + 1, pair_string.length());
 
-        // 3: volumes <span data-usd="value", data-btc="value">
-        CNode span = fields.nodeAt(3).find("span").nodeAt(0);
-        long long int day_volume_usd = std::stoull(span.attribute("data-usd"));
-        double day_volume_btc = std::stod(span.attribute("data-btc"));
+        // 3: volumes <div>$1,2,34,5</div>
+        std::string usd_volume_string =
+            _to_number_string(fields.nodeAt(3).find("div").nodeAt(0).text());
+
+        long long int day_volume_usd = std::stoull(usd_volume_string);
+
+        // 4: prices <tr>$12,12,12.xx</tr>
+        std::string price_usd_string =
+            _to_number_string(fields.nodeAt(4).text());
 
-        // 4: prices <span data-usd, data-btc
-        span = fields.nodeAt(4).find("span").nodeAt(0);
-        double price_usd = std::stod(span.attribute("data-usd"));
-        double price_btc = std::stod(span.attribute("data-btc"));
+        // If there is a * in the string, the price is an outlier
+        // and we ignore this row.
+        if (price_usd_string.find('*') != std::string::npos) {
+            continue;
+        }
+        double price_usd = std::stod(price_usd_string);
 
-        // 5: xx.yy% percentage
-        std::string percentage_string = fields.nodeAt(5).text();
+        // 5: xx.yy% percentage <div>a.b%</div>
+        std::string percentage_string =
+            fields.nodeAt(5).find("div").nodeAt(0).text();
         // remove %
         percentage_string.pop_back();
         float percent_volume = std::stof(percentage_string);
 
-        // 6,7: category, feed type unused
+        // 6 effective liquidity: unused
+        // 7 category: unused
+        // 8 fee type unused
 
         cm_market_t market{
             .name = name,
             .pair = currency_pair_t(first, second),
             .day_volume_usd = day_volume_usd,
-            .day_volume_btc = day_volume_btc,
             .price_usd = price_usd,
-            .price_btc = price_btc,
             .percent_volume = percent_volume,
             .last_updated = now,
         };