Merge pull request philipperemy#1 from philipperemy/quantiles

Quantiles

Author: Philippe Rémy

data_generator.py

@@ -5,54 +5,89 @@
 from uuid import uuid4
 
 import numpy as np
+import pandas as pd
 
 from data_manager import file_processor
+from returns_quantization import add_returns_in_place
 from utils import *
 
+np.set_printoptions(threshold=np.nan)
+pd.set_option('display.height', 1000)
+pd.set_option('display.max_rows', 500)
+pd.set_option('display.max_columns', 500)
+pd.set_option('display.width', 1000)
+
+
+def generate_quantiles(data_folder, bitcoin_file):
+    def get_label(btc_df, btc_slice, i, slice_size):
+        class_name = str(btc_df[i + slice_size:i + slice_size + 1]['close_price_returns_labels'].values[0])
+        return class_name
+
+    return generate_cnn_dataset(data_folder, bitcoin_file, get_label)
+
+
+def generate_up_down(data_folder, bitcoin_file):
+    def get_price_direction(btc_df, btc_slice, i, slice_size):
+        last_price = btc_slice[-2:-1]['price_close'].values[0]
+        next_price = btc_df[i + slice_size:i + slice_size + 1]['price_close'].values[0]
+        if last_price < next_price:
+            class_name = 'UP'
+        else:
+            class_name = 'DOWN'
+        return class_name
+
+    return generate_cnn_dataset(data_folder, bitcoin_file, get_price_direction)
+
+
+def generate_cnn_dataset(data_folder, bitcoin_file, get_class_name):
+    btc_df = file_processor(bitcoin_file)
+    btc_df, levels = add_returns_in_place(btc_df)
+
+    print('-' * 80)
+    print('Those values should be roughly equal to 1/len(levels):')
+    for ii in range(len(levels)):
+        print(ii, np.mean((btc_df['close_price_returns_labels'] == ii).values))
+    print(levels)
+    print('-' * 80)
 
-def generate(data_folder, bitcoin_file):
-    p = file_processor(bitcoin_file)
     slice_size = 40
     test_every_steps = 10
-    n = len(p) - slice_size
+    n = len(btc_df) - slice_size
 
     shutil.rmtree(data_folder, ignore_errors=True)
     for epoch in range(int(1e6)):
         st = time()
 
         i = np.random.choice(n)
-        sl = p[i:i + slice_size]
+        btc_slice = btc_df[i:i + slice_size]
 
-        if sl.isnull().values.any():
+        if btc_slice.isnull().values.any():
             # sometimes prices are discontinuous and nothing happened in one 5min bucket.
-            # in that case, we consider this slice as wrong and we ask for a new one.
+            # in that case, we consider this slice as wrong and we raise an exception.
             # it's likely to happen at the beginning of the data set where the volumes are low.
-            continue
-
-        last_price = sl[-2:-1]['price_close'].values[0]
-        next_price = p[i + slice_size:i + slice_size + 1]['price_close'].values[0]
-
-        if last_price < next_price:
-            direction = 'UP'
-        else:
-            direction = 'DOWN'
+            raise Exception('NaN values detected. Please remove them.')
 
-        save_dir = os.path.join(data_folder, 'train', direction)
+        class_name = get_class_name(btc_df, btc_slice, i, slice_size)
+        save_dir = os.path.join(data_folder, 'train', class_name)
         if epoch % test_every_steps == 0:
-            save_dir = os.path.join(data_folder, 'test', direction)
+            save_dir = os.path.join(data_folder, 'test', class_name)
         mkdir_p(save_dir)
-        save_to_file(sl, filename=save_dir + '/' + str(uuid4()) + '.png')
-
-        print('epoch = {0}, time = {1:.3f}'.format(str(epoch).zfill(8), time() - st))
+        filename = save_dir + '/' + str(uuid4()) + '.png'
+        save_to_file(btc_slice, filename=filename)
+        print('epoch = {0}, time = {1:.3f}, filename = {2}'.format(str(epoch).zfill(8), time() - st, filename))
 
 
 def main():
-    arg = sys.argv
-    assert len(arg) == 3, 'Usage: python3 {} DATA_FOLDER_TO_STORE_GENERATED_DATASET ' \
-                          'BITCOIN_MARKET_DATA_CSV_PATH'.format(arg[0])
-    data_folder = arg[1]
-    bitcoin_file = arg[2]
-    generate(data_folder, bitcoin_file)
+    args = sys.argv
+    assert len(args) == 4, 'Usage: python3 {} DATA_FOLDER_TO_STORE_GENERATED_DATASET ' \
+                           'BITCOIN_MARKET_DATA_CSV_PATH USE_QUANTILES'.format(args[0])
+    data_folder = args[1]
+    bitcoin_file = args[2]
+    use_quantiles = int(args[3])
+
+    data_gen_func = generate_quantiles if use_quantiles else generate_up_down
+    print('Using: {}'.format(data_gen_func))
+    data_gen_func(data_folder, bitcoin_file)
 
 
 if __name__ == '__main__':

data_manager.py

@@ -15,5 +15,11 @@ def file_processor(data_file):
     v = pd.DataFrame(d['volume'].resample('5Min').sum())
     v.columns = ['volume']
     p['volume'] = v['volume']
+
+    # drop NaN values.
+    # for example sometimes we don't have data for like one hour in a row.
+    # So we have NaN buckets of 5Min in this particular hour.
+    # Our convention is to avoid those NaN values and drop them!
+    p = p.dropna()
     p.to_csv('/tmp/bitcoin_coinbase_M5.csv', sep='\t')
     return p

returns_quantization.py

@@ -6,23 +6,30 @@
 from utils import compute_returns
 
 
-def generate(data_folder, bitcoin_file):
-    p = file_processor(bitcoin_file)
-    close_prices_returns = compute_returns(p)
+def add_returns_in_place(df):  # modifies df
+    close_prices_returns = compute_returns(df)
     num_bins = 10
-    returns_bins = pd.cut(close_prices_returns, num_bins).values.categories
-    returns_labels = pd.cut(close_prices_returns, num_bins, labels=False)
-    print(returns_bins)
-    print(returns_labels)
+    returns_bins = pd.qcut(close_prices_returns, num_bins)
+    bins_categories = returns_bins.values.categories
+    returns_labels = pd.qcut(close_prices_returns, num_bins, labels=False)
+
+    df['close_price_returns'] = close_prices_returns
+    df['close_price_returns_bins'] = returns_bins
+    df['close_price_returns_labels'] = returns_labels
+
+    return df, bins_categories
+
+
+def generate_bins(bitcoin_file):
+    p = file_processor(bitcoin_file)
+    print(add_returns_in_place(p))
 
 
 def main():
     arg = sys.argv
-    assert len(arg) == 3, 'Usage: python3 {} DATA_FOLDER_TO_STORE_GENERATED_DATASET ' \
-                          'BITCOIN_MARKET_DATA_CSV_PATH'.format(arg[0])
-    data_folder = arg[1]
-    bitcoin_file = arg[2]
-    generate(data_folder, bitcoin_file)
+    assert len(arg) == 2, 'Usage: python3 {} BITCOIN_MARKET_DATA_CSV_PATH'.format(arg[0])
+    bitcoin_file = arg[1]
+    generate_bins(bitcoin_file)
 
 
 if __name__ == '__main__':

utils.py

@@ -1,7 +1,7 @@
 def compute_returns(p):
     close_prices = p['price_close']
     close_prices_returns = 100 * ((close_prices.shift(-1) - close_prices) / close_prices).fillna(0.0)
-    return close_prices_returns
+    return close_prices_returns.shift(1).fillna(0)
 
 
 def plot_p(df):