richwu · JamesKBowler · May 23, 2017 · Jun 21, 2017 · Jun 21, 2017 · Jun 21, 2017
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,96 @@
+FROM ubuntu:18.04
+# Based on instructions in readme.md
+#	and https://github.com/JamesKBowler/python-forexconnect/tree/python3-forexconnect
+
+RUN apt-get -y update && apt-get -y install python3 python3-pip \
+    python3-numpy python3-pymysql mariadb-server mariadb-client git \
+    build-essential python3-dev libboost-log-dev libboost-date-time-dev \
+    libboost-python-dev libtool m4 automake autogen checkinstall wget \
+    python3-termcolor sudo libcurl4-gnutls-dev libarchive-dev cmake gosu
+
+RUN pip3 install cprint pytz ipython
+
+RUN adduser --disabled-password --gecos "" nonroot
+
+
+
+ARG WITH_PHPMYADMIN=0
+ENV WITH_PHPMYADMIN=$WITH_PHPMYADMIN
+
+RUN if [ "$WITH_PHPMYADMIN" -gt "0" ] ; then \
+		export DEBIAN_FRONTEND=noninteractive; \
+		echo "phpmyadmin phpmyadmin/dbconfig-install boolean false" | debconf-set-selections; \
+		echo "phpmyadmin phpmyadmin/reconfigure-webserver multiselect apache2" | debconf-set-selections; \
+		apt-get -q -y install phpmyadmin; \
+	fi
+
+
+# --- Install libuv
+
+RUN wget https://github.com/libuv/libuv/archive/v1.19.1/libuv-1.19.1.tar.gz && \
+	tar -xvzf libuv-1.19.1.tar.gz && \
+	cd libuv-1.19.1/ && \
+	sh autogen.sh  && \
+	./configure --prefix=/usr --disable-static && \
+	make && \
+	sudo checkinstall
+
+# --- Install libarchive
+
+#  -> Using distribution packages
+#RUN wget http://www.libarchive.org/downloads/libarchive-3.3.2.tar.gz && \
+#	tar -xvzf libarchive-3.3.2.tar.gz && \
+#	cd libarchive-3.3.2/ && \
+#	./configure --prefix=/usr --disable-static && \
+#	make && \
+#	sudo checkinstall
+
+# --- Install curl
+
+#  -> Using distribution packages
+
+# --- Installing boost
+
+#  -> Using distribution packages, Bionic is already at 1.65.
+RUN apt-get -y install libboost-all-dev
+
+# --- Install python3-forexconnect
+
+RUN cd /home/nonroot; \
+	echo "export BOOST_ROOT=/usr" >> .profile; \
+	echo "export BOOST_INCLUDEDIR=/usr/include/" >> .profile; \
+	echo "export BOOST_LIBRARYDIR=/usr/lib/" >> .profile; \
+	echo "export INCLUDE=/usr/include/boost/:\$INCLUDE" >> .profile; \
+	echo "export LIBRARY_PATH=/usr/local/:\$LIBRARY_PATH" >> .profile; \
+	echo "export FOREXCONNECT_ROOT=\$(pwd)/ForexConnectAPI" >> .profile; \
+	echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$(pwd)/ForexConnectAPI/lib" >> .profile; \
+	. ./.profile; \
+	ldconfig
+
+RUN cd /home/nonroot; \
+	su - nonroot -c "wget http://fxcodebase.com/bin/forexconnect/1.4.1/ForexConnectAPI-1.4.1-Linux-x86_64.tar.gz && \
+		tar xvf ForexConnectAPI-1.4.1-Linux-x86_64.tar.gz && \  
+		mv ForexConnectAPI-1.4.1-Linux-x86_64 ForexConnectAPI && \
+		git clone -b python3-forexconnect https://github.com/JamesKBowler/python-forexconnect.git && \
+		cd python-forexconnect && mkdir build && cd build && \  
+		cmake .. -DDEFAULT_FOREX_URL='http://www.fxcorporate.com/Hosts.jsp' " && \
+	cd python-forexconnect/build && \
+	make install  
+
+# --- Database setup
+
+RUN service mysql start && \
+	echo 	"CREATE USER 'sec_master'@'localhost' IDENTIFIED BY 'password';  \
+			GRANT ALL PRIVILEGES ON *.* TO 'sec_master'@'localhost';  \
+			FLUSH PRIVILEGES; \
+			set global max_connections = 1000; " | mysql -uroot
+
+RUN echo "127.0.0.1:sec_master:password" > /home/nonroot/.database_sec_master_credentials; \
+	chown nonroot:nonroot /home/nonroot/.database_sec_master_credentials
+
+
+COPY . /home/nonroot/fxcollect/
+
+RUN chown -R nonroot:nonroot /home/nonroot; chmod +x /home/nonroot/fxcollect/runMainDocker.sh;
+
+ENTRYPOINT ["/home/nonroot/fxcollect/runMainDocker.sh"] 
diff --git a/LICENSE b/LICENSE
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017 James K Bowler
+Copyright (c) 2018 James Bowler
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

diff --git a/README.md b/README.md
@@ -1,106 +1,100 @@
-# fxcmminer v1.0
+# fxcollect
 
-The purpose of this software is to fully automate the simultaneous collection of historical and live financial data from FXCM, then store these data in a database ready for backtesting or live execution.
+The purpose of 'fxcollect' is to automate the collection of historical and live financial  
+time series data from FXCM, then store these data in a MariaDB database ready for backtesting  
+or live execution.  
 
-### Setup
-The development has been carried out on an ESXi server, however this will run on any hypervisor and most hardware.
+### Quick Setup (using docker)
+You can use the provided dockerfile to quickly and painlessly setup the application. It has been 
+tested on a debian-based x86_64 distribution, but will likely work on many others. 
 
-Specification:  
-   2x CPUs  
-   4GB RAM  
-   100GB SSD/HDD (for testing hard drive is fine)  
-
-Total drive usage for the database (excluding OS etc) from 59 instruments, is 41GB as of 29/04/2017
-
-Operating System:  
-   Ubuntu Server 16.04 (without GUI) for testing, however this will run on most Linux OS with/without GUI.  
-
-1. First install all dependencies in the requiements.txt and Python 2.7
-
-2. Install MariaDB 10.x  
-  https://mariadb.org/download/
-
-3. Setup MariaDB to allow the user 'sec_master' to access the database with read and write permissions.
-
-   `$ mysql -u root -p`  
+ 1. Modify `settings.py` to at least add your FXCM login credentials.
+ 2. Modify `main.py` to adjust which symbols will be collected. 
+ 3. Browse into the top directory of the repository and build the docker container:
 
-   `mysql> CREATE USER 'sec_master'@'localhost' IDENTIFIED BY 'password';`  
-   `mysql> GRANT ALL PRIVILEGES ON '*.*' TO 'sec_master'@'localhost';`  
-   `mysql> FLUSH PRIVILEGES;`  
-   `mysql> set global max_connections = 1000;`  
-
-   (optional)  
-   `$ sudo service mysql restart`  
-
-4. Download forexconnect and follow instructions.  
- https://github.com/JamesKBowler/python-forexconnect  
-
-5. Download this repository and place in a convenient location
-
-6. Create a logs folder in the root directory  
- `$ mkdir ~/fxcmminer/fxcmminer_v1.0/logs`  
+``` 
+$ cd fxcollect 
+$ docker build .
+```
+Or you can use the additional `WITH_PHPMYADMIN` argument to build container with phpmyadmin, to be able to easily
+inspect the data right from the container:
+
+    $ docker build --build-arg WITH_PHPMYADMIN=1 .
+
+ 4. Run the container, optionally forward ports to browse using phpmyadmin.
+```
+$ docker run -p 127.0.0.1:8080:80/tcp -it <built-image-id>
+```
+
+
+### Manual Setup
+Specification: 
+ - Ubuntu Server 16.04  
+ - Python 3.x  
+ - 2x CPUs  
+ - 4GB RAM, but 2GB will work if tracking a few offers.  
+ - 100GB SSD/HDD (for testing hard drive is fine)  
+ 59 instruments = 41GB as of 29/04/2017  
 
-7. Set the system time zone to America/New_York, this is important as all data on FXCM servers are stored in America/New_York time zone.  
- `$ sudo timedatectl set-timezone America/New_York`  
-
-8. To start the process just execute:  
- `$ python ~/fxcmminer/fxcmminer_v1.0/engine.py`  
-
-##### All FXCM data will take about 30 hours to download.
-
-If you need assistance setting this up or find any bugs, please report using the Issue section.
-
-### TODO:
-
-1. Prioritize queue so that lower time frame data is written to database before higher time frame.
-
-2. Improve logging
-
-3. Add auto offer removal to the fxscout
-
-4. Clean up code!
-
-### Code Explanation
-
-#### Scout
-
-The process of collecting data is started by executing the engine.py script. This in turn will start the fxscout.py, who's primary job is to scout FXCM for currently tradable instruments (also know as 'offers'). Once the Scout has found 'offers' available, it will contact FXCM for the .xml catalogue and make a local copy to be accessed later by fxcmminer. If FXCM add another 'offer' the Scout will then make a new local copy of the catalogue. 
-
-The scout will continue checking FXCM for the entire duration whilst the program is running, and will only place an 'OFFER' event in the events queue on system startup or if a new 'offer' is added in the future.
-
-#### DatabaseManager
-
-If an 'OFFER' event is placed in the queue, the Engine class will pass the event over to the DatabaseManager located in db_manager.py. DatabaseManager will compare its local database with the offer. If the database already exists the creation is skipped, if not the corresponding database and tables for the following time frames will be created.
-
-  `{GBP/USD : ['M1','W1','D1','H8', 'H4', 'H2', 'H1','m30', 'm15', 'm5', 'm1']}`
-
-The schema is one database per offer as this will provide plenty of space for future expansion.
-
-#### HistoricalCollector 
-
-After a database check or creation has been carried out, a 'DBReady' event is placed into the queue, which is then passed over to the HistoricalCollector class located in historical.py .
-The HistoricalCollector asks the DatabaseManager for the lastest date in the database, if this is a new offer or first time system startup, the DatabaseManager will return a date from the .xml catalogue. If the catalog does not have a corresponding date an artificial low date of 2007-01-01 00:00:00 is returned.
-Now the HistoricalCollector has a starting point, it will begin to call FXCM's API and collect data. Once data is returned, a 'HISTDATA' event is created and placed in the queue, which in turn will be passed to the DatabaseManager and written to the database.
-After all historical data has been collected for the offer, a 'LIVEREADY' event is placed into the queue and the HistoricalCollector process will exit.
-
-#### TimeKeeper
-The apscheduler will fire off at market invertals such as 1 minute, 5 minutes etc. On each fire a 'GETLIVE' event is placed into the queue.
-
-#### LiveDataMiner 
-
-On receipt of the 'LIVEREADY' event, LiveDataMiner located in live.py, will update its current list of 'LIVEREADY' offers, and is now waiting for a 'GETLIVE' event from TimeKeeper. On receipt of an 'GETLIVE' event, LiveDataMiner will loop through the list of live ready offers for the corrsponding 'GETLIVE' event time_frame. Once data is collected a 'LIVEDATA' event is placed into the queue for the processing by the DatabaseManager and written to the database.
-
-#### Other classes 
-
-TimeDelta will provide a range in minutes for each time frame, that will not exceed to 300 bars (data points). This is because the maximum bars return for any one API call to FXCM is 300.
-Using the information from TimeDelta, DateRange will provide a date block which is used when calling the API. At the moment this is the only way I could exclude calling for data at the weekends whilst FXCM is closed, and due to the nature of the foreign exchange, holidays are not equal in all countries.
-
-#### Queues
-There are two queues one for historical data and one for live data.
+ 1. Install dependencies
+  - numpy  
+  - pymysql  
+  - cprint  
+  - pytz 
+  - termcolor 
+
+ 2. Install MariaDB 10.x  
+   https://mariadb.org/download/
+
+ 3. Setup MariaDB to allow the user 'sec_master' to access the database with read and write permissions.  
+
+    `$ mysql -u root -p`  
+
+    `mysql> CREATE USER 'sec_master'@'localhost' IDENTIFIED BY 'password';`  
+    `mysql> GRANT ALL PRIVILEGES ON *.* TO 'sec_master'@'localhost';`  
+    `mysql> FLUSH PRIVILEGES;`  
+    `mysql> set global max_connections = 1000;`  
+
+    (optional)  
+    `$ sudo service mysql restart`  
+
+ 4. Download forexconnect and follow instructions.  
+     https://github.com/JamesKBowler/python-forexconnect/tree/python3-forexconnect   
+
+ 5. Download this repository  
+  `$ git clone https://github.com/JamesKBowler/fxcollect ~/`  
+
+ ##### FXCM price data stored in UTC time zone.  
+ 6. Set the system time zone to UTC  
+  `$ sudo timedatectl set-timezone UTC`  
+
+ 7. Execute:  
+  `$ python3 ~/fxcollect/fx_collect/main.py`  
+
+##### Each instrument symbol data will take about 1 hour to download, then data will be collected at every time frame interval   
+        +---------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+--------+
+        | date                | bidopen   | bidhigh   | bidlow    | bidclose  | askopen   | askhigh   | asklow    | askclose  | volume |
+        +---------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+--------+
+        | 2017-04-27 10:01:00 | 17.294000 | 17.296000 | 17.289000 | 17.290000 | 17.340000 | 17.340000 | 17.334000 | 17.335000 |    113 |
+        | 2017-04-27 10:02:00 | 17.290000 | 17.298000 | 17.285000 | 17.295000 | 17.335000 | 17.342000 | 17.330000 | 17.340000 |    114 |
+        | 2017-04-27 10:03:00 | 17.295000 | 17.301000 | 17.289000 | 17.299000 | 17.340000 | 17.347000 | 17.340000 | 17.344000 |     98 |
+        | 2017-04-27 10:04:00 | 17.299000 | 17.300000 | 17.286000 | 17.295000 | 17.344000 | 17.345000 | 17.330000 | 17.340000 |    124 |
+        | 2017-04-27 10:05:00 | 17.295000 | 17.295000 | 17.285000 | 17.292000 | 17.340000 | 17.340000 | 17.330000 | 17.336000 |    130 |
+        | 2017-04-27 10:06:00 | 17.292000 | 17.292000 | 17.279000 | 17.292000 | 17.336000 | 17.336000 | 17.328000 | 17.332000 |     65 |
+        | 2017-04-27 10:07:00 | 17.292000 | 17.304000 | 17.287000 | 17.298000 | 17.332000 | 17.348000 | 17.332000 | 17.345000 |    144 |
+        | 2017-04-27 10:08:00 | 17.298000 | 17.306000 | 17.297000 | 17.302000 | 17.345000 | 17.350000 | 17.343000 | 17.346000 |     96 |
+        | 2017-04-27 10:09:00 | 17.302000 | 17.303000 | 17.294000 | 17.294000 | 17.346000 | 17.346000 | 17.338000 | 17.338000 |     50 |
+        | 2017-04-27 10:10:00 | 17.294000 | 17.296000 | 17.281000 | 17.291000 | 17.338000 | 17.338000 | 17.328000 | 17.333000 |     50 |
+
+If you need assistance setting this up or find any bugs, please report using the Issue section.  
+
+### TODO:  
+1. Update C++ code  
+2. Update documentation  
 
 # License Terms  
 
-Copyright (c) 2017 James K Bowler  
+## Copyright (c) 2017 James K Bowler  
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:  
 
@@ -109,5 +103,4 @@ The above copyright notice and this permission notice shall be included in all c
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  
 
 # Forex Trading Disclaimer  
-
 Trading foreign exchange on margin carries a high level of risk, and may not be suitable for all investors. Past performance is not indicative of future results. The high degree of leverage can work against you as well as for you. Before deciding to invest in foreign exchange you should carefully consider your investment objectives, level of experience, and risk appetite. The possibility exists that you could sustain a loss of some or all of your initial investment and therefore you should not invest money that you cannot afford to lose. You should be aware of all the risks associated with foreign exchange trading, and seek advice from an independent financial advisor if you have any doubts.
diff --git a/__init__.py b/__init__.py
diff --git a/fx_collect/__init__.py b/fx_collect/__init__.py
diff --git a/fx_collect/broker/__init__.py b/fx_collect/broker/__init__.py
diff --git a/fx_collect/broker/base.py b/fx_collect/broker/base.py
@@ -0,0 +1,9 @@
+from abc import ABCMeta
+
+class AbstractBroker(object):
+    """
+    TODO doc string.....
+    """
+    __metaclass__ = ABCMeta
+
+    pass
diff --git a/fx_collect/broker/fxcm/__init__.py b/fx_collect/broker/fxcm/__init__.py
diff --git a/fx_collect/broker/fxcm/base.py b/fx_collect/broker/fxcm/base.py
@@ -0,0 +1,77 @@
+from ..base import AbstractBroker
+from abc import ABCMeta
+from forexconnect import (
+    ForexConnectHistoryClient,
+    ForexConnectOffersClient,
+    ForexConnectTradingClient,
+    set_log_level
+)
+from traceback import print_exc
+from ...settings import FXCM_CREDENTIALS, FXCM_CREDENTIALS_FILE
+
+set_log_level(9)
+
+
+
+class AbstractFXCMBroker(AbstractBroker):
+    """
+    The AbstractFXCMBroker object is designed to interact directly
+    with FXCM using the python-forexconnect API.
+    """
+
+    __metaclass__ = ABCMeta
+
+    def url(self):
+        return 'http://www.fxcorporate.com/Hosts.jsp'
+
+    def whoami(self):
+        return 'fxcm'
+
+    def supported_timeframes(self):
+        return [
+            'm1', 'm5', 'm15', 'm30', 'H1',
+            'H2', 'H4', 'H8', 'D1', 'W1', 'M1'
+        ]
+
+    def is_connected(self):
+        return self._session.is_connected()
+
+    def _offers_table(self):
+        return self._create_session(ForexConnectOffersClient)
+
+    def _market_data(self):
+        return self._create_session(ForexConnectHistoryClient)
+
+    def _trading(self):
+        return self._create_session(ForexConnectTradingClient)
+
+    def _create_session(self, session_type):    
+        import time
+        rest = 0
+        if FXCM_CREDENTIALS:
+            credentials = FXCM_CREDENTIALS
+        else:
+            if FXCM_CREDENTIALS_FILE:
+                dir = FXCM_CREDENTIALS_FILE
+                with open(dir) as f:
+                    credentials = f.readlines()
+                    credentials = credentials[0]
+            else:
+                raise (Exception("No FXCM credentials found. Modify settings.py before running the application."))
+
+        fxcm_env, user, passwd = credentials.strip().split(':') 
+        while True:
+            if rest<60: rest+=1
+            try:
+                session = session_type(
+                    user.encode(), passwd.encode(),
+                    fxcm_env.encode(), self.url().encode()
+                )
+                if session.is_connected():
+                    return session
+            except RuntimeError as e:
+                print_exc()
+                time.sleep(rest)
+
+    def _logout_session(self):
+        self._session.logout()