Adjusted the rest of the database files.

Author: Michal-Fularz

decision_trees/datasets/boston_house_prices_raw.py

@@ -1,3 +1,4 @@
+from typing import Tuple
 import numpy as np
 from sklearn import datasets
 from sklearn.utils import shuffle
@@ -10,7 +11,7 @@ def __init__(self, number_of_train_samples: int, number_of_test_samples: int):
         self._number_of_train_samples = number_of_train_samples
         self._number_of_test_samples = number_of_test_samples
 
-    def _load_data(self):
+    def load_data(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
         boston = datasets.load_boston()
         # print(boston.data.shape)
         # print(boston.target.shape)
@@ -24,12 +25,15 @@ def _load_data(self):
         test_data = boston.data[self._number_of_train_samples:self._number_of_train_samples+self._number_of_test_samples]
         test_target = boston.target[self._number_of_train_samples:self._number_of_train_samples+self._number_of_test_samples]
 
+        # TODO(MF): insert normalisation routine here
+
         return train_data, train_target, test_data, test_target
 
     @staticmethod
     def _normalise(data: np.ndarray):
         # in case of MNIST data it is possible to just divide each data by maximum value
         # each feature is in range 0-255
+        # TODO(MF): add normalisation
         data = data / 255
 
         return data
@@ -56,13 +60,10 @@ def test_boston_raw():
 if __name__ == "__main__":
     d = BostonRaw(400, 100)
 
-    train_data, train_target, test_data, test_target = d._load_data()
+    train_data, train_target, test_data, test_target = d.load_data()
 
     print(f"train_data.shape: {train_data.shape}")
 
-    # train_data = d._normalise(train_data)
-    # test_data = d._normalise(test_data)
-
     from decision_trees import dataset_tester
 
     # dataset_tester.perform_experiment(train_data[:60000], train_target[:60000],
@@ -83,4 +84,4 @@ def test_boston_raw():
     # dataset_tester.test_dataset(40,
     #                             train_data, train_target, test_data, test_target,
     #                             dataset_tester.ClassifierType.random_forest_regressor,
-    #                             )
+    #                             )

decision_trees/datasets/digits_raw.py

@@ -1,3 +1,4 @@
+from typing import Tuple
 import matplotlib.pyplot as plt
 import numpy as np
 from sklearn import datasets
@@ -56,7 +57,7 @@ def __init__(self, number_of_train_samples: int, number_of_test_samples: int):
         self._number_of_train_samples = number_of_train_samples
         self._number_of_test_samples = number_of_test_samples
 
-    def _load_data(self):
+    def load_data(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
         digits = datasets.load_digits()
         # print(digits.data.shape)
         # print(digits.target.shape)
@@ -66,6 +67,8 @@ def _load_data(self):
         data = digits.data.reshape((len(digits.data), -1))
         # print(len(data))
 
+        data = self._normalise(data)
+
         train_data = data[:self._number_of_train_samples]
         train_target = digits.target[:self._number_of_train_samples]
         test_data = data[
@@ -108,5 +111,5 @@ def test_digits_raw():
 
 
 if __name__ == "__main__":
-    #sample_from_scikit()
+    # sample_from_scikit()
     test_digits_raw()

decision_trees/datasets/emg_raw.py

@@ -1,7 +1,6 @@
+from typing import Tuple, List
 import csv
 import os
-from typing import Tuple, List
-
 import numpy as np
 
 from decision_trees.datasets.dataset_base import DatasetBase
@@ -50,7 +49,7 @@ def _load_files(self, files_paths: List[str], is_output: bool) -> np.ndarray:
 
         return data_array
 
-    def _load_data(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    def load_data(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
         input_train_files = []
         output_train_files = []
         input_test_files = []
@@ -71,6 +70,9 @@ def _load_data(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
         input_test_data = self._load_files(input_test_files, is_output=False)
         output_test_data = self._load_files(output_test_files, is_output=True)
 
+        input_train_data = self._normalise(input_train_data)
+        input_test_data = self._normalise(input_test_data)
+
         return input_train_data, output_train_data, input_test_data, output_test_data
 
     def _normalise(self, data: np.ndarray):
@@ -83,16 +85,13 @@ def _normalise(self, data: np.ndarray):
 if __name__ == "__main__":
     d = EMGRaw("./../../data/EMG/")
 
-    train_data, train_target, test_data, test_target = d._load_data()
+    train_data, train_target, test_data, test_target = d.load_data()
 
     print(f"train_data.shape: {train_data.shape}")
     print(f"test_data.shape: {test_data.shape}")
     print(f"np.unique(train_target): {np.unique(train_target)}")
     print(f"np.unique(test_target): {np.unique(test_target)}")
 
-    train_data = d._normalise(train_data)
-    test_data = d._normalise(test_data)
-
     from decision_trees import dataset_tester
 
     dataset_tester.perform_gridsearch(train_data[:19000], train_target[:19000],
@@ -106,4 +105,4 @@ def _normalise(self, data: np.ndarray):
     #                             train_data[:19000], train_target[:19000],
     #                             test_data[:10000], test_target[:10000],
     #                             dataset_tester.ClassifierType.RANDOM_FOREST,
-    #                             )
+    #                             )

decision_trees/datasets/fashion_mnist_raw.py

@@ -1,5 +1,4 @@
 import numpy as np
-from sklearn.utils import shuffle
 from typing import Tuple
 
 from decision_trees.datasets.dataset_base import DatasetBase
@@ -13,13 +12,13 @@ class FashionMnistRaw(DatasetBase):
     def __init__(self):
         ...
 
-    def _load_data(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    def load_data(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
         X_train, y_train = load_mnist('./../../submodules/fashion-mnist/data/fashion', kind='train')
         X_test, y_test = load_mnist('./../../submodules/fashion-mnist/data/fashion/', kind='t10k')
 
-        train_data = X_train
+        train_data = self._normalise(X_train)
         train_target = y_train
-        test_data = X_test
+        test_data = self._normalise(X_test)
         test_target = y_test
 
         return train_data, train_target, test_data, test_target
@@ -48,17 +47,14 @@ def test_mnist_raw():
     assert True
 
 
-if __name__ == "__main__":
+def main():
     d = FashionMnistRaw()
 
-    train_data, train_target, test_data, test_target = d._load_data()
+    train_data, train_target, test_data, test_target = d.load_data()
 
     print(f"train_data.shape: {train_data.shape}")
     print(f"np.unique(test_target): {np.unique(test_target)}")
 
-    train_data = d._normalise(train_data)
-    test_data = d._normalise(test_data)
-
     from decision_trees import dataset_tester
 
     dataset_tester.perform_gridsearch(train_data[:60000], train_target[:60000],
@@ -72,3 +68,7 @@ def test_mnist_raw():
     #                             train_data[:60000], train_target[:60000], test_data[:10000], test_target[:10000],
     #                             dataset_tester.ClassifierType.DECISION_TREE,
     #                             )
+
+
+if __name__ == "__main__":
+    main()

decision_trees/datasets/inria_hog.py

@@ -1,5 +1,5 @@
+from typing import Tuple
 import pickle
-
 import numpy as np
 
 from decision_trees.datasets.dataset_base import DatasetBase
@@ -13,7 +13,7 @@ def __init__(self, data_filename: str, nr_pos_train: int, nr_pos_test: int, nr_n
         self._nr_neg_train = nr_neg_train
         self._nr_neg_test = nr_neg_test
 
-    def _load_data(self):
+    def load_data(self)-> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
         # prepare the training data
         with open("..\\data\\positive_train_" + self._data_filename + ".pickle", "rb") as f:
             train_data_positive = pickle.load(f)
@@ -69,5 +69,6 @@ def test_inria_hog():
 
     assert True
 
+
 if __name__ == "__main__":
     test_inria_hog()

decision_trees/datasets/mnist_raw.py

@@ -1,3 +1,4 @@
+from typing import Tuple
 import numpy as np
 from sklearn import datasets
 from sklearn.utils import shuffle
@@ -10,7 +11,7 @@ def __init__(self, number_of_train_samples: int, number_of_test_samples: int):
         self._number_of_train_samples = number_of_train_samples
         self._number_of_test_samples = number_of_test_samples
 
-    def _load_data(self):
+    def load_data(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
         mnist = datasets.fetch_mldata('MNIST original', data_home=".//data//MNIST//")
         # print(mnist.data.shape)
         # print(mnist.target.shape)
@@ -19,10 +20,14 @@ def _load_data(self):
         # it is necessary to shuffle the data as all 0's are at the front and all 9's are at the end
         mnist.data, mnist.target = shuffle(mnist.data, mnist.target)
 
-        train_data = mnist.data[:self._number_of_train_samples]
+        train_data = self._normalise(mnist.data[:self._number_of_train_samples])
         train_target = mnist.target[:self._number_of_train_samples]
-        test_data = mnist.data[self._number_of_train_samples:self._number_of_train_samples+self._number_of_test_samples]
-        test_target = mnist.target[self._number_of_train_samples:self._number_of_train_samples+self._number_of_test_samples]
+        test_data = self._normalise(
+            mnist.data[self._number_of_train_samples:self._number_of_train_samples+self._number_of_test_samples]
+        )
+        test_target = mnist.target[
+                      self._number_of_train_samples:self._number_of_train_samples+self._number_of_test_samples
+                      ]
 
         return train_data, train_target, test_data, test_target