refactor: rename variables and classes / feat: add a FMNIST classifier

Author: HyunjoonCho

DeepHyperion-FMNIST/README.md

@@ -1,4 +1,4 @@
-# DeepHyperion-MNIST #
+# DeepHyperion-FMNIST #
 
 Input test generator using illumination search algorithm
 
@@ -8,7 +8,7 @@ This tool is developed in Python on top of the DEAP evolutionary computation fra
 
 ## Dependencies ##
 
-> NOTE: If you want to use DeepHyperion-MNIST easily without configuring your environment from scratch, you can also see [__Getting Started__](../documentation/getting_started.md)
+> NOTE: If you want to use DeepHyperion-FMNIST easily without configuring your environment from scratch, you can also see [__Getting Started__](../documentation/getting_started.md)
 
 ### Configuring Ubuntu ###
 Pull an Ubuntu Docker image, run and configure it by typing in the terminal:
@@ -29,11 +29,11 @@ apt install -y git
 
 ### Copy the project into the docker container ###
 
-To copy DeepHyperion-MNIST inside the docker container, open another console and run:
+To copy DeepHyperion-FMNIST inside the docker container, open another console and run:
 
 ``` 
 cd <DEEP_HYPERION_HOME>
-docker cp DeepHyperion-MNIST/ <DOCKER_ID>:/
+docker cp DeepHyperion-FMNIST/ <DOCKER_ID>:/
 ```
 
 Where `<DEEP_HYPERION_HOME>` is the location in which you downloaded the artifact and `<DOCKER_ID>` is the ID of the ubuntu docker image just started.
@@ -93,7 +93,7 @@ apt install -y python3-venv
 Create the python virtual environment:
 
 ```
-cd /DeepHyperion-MNIST
+cd /DeepHyperion-FMNIST
 python3 -m venv .venv
 ```
 
@@ -134,23 +134,23 @@ apt install -y libgirepository1.0-dev gcc libcairo2-dev pkg-config python3-dev g
 This tool has other dependencies, including `tensorflow` and `deap`, that can be installed via `pip`:
 
 ```
-cd /DeepHyperion-MNIST
+cd /DeepHyperion-FMNIST
 pip install -r requirements.txt
 ``` 
 
 ## Usage ##
 ### Input ###
 
 * A trained model in h5 format. The default one is in the folder `models`;
-* A list of seeds used for the input generation. In this implementation, the seeds are indexes of elements of the MNIST dataset. The default list is in the file `bootstraps_five`;
+* A list of seeds used for the input generation. In this implementation, the seeds are indexes of elements of the FMNIST dataset. The default list is in the file `bootstraps_five`;
 * `properties.py` containing the configuration of the tool selected by the user.
 
 ### Run the Tool ###
 
 To run the command execute:
 
 ```
-python mapelites_mnist.py
+python mapelites_fmnist.py
 ```
 
 ### Output ###
@@ -164,7 +164,7 @@ When the run is finished, the tool produces the following outputs in the `logs`
 
 ### Generate Processed Data and Rescaled Maps ###
 
-* [__DeepHyperion-MNIST/report_generator__](../DeepHyperion-MNIST/report_generator/README.md)
+* [__DeepHyperion-FMNIST/report_generator__](../DeepHyperion-FMNIST/report_generator/README.md)
 
 
 ## Troubleshooting ##

DeepHyperion-FMNIST/digit_input.py renamed to DeepHyperion-FMNIST/clothes_input.py

@@ -8,12 +8,12 @@
 import numpy as np
 
 
-class Digit:
+class Clothes:
     COUNT = 0
 
     def __init__(self, desc, label, seed):
         self.timestamp, self.elapsed_time = Timer.get_timestamps()
-        self.id = Digit.COUNT
+        self.id = Clothes.COUNT
         self.run = RUN
         self.seed = seed
         # TODO
@@ -24,7 +24,7 @@ def __init__(self, desc, label, seed):
         self.expected_label = label
         self.predicted_label = None
         self.confidence = None
-        Digit.COUNT += 1
+        Clothes.COUNT += 1
 
     def to_dict(self):
         return {'id': str(self.id),
@@ -78,5 +78,5 @@ def export(self, all=False):
         self.save_svg(dst)
 
     def clone(self):
-        clone_digit = Digit(self.xml_desc, self.expected_label, self.seed)
-        return clone_digit
+        clone_clothes = Clothes(self.xml_desc, self.expected_label, self.seed)
+        return clone_clothes

DeepHyperion-FMNIST/digit_mutator.py renamed to DeepHyperion-FMNIST/clothes_mutator.py

@@ -8,13 +8,13 @@
 import keras
 
 
-class DigitMutator:
+class ClothesMutator:
     # load the FMNIST dataset
     fmnist = keras.datasets.fashion_mnist
     (_, _), (x_test, _) = fmnist.load_data()
 
-    def __init__(self, digit):
-        self.digit = digit
+    def __init__(self, clothes):
+        self.clothes = clothes
 
     def mutate(self):
         condition = True
@@ -35,30 +35,30 @@ def mutate(self):
                     mutation = 4
 
             counter_mutations += 1
-            mutant_vector = mutation_manager.mutate(self.digit.xml_desc, mutation, counter_mutations/20)
+            mutant_vector = mutation_manager.mutate(self.clothes.xml_desc, mutation, counter_mutations/20)
             mutant_xml_desc = vectorization_tools.create_svg_xml(mutant_vector)
-            rasterized_digit = rasterization_tools.rasterize_in_memory(mutant_xml_desc)
+            rasterized_clothes = rasterization_tools.rasterize_in_memory(mutant_xml_desc)
 
-            distance_inputs = get_distance(self.digit.purified, rasterized_digit)
+            distance_inputs = get_distance(self.clothes.purified, rasterized_clothes)
 
             if (TSHD_TYPE == '0'):
                 if distance_inputs != 0:
                     condition = False
             elif (TSHD_TYPE == '1'):
-                seed_image = DigitMutator.x_test[int(self.digit.seed)]
+                seed_image = ClothesMutator.x_test[int(self.clothes.seed)]
                 xml_desc = vectorization_tools.vectorize(seed_image)
                 seed = rasterization_tools.rasterize_in_memory(xml_desc)
-                distance_seed = get_distance(seed, rasterized_digit)
+                distance_seed = get_distance(seed, rasterized_clothes)
                 if distance_inputs != 0 and distance_seed <= DISTANCE and distance_seed != 0:
                     condition = False
             elif (TSHD_TYPE == '2'):
-                seed = reshape(DigitMutator.x_test[int(self.digit.seed)])
-                distance_seed = get_distance(seed, rasterized_digit)
+                seed = reshape(ClothesMutator.x_test[int(self.clothes.seed)])
+                distance_seed = get_distance(seed, rasterized_clothes)
                 if distance_inputs != 0 and distance_seed <= DISTANCE_SEED and distance_seed != 0:
                     condition = False
 
-        self.digit.xml_desc = mutant_xml_desc
-        self.digit.purified = rasterized_digit
-        self.digit.predicted_label = None
-        self.digit.confidence = None
+        self.clothes.xml_desc = mutant_xml_desc
+        self.clothes.purified = rasterized_clothes
+        self.clothes.predicted_label = None
+        self.clothes.confidence = None
 

DeepHyperion-FMNIST/features.py

@@ -8,8 +8,8 @@
 NAMESPACE = '{http://www.w3.org/2000/svg}'
 
 
-def bitmap_count(digit, threshold):
-    image = deepcopy(digit.purified)
+def bitmap_count(clothes, threshold):
+    image = deepcopy(clothes.purified)
     bw = np.asarray(image)
     #bw = bw / 255.0
     count = 0
@@ -19,8 +19,8 @@ def bitmap_count(digit, threshold):
     return count
 
 
-def move_distance(digit):
-    root = ET.fromstring(digit.xml_desc)
+def move_distance(clothes):
+    root = ET.fromstring(clothes.xml_desc)
     svg_path = root.find(NAMESPACE + 'path').get('d')
     pattern = re.compile('([\d\.]+),([\d\.]+)\sM\s([\d\.]+),([\d\.]+)')
     segments = pattern.findall(svg_path)
@@ -38,10 +38,10 @@ def move_distance(digit):
         return 0
 
 
-def orientation_calc(digit, threshold):
+def orientation_calc(clothes, threshold):
     x = []
     y = []
-    image = deepcopy(digit.purified)
+    image = deepcopy(clothes.purified)
     bw = np.asarray(image)
     for _, ix, iy, _ in np.ndindex(bw.shape):
         if bw[_, ix, iy, _] > threshold:

DeepHyperion-FMNIST/individual.py

@@ -1,5 +1,5 @@
 import predictor
-from digit_mutator import DigitMutator
+from clothes_mutator import ClothesMutator
 
 
 class Individual(object):
@@ -27,6 +27,6 @@ def evaluate(self):
         return self.ff
 
     def mutate(self):
-        DigitMutator(self.member).mutate()
+        ClothesMutator(self.member).mutate()
         self.reset()
 

DeepHyperion-FMNIST/mapelites_mnist.py renamed to DeepHyperion-FMNIST/mapelites_fmnist.py

@@ -13,7 +13,7 @@
 from feature_dimension import FeatureDimension
 from features import move_distance, bitmap_count, orientation_calc
 import vectorization_tools
-from digit_input import Digit
+from clothes_input import Clothes
 from individual import Individual
 from properties import NGEN, \
     POPSIZE, EXPECTED_LABEL, INITIALPOP, \
@@ -33,16 +33,16 @@
     assert (len(starting_seeds) == POPSIZE)
 
 
-def generate_digit(seed):
+def generate_clothes(seed):
     seed_image = x_test[int(seed)]
     xml_desc = vectorization_tools.vectorize(seed_image)
-    return Digit(xml_desc, EXPECTED_LABEL, seed)
+    return Clothes(xml_desc, EXPECTED_LABEL, seed)
 
 
-class MapElitesMNIST(MapElites):
+class MapElitesFMNIST(MapElites):
 
     def __init__(self, *args, **kwargs):
-        super(MapElitesMNIST, self).__init__(*args, **kwargs)
+        super(MapElitesFMNIST, self).__init__(*args, **kwargs)
 
     def map_x_to_b(self, x):
         """
@@ -100,8 +100,8 @@ def generate_random_solution(self):
             seed = starting_seeds[Individual.COUNT - 1]
             Individual.SEEDS.add(seed)
 
-        digit1 = generate_digit(seed)
-        individual = Individual(digit1, seed)
+        clothes1 = generate_clothes(seed)
+        individual = Individual(clothes1, seed)
         individual.seed = seed
 
         return individual
@@ -198,15 +198,15 @@ def main():
     utils.setup_logging(log_to, debug)
     print("Logging results to " + log_to)
 
-    map_E = MapElitesMNIST(NGEN, POPSIZE, log_dir_name, True)
-    MapElitesMNIST.print_config()
+    map_E = MapElitesFMNIST(NGEN, POPSIZE, log_dir_name, True)
+    MapElitesFMNIST.print_config()
     map_E.run()
 
     Individual.COUNT = 0
 
     print("Exporting inputs ...")
-    for digit in Exploration.all_inputs:
-        digit.export(all=True)
+    for clothes in Exploration.all_inputs:
+        clothes.export(all=True)
 
     print("Done")
 

DeepHyperion-FMNIST/models/cnnClassifier.h5

@@ -8,7 +8,7 @@ class MissingEnvironmentVariable(Exception):
 POPSIZE          = int(os.getenv('DH_POPSIZE', '800'))
 NGEN             = int(os.getenv('DH_NGEN', '500000'))
 
-RUNTIME          = int(os.getenv('DH_RUNTIME', '180'))
+RUNTIME          = int(os.getenv('DH_RUNTIME', '10'))
 INTERVAL         = int(os.getenv('DH_INTERVAL', '60'))
 
 # Mutation Hyperparameters
@@ -31,8 +31,7 @@ class MissingEnvironmentVariable(Exception):
 
 INITIALPOP       = os.getenv('DH_INITIALPOP', 'seeded')
 
-MODEL            = os.getenv('DH_MODEL', 'models/model_mnist.h5')
-#MODEL            = os.getenv('DH_MODEL', 'models/cnnClassifierTest.h5')
+MODEL            = os.getenv('DH_MODEL', 'models/fmnist_cnn.h5')
 
 ORIGINAL_SEEDS   = os.getenv('DH_ORIGINAL_SEEDS', 'bootstraps_five')
 

DeepHyperion-FMNIST/models/model_mnist.h5 renamed to DeepHyperion-FMNIST/models/fmnist_cnn.h5

@@ -9,13 +9,13 @@ and compute metrics of interests (misbehaviors, density, etc.)
 
 We assume to have access to the raw data about each experiment, which consists in a `timestamped` folder that
 contains all the individuals (e.g., `npy` vectors) generated by the test generators and possibly the outcome of the
-test, i.e., misclassified digits for MNIST.
+test, i.e., misclassified clothes for FMNIST.
 
 The steps to generate the maps are:
 
 1. Process the output of the tools and generate an `info_<ID>.json` file for each sample. This file contains the features, and various metadata about the sample (timestamp, sample-id, file location, tool name, run id, etc.)
 
-   To do it, go to the root of the project (`./DeepHyperion-MNIST`) and run the following command to process a dataset folder (folders will be recursively checked):
+   To do it, go to the root of the project (`./DeepHyperion-FMNIST`) and run the following command to process a dataset folder (folders will be recursively checked):
 
    ```
    export LC_ALL=C.UTF-8
@@ -25,7 +25,7 @@ The steps to generate the maps are:
    > NOTE: the set of features to be computed is predefined for each tool.
     
 
-2. Process all the json files corresponding to the samples of all the runs for all the tools, to extract the maps `extrema` for each feature. Go to the root of the project (`./DeepHyperion-MNIST`) and run the following command:
+2. Process all the json files corresponding to the samples of all the runs for all the tools, to extract the maps `extrema` for each feature. Go to the root of the project (`./DeepHyperion-FMNIST`) and run the following command:
 
    ```
    python report_generator/app.py extract-stats --parsable --feature <NAME> --feature <NAME> ./logs/run_XXX/archive

DeepHyperion-FMNIST/properties.py

@@ -177,22 +177,22 @@ def generate_samples(ctx, force_attribute, filter_samples_by_tshd, asfault_n_sec
 
     """
 
-    # --filter-samples-by-tshd applies only to MNIST
-    mnist_filter = None
+    # --filter-samples-by-tshd applies only to FMNIST
+    fmnist_filter = None
     if filter_samples_by_tshd[0] is not None:
         # Make sure the import is done only if needed, this code is problematic if the libraries are not installed
         if ctx.obj['show-progress']:
             print("Filtering samples by TSHD, %s, %s" % (filter_samples_by_tshd))
             from report_generator.tshd_selection import is_valid_digit
-            # Use the is_valid_digit and the input to create a partial function to filter the mnist inputs.
+            # Use the is_valid_digit and the input to create a partial function to filter the fmnist inputs.
             # the partial function accepts now only ONE input, the sample, and is configured automatically with type
             # and val
             # The lambda makes sure that we apply partial correctly
-            mnist_filter = partial( lambda tshd_type, tshd_val, sample :
+            fmnist_filter = partial( lambda tshd_type, tshd_val, sample :
                                                 is_valid_digit(sample, tshd_type, tshd_val),
                                     str(filter_samples_by_tshd[0]), float(filter_samples_by_tshd[1]))
             # Make sure we record the filter setting so we can store it inside th sample file
-            setattr(mnist_filter, "filter_name", "-".join(["tshd",
+            setattr(fmnist_filter, "filter_name", "-".join(["tshd",
                                                            str(filter_samples_by_tshd[0]), str(filter_samples_by_tshd[1])]))
 
     # Setup the generation. Select the
@@ -246,10 +246,10 @@ def generate_samples(ctx, force_attribute, filter_samples_by_tshd, asfault_n_sec
                     if cast_as == "DLFuzz":
                         sample = DLFuzzSample(os.path.splitext(sample_file)[0])
                         # add validity and re-dump if necessaty
-                        if mnist_filter is not None:
-                            is_valid = mnist_filter(sample)
+                        if fmnist_filter is not None:
+                            is_valid = fmnist_filter(sample)
                             sample.is_valid = is_valid
-                            sample.valid_according_to = mnist_filter.filter_name
+                            sample.valid_according_to = fmnist_filter.filter_name
                             # We need this because dump is done automatically in the constructor,
                             # BEFORE we can check for validity
                             sample.dump()
@@ -259,10 +259,10 @@ def generate_samples(ctx, force_attribute, filter_samples_by_tshd, asfault_n_sec
                     elif cast_as == "DeepJanus":
                         sample = DeepJanusSample(os.path.splitext(sample_file)[0])
                         # add validity and re-dump if necessaty
-                        if mnist_filter is not None:
-                            is_valid = mnist_filter(sample)
+                        if fmnist_filter is not None:
+                            is_valid = fmnist_filter(sample)
                             sample.is_valid = is_valid
-                            sample.valid_according_to = mnist_filter.filter_name
+                            sample.valid_according_to = fmnist_filter.filter_name
                             # We need this because dump is done automatically in the constructor,
                             # BEFORE we can check for validity
                             sample.dump()
@@ -271,10 +271,10 @@ def generate_samples(ctx, force_attribute, filter_samples_by_tshd, asfault_n_sec
                     elif cast_as == "DeepHyperion":
                         sample = DeepHyperionSample(os.path.splitext(sample_file)[0])
                         # add validity and re-dump if necessaty
-                        if mnist_filter is not None:
-                            is_valid = mnist_filter(sample)
+                        if fmnist_filter is not None:
+                            is_valid = fmnist_filter(sample)
                             sample.is_valid = is_valid
-                            sample.valid_according_to = mnist_filter.filter_name
+                            sample.valid_according_to = fmnist_filter.filter_name
                             # We need this because dump is done automatically in the constructor,
                             # BEFORE we can check for validity
                             sample.dump()
@@ -320,7 +320,7 @@ def extract_stats(ctx, report_missing_features, parsable, feature, dataset_folde
         feature: the list of name of the features to consider during the analysis
 
         dataset_folder: this is the root folder that contains all the results (i.e., samples as json) from all the
-        tools considered in one analysis (e.g., MNIST, BeamNG)
+        tools considered in one analysis (e.g., FMNIST, BeamNG)
 
         parsable: a flag to generate an easy to parse data