Lazy load pandas

Author: PetrochukM

third_party/lazy_loader.py

@@ -0,0 +1,58 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+""" A LazyLoader class. """
+
+import importlib
+import types
+
+
+class LazyLoader(types.ModuleType):
+    """
+    Lazily import a module, mainly to avoid pulling in large dependencies. `contrib`, and
+    `ffmpeg` are examples of modules that are large and not always needed, and this allows them to
+    only be loaded when they are used.
+    """
+
+    def __init__(self, local_name, parent_module_globals, name):
+        self._local_name = local_name
+        self._parent_module_globals = parent_module_globals
+
+        super(LazyLoader, self).__init__(name)
+
+    def _load(self):
+        """ Load the module and insert it into the parent's globals. """
+
+        # Import the target module and insert it into the parent's namespace
+        module = importlib.import_module(self.__name__)
+        self._parent_module_globals[self._local_name] = module
+
+        # Update this object's dict so that if someone keeps a reference to the
+        #   LazyLoader, lookups are efficient (__getattr__ is only called on lookups
+        #   that fail).
+        self.__dict__.update(module.__dict__)
+
+        return module
+
+    def __call__(self, *args, **kwargs):
+        module = self._load()
+        return module(*args, **kwargs)
+
+    def __getattr__(self, item):
+        module = self._load()
+        return getattr(module, item)
+
+    def __dir__(self):
+        module = self._load()
+        return dir(module)

torchnlp/datasets/dataset.py

@@ -1,4 +1,6 @@
-import pandas as pd
+from third_party.lazy_loader import LazyLoader
+
+pd = LazyLoader('pd', globals(), 'pandas')
 
 from torch.utils import data
 

torchnlp/datasets/simple_qa.py

@@ -1,6 +1,8 @@
 import os
 
-import pandas as pd
+from third_party.lazy_loader import LazyLoader
+
+pd = LazyLoader('pd', globals(), 'pandas')
 
 from torchnlp.datasets.dataset import Dataset
 from torchnlp.download import download_file_maybe_extract