Merge branch 'language-data-loader' into feature-iso-date-format-with-non-english-language

Author: waqasshabbir

dateparser/date.py

@@ -11,7 +11,9 @@
 from dateparser.date_parser import date_parser
 from dateparser.freshness_date_parser import freshness_date_parser
 from dateparser.languages import default_language_loader
+from dateparser.languages.loader import LanguageDataLoader
 from dateparser.languages.detection import AutoDetectLanguage, ExactLanguages
+from dateparser.conf import settings
 
 
 def sanitize_spaces(html_string):
@@ -244,10 +246,16 @@ class DateDataParser(object):
     :raises:
             ValueError - Unknown Language, TypeError - Languages argument must be a list
     """
+    _default_args = None
+    _skip_tokens = None
 
     def __init__(self, languages=None, allow_redetect_language=False):
+        available_language_map = default_language_loader.get_language_map()
+
+        self._default_args = [languages, allow_redetect_language]
+        self._skip_tokens = settings.SKIP_TOKENS
+
         if isinstance(languages, (list, tuple, collections.Set)):
-            available_language_map = default_language_loader.get_language_map()
 
             if all([language in available_language_map for language in languages]):
                 languages = [available_language_map[language] for language in languages]
@@ -258,12 +266,14 @@ def __init__(self, languages=None, allow_redetect_language=False):
             raise TypeError("languages argument must be a list (%r given)" % type(languages))
 
         if allow_redetect_language:
-            self.language_detector = AutoDetectLanguage(languages=languages if languages else None,
-                                                        allow_redetection=True)
+            self.language_detector = AutoDetectLanguage(
+                    languages if languages else available_language_map.values(),
+                    allow_redetection=True)
         elif languages:
             self.language_detector = ExactLanguages(languages=languages)
         else:
-            self.language_detector = AutoDetectLanguage(languages=None, allow_redetection=False)
+            self.language_detector = AutoDetectLanguage(
+                available_language_map.values(), allow_redetection=False)
 
     def get_date_data(self, date_string, date_formats=None):
         """
@@ -304,6 +314,11 @@ def get_date_data(self, date_string, date_formats=None):
         TODO: Timezone issues
 
         """
+        global default_language_loader
+        if settings.SKIP_TOKENS != self._skip_tokens:
+            default_language_loader = LanguageDataLoader()
+            self = DateDataParser(*self._default_args)
+
         date_string = date_string.strip()
         date_string = sanitize_date(date_string)
 

dateparser/languages/detection.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 from functools import wraps
 
-from dateparser.languages import default_language_loader
-
 
 def _restore_languages_on_generator_exit(method):
     @wraps(method)
@@ -39,9 +37,7 @@ def _filter_languages(date_string, languages):
 
 
 class AutoDetectLanguage(BaseLanguageDetector):
-    def __init__(self, languages=None, allow_redetection=False):
-        if languages is None:
-            languages = default_language_loader.get_languages()
+    def __init__(self, languages, allow_redetection=False):
         super(AutoDetectLanguage, self).__init__(languages=languages[:])
         self.language_pool = languages[:]
         self.allow_redetection = allow_redetection

dateparser/languages/loader.py

@@ -4,10 +4,12 @@
 from yaml import load as load_yaml
 
 from .language import Language
+from ..conf import settings
 
 
 class LanguageDataLoader(object):
     _data = None
+    _raw_data = None
 
     def __init__(self, file=None):
         if isinstance(file, basestring):
@@ -34,10 +36,14 @@ def _load_data(self):
             data = get_data('data', 'languages.yaml')
         else:
             data = self.file.read()
-        data = load_yaml(data)
-        base_data = data.pop('base', {})
+        
+        if not self._raw_data:
+            self._raw_data = load_yaml(data)
+
+        base_data = self._raw_data.pop('base', {'skip': []})
+        base_data['skip'] += settings.SKIP_TOKENS
         known_languages = {}
-        for shortname, language_info in data.iteritems():
+        for shortname, language_info in self._raw_data.iteritems():
             self._update_language_info_with_base_info(language_info, base_data)
             language = Language(shortname, language_info)
             if language.validate_info():