PR244 follow-up: allow overriding non-essential GLOBAL and SITE conf options through include sections

mig/shared/configuration.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -20,7 +20,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 # -- END_HEADER ---
 #
@@ -56,13 +56,14 @@
         valid_filter_methods, default_twofactor_auth_apps, \
         mig_conf_section_dirname
     from mig.shared.logger import Logger, SYSLOG_GDP
     from mig.shared.htmlgen import menu_items, vgrid_items
     from mig.shared.fileio import read_file, load_json, write_file
 except ImportError as ioe:
     print("could not import migrid modules")
 
 
-def include_section_contents(logger, config, section, load_path, verbose=False):
+def include_section_contents(logger, config, section, load_path, verbose=False,
+                             reject_overrides=[]):
     """Include additional section contents from load_path in config."""
     if not os.path.exists(load_path):
         msg = "no such %r section config in %s" % (section, load_path)
@@ -90,6 +91,13 @@
         logger.debug("add %r section to main config" % section)
         config.add_section(section)
     for (key, val) in section_config.items(section):
+        if key in reject_overrides:
+            msg = "reject override core option %r in %r section" % (key,
+                                                                    section)
+            if verbose:
+                print(msg)
+            logger.warning(msg)
+            continue
         logger.debug("add config key %r in %r section" % (key, section))
         config.set(section, key, val)
     logger.debug("done including %r section to main config" % section)
@@ -108,9 +116,9 @@
     if val.find('::') == -1:
         # logger.debug("nothing to expand in %r" % val)
         return val
-    #logger.debug("expand any ENV and FILE content in %r" % val)
+    # logger.debug("expand any ENV and FILE content in %r" % val)
     env_pattern = "[a-zA-Z][a-zA-Z0-9_]+"
     cache_pattern = file_pattern = "[^ ]+"
     expanded = val
     expand_pairs = [(keyword_env, env_pattern), (keyword_file, file_pattern)]
     for (key, pattern) in expand_pairs:
@@ -138,7 +146,7 @@
                     #    "reading conf content from file in %r" % cache)
                     content = read_file(path, logger)
                     if cache and content:
-                        #logger.debug("caching conf content salt in %r" % cache)
+                        # logger.debug("caching conf content salt in %r" % cache)
                         write_file(content, cache, logger)
                 if not content:
                     logger.warning("salt file not found or empty: %s" % path)
@@ -159,7 +167,7 @@
     return expanded
 
 
 def fix_missing(config_file, verbose=True):
     """Add missing configuration options - used by checkconf script"""
 
     config = ConfigParser()
@@ -212,7 +220,7 @@
         'server_home': '~/state/server_home/',
         'webserver_home': '~/state/webserver_home/',
         'sessid_to_mrsl_link_home': '~/state/sessid_to_mrsl_link_home/',
         'sessid_to_jupyter_mount_link_home': '~/state/sessid_to_jupyter_mount_link_home/',
         'mig_system_files': '~/state/mig_system_files/',
         'mig_system_storage': '~/state/mig_system_storage',
         'mig_system_run': '~/state/mig_system_run/',
@@ -351,7 +359,7 @@
         'architectures': 'X86 AMD64 IA64 SPARC SPARC64 ITANIUM SUN4U SPARC-T1',
         'scriptlanguages': 'sh python java',
         'jobtypes': 'batch interactive bulk all',
         'lrmstypes': 'Native Native-execution-leader Batch Batch-execution-leader',
     }
     scheduler_section = {'algorithm': 'FairFit',
                          'expire_after': '99999999999',
@@ -389,7 +397,7 @@
         'QUOTA': quota_section,
     }
     for section in defaults:
         if not section in config.sections():
             config.add_section(section)
 
     modified = False
@@ -403,7 +411,7 @@
                 modified = True
     if modified:
         backup_path = '%s.%d' % (config_file, time.time())
         print('Backing up existing configuration to %s as update removes all comments'
               % backup_path)
         fd = open(config_file, 'r')
         backup_fd = open(backup_path, 'w')
@@ -681,7 +689,7 @@
                        'VERIFYFILES', 'VGRID', 'SANDBOX'],
     'job_cond_yellow': ['DISK', 'MEMORY', 'CPUTIME'],
     'job_cond_orange': ['CPUCOUNT', 'NODECOUNT'],
     'job_cond_red': ['EXECUTABLES', 'INPUTFILES', 'REGISTERED', 'SEEN_WITHIN_X'],
     'enable_suggest': False,
     'suggest_threshold': 'GREEN',
 
@@ -725,7 +733,7 @@
     def __init__(self, config_file, verbose=False, skip_log=False,
                  disable_auth_log=False):
         self.config_file = config_file
         self.mig_server_id = None
         # Explicitly init a few helpers hot-plugged and used in ways where it's
         # less obvious if they are always guaranteed to already be initialized.
         self.default_page = None
@@ -761,7 +769,7 @@
         try:
             if self.logger:
                 self.logger.info('reloading configuration and reopening log')
         except:
             pass
 
         try:
@@ -795,7 +803,7 @@
             self.log_dir = config.get('GLOBAL', 'log_dir')
             self.logfile = config.get('GLOBAL', 'logfile')
             self.loglevel = config.get('GLOBAL', 'loglevel')
         except:
 
             # Fall back to file in current dir
 
@@ -823,13 +831,92 @@
         # print "logger initialized (level " + logger_obj.loglevel() + ")"
         # logger.debug("logger initialized")
 
+        # Allow section confs included from file
+        if config.has_option('GLOBAL', 'include_sections'):
+            self.include_sections = config.get('GLOBAL', 'include_sections')
+        else:
+            # Fall back to config section dir along with actual conf
+            if config.has_option('GLOBAL', 'mig_server_home'):
+                _config_file_dir = config.get('GLOBAL', 'mig_server_home')
+            else:
+                _config_file_dir = os.path.dirname(_config_file)
+            self.include_sections = os.path.join(_config_file_dir,
+                                                 mig_conf_section_dirname)
+
+        # NOTE: for simplicity we do NOT allow core overrides in GLOBAL and
+        #       SITE sections. Especially the options affecting base paths
+        #       and service daemons as that might interfere with external
+        #       dependencies e.g. in the apache web server.
+        no_override_section_options = {
+            'GLOBAL': ['include_sections', 'mig_path', 'mig_server_home',
+                       'state_path', 'certs_path',
+                       'logfile', 'loglevel',
+                       'server_fqdn',
+                       'migserver_public_url',
+                       'migserver_public_alias_url',
+                       'migserver_http_url',
+                       'migserver_https_url',
+                       'migserver_https_mig_oid_url',
+                       'migserver_https_ext_oid_url',
+                       'migserver_https_mig_oidc_url',
+                       'migserver_https_ext_oidc_url',
+                       'migserver_https_mig_cert_url',
+                       'migserver_https_ext_cert_url',
+                       'migserver_https_sid_url',
+                       'user_openid_address', 'user_openid_port',
+                       'user_openid_key', 'user_openid_log',
+                       'user_davs_address', 'user_davs_port',
+                       'user_davs_key', 'user_davs_log',
+                       'user_sftp_address', 'user_sftp_port',
+                       'user_sftp_key', 'user_sftp_log',
+                       'user_sftp_subsys_address', 'user_sftp_subsys_port',
+                       'user_sftp_subsys_log',
+                       'user_ftps_address', 'user_ftps_ctrl_port',
+                       'user_ftps_pasv_ports',
+                       'user_ftps_key', 'user_ftps_log'],
+            'SITE': ['enable_openid', 'enable_davs', 'enable_ftps',
+                     'enable_sftp', 'enable_sftp_subsys', 'enable_crontab',
+                     'enable_events', 'enable_notify', 'enable_imnotify',
+                     'enable_transfers']
+        }
+        self.include_sections = os.path.normpath(self.include_sections)
+        if os.path.isdir(self.include_sections):
+            msg = "read extra config sections from %s" % self.include_sections
+            if verbose:
+                print(msg)
+            logger.debug(msg)
+            for section_filename in os.listdir(self.include_sections):
+                # skip dotfiles and non-confs
+                if section_filename.startswith('.'):
+                    continue
+                if not section_filename.endswith('.conf'):
+                    msg = "%r is not on required sectionname.conf form" % \
+                        section_filename
+                    if verbose:
+                        print(msg)
+                    logger.warning(msg)
+                    continue
+                section_path = os.path.join(self.include_sections,
+                                            section_filename)
+                section = section_filename.replace('.conf', '').upper()
+                reject_overrides = []
+                if section in no_override_section_options:
+                    reject_overrides = no_override_section_options[section]
+                    msg = "filter %r section override in %r for %s" % \
+                        (section, section_filename, reject_overrides)
+                    if verbose:
+                        print(msg)
+                    logger.debug(msg)
+                include_section_contents(logger, config, section, section_path,
+                                         verbose, reject_overrides)
+
         # Mandatory options first
 
         try:
             self.mig_server_id = config.get('GLOBAL', 'mig_server_id')
             self.mrsl_files_dir = config.get('GLOBAL', 'mrsl_files_dir')
             self.re_files_dir = config.get('GLOBAL', 're_files_dir')
             self.re_pending_dir = config.get('GLOBAL', 're_pending_dir')
             self.re_home = config.get('GLOBAL', 're_home')
             self.grid_stdin = config.get('GLOBAL', 'grid_stdin')
             self.im_notify_stdin = config.get('GLOBAL',
@@ -866,7 +953,7 @@
             self.jobtypes = config.get('GLOBAL', 'jobtypes').split()
             self.lrmstypes = config.get('GLOBAL', 'lrmstypes').split()
             self.sessid_to_mrsl_link_home = config.get('GLOBAL',
                                                        'sessid_to_mrsl_link_home')
             self.mig_system_files = config.get('GLOBAL',
                                                'mig_system_files')
             self.mig_system_storage = config.get('GLOBAL',
@@ -935,45 +1022,6 @@
         else:
             self.user_db_home = os.path.join(self.state_path, 'user_db_home')
 
-        # Allow section confs included from file
-        if config.has_option('GLOBAL', 'include_sections'):
-            self.include_sections = config.get('GLOBAL', 'include_sections')
-        else:
-            self.include_sections = os.path.join(self.mig_server_home,
-                                                 mig_conf_section_dirname)
-
-        # NOTE: for simplicity we do NOT allow overrides in GLOBAL section
-        no_override_sections = ['GLOBAL']
-        self.include_sections = os.path.normpath(self.include_sections)
-        if os.path.isdir(self.include_sections):
-            msg = "read extra config sections from %s" % self.include_sections
-            if verbose:
-                print(msg)
-            logger.info(msg)
-            for section_filename in os.listdir(self.include_sections):
-                # skip dotfiles and non-confs
-                if section_filename.startswith('.'):
-                    continue
-                if not section_filename.endswith('.conf'):
-                    msg = "%r is not on required sectionname.conf form" % \
-                        section_filename
-                    if verbose:
-                        print(msg)
-                    logger.warning(msg)
-                    continue
-                section_path = os.path.join(self.include_sections,
-                                            section_filename)
-                section = section_filename.replace('.conf', '').upper()
-                if section in no_override_sections:
-                    msg = "skip unsupported %r section override in %r" % \
-                        (section, section_filename)
-                    if verbose:
-                        print(msg)
-                    logger.warning(msg)
-                    continue
-                include_section_contents(logger, config, section, section_path,
-                                         verbose)
-
         if config.has_option('GLOBAL', 'admin_list'):
             # Parse semi-colon separated list of admins with optional spaces
             admins = config.get('GLOBAL', 'admin_list')
@@ -1417,7 +1465,7 @@
             protos = [i for i in plain_val.split() if i]
             # Append missing supported protocols for AUTO and filter invalid
             if keyword_auto in protos:
-                protos = [i for i in protos if i != keyword_auto] +\
+                protos = [i for i in protos if i != keyword_auto] + \
                          [i for i in allowed_protos if i not in protos]
             valid_protos = [i for i in protos if i in allowed_protos]
             if protos != valid_protos:
@@ -1696,7 +1744,7 @@
             protos = [i for i in plain_val.split() if i]
             # Append missing supported protocols for AUTO and filter invalid
             if keyword_auto in protos:
-                protos = [i for i in protos if i != keyword_auto] +\
+                protos = [i for i in protos if i != keyword_auto] + \
                          [i for i in allowed_protos if i not in protos]
             valid_protos = [i for i in protos if i in allowed_protos]
             if protos != valid_protos:

tests/data/MiGserver.d/dummy

@@ -0,0 +1,3 @@
+# Dummy conf section with missing conf extension
+[SETTINGS]
+language = Pig Latin

tests/data/MiGserver.d/global.conf

@@ -0,0 +1,21 @@
+# Used to test that core GLOBAL section overrides are rejected by policy while
+# peripheral overrides are accepted.
+[GLOBAL]
+# These overrides are allowed
+admin_email = [email protected]
+vgrid_resources = resources.custom
+vgrid_triggers = triggers.custom
+vgrid_sharelinks = sharelinks.custom
+vgrid_monitor = monitor.custom
+# but these core ones aren't allowed
+include_sections = /tmp/MiGserver.d
+mig_path = /tmp/mig/mig
+logfile = /tmp/mig.log
+loglevel = warning
+server_fqdn = somewhere.org
+migserver_public_url = https://somewhere.org
+migserver_https_sid_url = https://somewhere.org
+user_openid_address = somewhere.org
+user_openid_port = 4242
+user_openid_key = /tmp/openid.key
+user_openid_log = /tmp/openid.log

tests/data/MiGserver.d/multi.conf

@@ -0,0 +1,7 @@
+# Sample multi conf to test that superflous other sections in conf are ignored
+[MULTI]
+blabla = yes
+
+# Other sections like SETTINGS must be in SECTIONAME.conf and are ignored here
+[SETTINGS]
+language = Spanglish

tests/data/MiGserver.d/section-mismatch.conf

@@ -0,0 +1,3 @@
+# Dummy section conf with wrong name
+[SETTINGS]
+language = Pig Latin

tests/data/MiGserver.d/site.conf

@@ -0,0 +1,27 @@
+# Used to test that core SITE section overrides are rejected by policy while
+# peripheral overrides are accepted.
+[SITE]
+# These overrides are allowed
+short_title = ACME Site
+support_text = Custom support text
+privacy_text = Custom privacy text
+peers_notice = Custom peers notice
+peers_contact_hint = Custom peers contact hint
+new_user_default_ui = V3
+password_legacy_policy = MEDIUM
+freeze_admins = BOFH
+freeze_to_tape = Custom freeze to tape
+freeze_doi_text = Custom freeze doi text
+freeze_doi_url = https://somewhere.org/mint-doi
+freeze_doi_url_field = archiveurl
+# but these core ones aren't allowed
+enable_openid = True
+enable_davs = True
+enable_ftps = True
+enable_sftp = True
+enable_sftp_susbsys = True
+enable_crontab = True
+enable_events = True
+enable_notify = True
+enable_imnotify = True
+enable_transfers = True