Merge remote-tracking branch 'origin/master' into prod

Author: jkeifer

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+default_language_version:
+    python: python3.12
+repos:
+  - repo: local
+    hooks:
+      - id: ruff_check
+        name: ruff check
+        entry: ruff check --force-exclude
+        language: python
+        'types_or': [python, pyi]
+        args: [--fix, --exit-non-zero-on-fix]
+        require_serial: true
+      - id: ruff_format
+        name: ruff format
+        entry: ruff format --force-exclude
+        language: python
+        'types_or': [python, pyi]
+        args: []
+        require_serial: true
+      - id: check-added-large-files
+        name: Check for added large files
+        entry: check-added-large-files
+        language: system
+      - id: check-toml
+        name: Check Toml
+        entry: check-toml
+        language: system
+        types: [toml]
+      - id: check-yaml
+        name: Check Yaml
+        entry: check-yaml
+        language: system
+        types: [yaml]
+      - id: end-of-file-fixer
+        name: Fix End of Files
+        entry: end-of-file-fixer
+        language: system
+        types: [text]
+        stages: [commit, push, manual]
+      - id: trailing-whitespace
+        name: Trim Trailing Whitespace
+        entry: trailing-whitespace-fixer
+        language: system
+        types: [text]
+        stages: [commit, push, manual]
+      - id: mypy
+        name: mypy
+        entry: mypy
+        language: python
+        'types_or': [python, pyi]
+        args: []
+        require_serial: true
+        exclude: 'snodas/settings/*'

Dockerfile

@@ -1,4 +1,4 @@
-FROM postgis/postgis:15-3.3
+FROM postgis/postgis:16-3.4
 
 RUN set -x && \
     apt-get update && \

MANIFEST.in

@@ -1,3 +1,2 @@
-recursive-include snodas *
-include *.md
-include *.txt
+graft snodas
+global-exclude *.py[cod]

VERSION.txt

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+
+find_this () {
+    THIS="${1:?'must provide script path, like "${BASH_SOURCE[0]}" or "$0"'}"
+    trap "echo >&2 'FATAL: could not resolve parent directory of ${THIS}'" EXIT
+    [ "${THIS:0:1}"  == "/" ] || THIS="$(pwd -P)/${THIS}"
+    THIS_DIR="$(dirname -- "${THIS}")"
+    THIS_DIR="$(cd -P -- "${THIS_DIR}" && pwd)"
+    THIS="${THIS_DIR}/$(basename -- "${THIS}")"
+    trap "" EXIT
+}
+
+
+maybe_print_help() {
+    for arg in "$@"; do
+        case "$arg" in
+            -h|--help)
+                local help
+                help="$(pip-compile --help | tail -n +2)"
+                echo >&2 "Usage: $CUSTOM_COMPILE_COMMAND [OPTIONS] [SRC_FILES]..."
+                echo >&2 ""
+                echo >&2 "This is a script wrapping pip-compile."
+                echo >&2 "The following is the pip-compile usage."
+                echo >&2 "$help"
+                exit
+                ;;
+        esac
+    done
+}
+
+
+main () {
+    export CUSTOM_COMPILE_COMMAND="$0"
+    find_this "${BASH_SOURCE[0]}"
+
+    maybe_print_help "$@"
+
+    cd "${THIS_DIR}/.."
+
+    local compile="pip-compile --strip-extras --generate-hashes --allow-unsafe"
+
+    $compile pyproject.toml "$@"
+
+    local infile
+    for infile in *.in; do
+        [ "$infile" == 'MANIFEST.in' ] && continue
+        $compile -o "${infile%.*}.txt" "${infile}" "$@"
+    done
+}
+
+
+main "$@"

bin/compile-requirements.bash

@@ -4,7 +4,8 @@ services:
     build: .
     restart: always
     environment:
-      - POSTGRES_USER=${PGUSER:-postgres}
-      - POSTGRES_PASSWORD=${PGPASSWORD:-password}
+      POSTGRES_USER: "${PGUSER:-postgres}"
+      POSTGRES_PASSWORD: "${PGPASSWORD:-password}"
+      POSTGIS_GDAL_ENABLED_DRIVERS: 'GTiff PNG SNODAS'
     ports:
       - '${PGPORT:-5432}:5432'

bin/settings-yml-to-json-files.py

@@ -1,68 +1,52 @@
 #!/usr/bin/env python
-import os
-import sys
 import logging
-import datetime
 import logging.handlers
+import os
+import sys
 
+from pathlib import Path
 
 logger = logging.getLogger(__name__)
 LOG_SIZE = 50 * (10**6)  # about 50 MB
 LOG_COUNT = 5
-LOG_FILE = os.path.join(os.path.dirname(__file__), 'manage.log')
 
-CONF_FILE = os.path.join(os.path.dirname(__file__), 'project.conf')
+THIS = Path(__file__).resolve()
+THIS_DIR = THIS.parent
+LOG_FILE = THIS_DIR / 'manage.log'
+CONF_FILE = THIS_DIR / 'project.conf'
 
 
-def activate_help():
-    import yaml
-    with open(CONF_FILE) as f:
-        instance_name = yaml.load(f)['INSTANCE_NAME']
-    return (
-        'ERROR: snodas.settings could not be imported.\n'
-        'It looks like you need to activate the conda environment '
-        'for this instance, which you can do by running '
-        '`activate {}`'.format(instance_name)
-    )
+ACTIVATE_HELP: str = (
+    'ERROR: snodas.settings could not be imported.\n'
+    'It looks like you need to activate the proper conda/virtual environment.'
+)
 
 
 def install(help=False):
-    this_dir = os.path.dirname(os.path.realpath(__file__))
     # first we append the path with the management package
     # so we can import utils in the install module
-    sys.path.append(
-        os.path.join(
-            this_dir,
-            'snodas',
-            'management',
-        )
-    )
+    sys.path.append(str(THIS_DIR / 'snodas' / 'management'))
+
     # then we add the commands package to the path
     # so we have access to the install module
-    sys.path.append(
-        os.path.join(
-            this_dir,
-            'snodas',
-            'management',
-            'commands',
-        )
-    )
+    sys.path.append(str(THIS_DIR / 'snodas' / 'management' / 'commands'))
+
     # and lastly we add the directory of this file
     # to the path so we can import from setup.py
-    sys.path.append(
-        os.path.join(
-            this_dir,
-        )
-    )
-    from install import Install
+    sys.path.append(str(THIS_DIR))
+
+    from install import Install  # type: ignore
+
     if help:
         Install.print_help(sys.argv[0], sys.argv[2])
     else:
         Install.run_from_argv(sys.argv)
 
 
 def default_django_command():
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "snodas.settings")
+    from importlib.util import find_spec
+
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'snodas.settings')
 
     # check to make sure we can import the settings
     # otherwise, we suspect the env has not been activated
@@ -71,14 +55,12 @@ def default_django_command():
     # activated and this would not prevent execution. I should
     # do some sort of more advanced check of the settings to verify that
     # they match the current project.
-    try:
-        import snodas.settings
-    except ImportError:
+    if find_spec('snodas.settings') is None:
         logger.debug(
-            'snodas.settings couldn\'t be imported; '
-            'looks like the correct env is not activated'
+            "snodas.settings couldn't be imported; "
+            'looks like the correct env is not activated',
         )
-        print(activate_help())
+        print(ACTIVATE_HELP)  # noqa: T201
         return 1
 
     # hacky workaround to allow snodas command to use the
@@ -89,20 +71,23 @@ def default_django_command():
     # which fails either because snodas cannot be found, or
     # sondas is found, and it is the snodas package in the
     # current directory.)
-    if len(sys.argv) > 1 and \
-            sys.argv[1] == 'runserver' and \
-            not ('--help' in sys.argv or '-h' in sys.argv):
+    if (
+        len(sys.argv) > 1
+        and sys.argv[1] == 'runserver'
+        and not ('--help' in sys.argv or '-h' in sys.argv)
+    ):
         logger.debug(
-            'rewritting sys.argv[0] from {} to {}'.format(
-                sys.argv[0],
-                __file__,
-            )
+            'rewritting sys.argv[0] from %s to %s',
+            sys.argv[0],
+            __file__,
         )
         sys.argv[0] = __file__
 
-    from django.core.management import execute_from_command_line
+    from django.core.management import execute_from_command_line  # type: ignore
+
     logger.debug(
-        'executing from the command line with sys.argv: {}'.format(sys.argv)
+        'executing from the command line with sys.argv: %s',
+        sys.argv,
     )
     return execute_from_command_line(sys.argv)
 
@@ -114,41 +99,40 @@ def main():
         backupCount=(LOG_COUNT - 1),
     )
     formatter = logging.Formatter(
-        '%(asctime)s %(name)s [%(levelname)-5s] %(message)s'
+        '%(asctime)s %(name)s [%(levelname)-5s] %(message)s',
     )
     manage_log.setFormatter(formatter)
     logger.setLevel(logging.DEBUG)
     logger.addHandler(manage_log)
-    #logging.basicConfig(
-    #    filename=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'manage_%s_%d.log' %(datetime.datetime.now().strftime('%y%m%d_%H%M%S'), os.getpid())),
-    #    filemode='w',
-    #    format='%(asctime)s [%(levelname)-5s] %(message)s',
-    #    level=logging.DEBUG,
-    #)
 
     if len(sys.argv) > 1 and sys.argv[1] == 'install':
         logger.debug('install command run')
         return install()
-    if len(sys.argv) > 1 and \
-            os.path.basename(sys.argv[0]) == 'manage.py' and \
-            sys.argv[1] == 'help' and \
-            sys.argv[2] == 'install':
+
+    if (
+        len(sys.argv) > 1
+        and Path(sys.argv[0]).name == 'manage.py'
+        and sys.argv[1] == 'help'
+        and sys.argv[2] == 'install'
+    ):
         logger.debug('install help run')
         return install(help=True)
-    elif not os.path.isfile(CONF_FILE):
-        logger.error('config file could not be loaded: {}'.format(CONF_FILE))
-        print('ERROR: Could not find configuration file {}.'.format(CONF_FILE))
-        print('Has this instance been installed? '
-              'Try running `python manage.py install`.')
-    else:
-        try:
-            return default_django_command()
-        except Exception as e:
-            logger.exception(e)
-            raise
-            # before I was returning -1, but I don't know why anymore
-            return -1
+
+    if not CONF_FILE.is_file():
+        logger.error('config file could not be loaded: %s', CONF_FILE)
+        print(f'ERROR: Could not find configuration file {CONF_FILE}.')  # noqa: T201
+        print(  # noqa: T201
+            'Has this instance been installed? '
+            'Try running `python manage.py install`.',
+        )
+        return None
+
+    try:
+        return default_django_command()
+    except Exception:
+        logger.exception('Failure running CLI')
+        raise
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     sys.exit(main())