Fixes offline-compression and updates static storage mechanism

Author: gdixon

app/app/settings.py

@@ -212,15 +212,19 @@
 if ENV not in ['local', 'test', 'staging', 'preview']:
     # compress offline (use './manage.py compress' to build manifest.json)
     COMPRESS_OFFLINE = True
-    # content based hashing
+    # Allow for the full path (url) to be included in the manifest
+    COMPRESS_INCLUDE_URLS = True
+    # Allow the placeholder insertion to be skipped
+    COMPRESS_SKIP_PLACEHOLDER = True
+    # use content based hashing so that we always match between servers
     COMPRESS_CSS_HASHING_METHOD = 'content'
-    # drop line comments
-    LIBSASS_SOURCE_COMMENTS = False
     # minification of sass output
     COMPRESS_CSS_FILTERS = [
         'compressor.filters.css_default.CssAbsoluteFilter',
         'compressor.filters.cssmin.rCSSMinFilter'
     ]
+    # drop line comments
+    LIBSASS_SOURCE_COMMENTS = False
 
 SITE_ID = env.int('SITE_ID', default=1)
 WSGI_APPLICATION = env('WSGI_APPLICATION', default='app.wsgi.application')

app/app/static_storage.py

@@ -1,65 +1,44 @@
 # -*- coding: utf-8 -*-
 """Define the custom static storage to surpress bad URL references."""
 import os
+from datetime import datetime
 from os.path import basename
 from secrets import token_hex
 
 from django.conf import settings
+from django.contrib.staticfiles.storage import HashedFilesMixin
 from django.core.files.storage import get_storage_class
-from django.contrib.staticfiles.storage import ManifestFilesMixin, HashedFilesMixin
 
-from storages.backends.s3boto3 import S3Boto3Storage, SpooledTemporaryFile
+from storages.backends.s3boto3 import S3ManifestStaticStorage, S3StaticStorage
 
 
-class SilentFileStorage(ManifestFilesMixin, S3Boto3Storage):
+class SilentFileStorage(S3ManifestStaticStorage, S3StaticStorage):
     """Define the static storage using S3 via boto3 with hashing.
 
     If Django cannot find a referenced url in an asset, it will silently pass.
 
     """
 
     location = settings.STATICFILES_LOCATION
+    bucket_name = settings.AWS_STORAGE_BUCKET_NAME
+    custom_domain = settings.AWS_S3_CUSTOM_DOMAIN
 
     def __init__(self, *args, **kwargs):
-        kwargs['bucket'] = settings.AWS_STORAGE_BUCKET_NAME
-        kwargs['custom_domain'] = settings.AWS_S3_CUSTOM_DOMAIN
-        # Init S3Boto3Storage and ManifestFilesMixin to send assets to S3
+        # Init S3StaticStorage and S3ManifestStaticStorage to send assets to S3
         super(SilentFileStorage, self).__init__(*args, **kwargs)
         # Init CompressorFileStorage to save local copies for compressor
         self.local_storage = get_storage_class("compressor.storage.CompressorFileStorage")()
         # Init HashedFilesMixin to get filenames with hashes present
         self.local_hashes = HashedFilesMixin()
 
-    def _save_content(self, obj, content, parameters):
-        """Create a clone of the content file to avoid premature closure.
-
-        When this is passed to boto3 it wrongly closes the file upon upload
-        where as the storage backend expects it to still be open.
-
-        """
-        # Seek our content back to the start
-        content.seek(0, os.SEEK_SET)
-
-        # Create a temporary file that will write to disk after a specified size
-        content_autoclose = SpooledTemporaryFile()
-
-        # Write our original content into our copy that will be closed by boto3
-        content_autoclose.write(content.read())
-
-        # Upload the object which will auto close the content_autoclose instance
-        super(SilentFileStorage, self)._save_content(obj, content_autoclose, parameters)
-
-        # Cleanup if this is fixed upstream our duplicate should always close
-        if not content_autoclose.closed:
-            content_autoclose.close()
-
     def save(self, name, content):
-        # record the clean file content (pre gzip)
+        """Save both a local and a remote copy of the given file"""
+        # Record the clean file content (pre gzip)
         file_content = content.file
         # Save remote copy to S3
-        super(SilentFileStorage, self).save(name, content)
-        # Only save .scss and .js files locally
-        if ".scss" in name or ".js" in name:
+        super(SilentFileStorage, self).save(name, content)    
+        # Only save files that are part of the compress blocks locally
+        if ".scss" in name or ".js" in name or ".css" in name:
             # restore the clean file_content
             content.file = file_content
             # Save a local copy for compressor
@@ -68,6 +47,19 @@ def save(self, name, content):
             self.local_storage._save(self.local_hashes.hashed_name(name, content), content)
         return name
 
+    def exists(self, name):
+        """Check if the named file exists in S3 storage"""
+        # Check file exists on S3
+        exists = super(SilentFileStorage, self).exists(name)
+        # This is a hack to get a status report during S3ManifestStaticStorage._postProcess
+        print("INFO " + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + 
+            " - Checking for matching file hash on S3 - " + name + ": " + (
+                "Skipping based on matching file hashes" if exists else "Hashes did not match"
+            )
+        )
+        return exists
+
+
     def url(self, name, force=True):
         """Handle catching bad URLs and return the name if route is unavailable."""
         try:
@@ -84,17 +76,13 @@ def _url(self, hashed_name_func, name, force=True, hashed_files=None):
             return name
 
 
-class MediaFileStorage(S3Boto3Storage):
+class MediaFileStorage(S3StaticStorage):
     """Define the media storage backend for user uploaded/stored files."""
 
     location = settings.MEDIAFILES_LOCATION
+    bucket_name = settings.MEDIA_BUCKET
+    custom_domain = settings.MEDIA_CUSTOM_DOMAIN
 
     def __init__(self, *args, **kwargs):
-        kwargs['bucket'] = settings.MEDIA_BUCKET
-        kwargs['custom_domain'] = settings.MEDIA_CUSTOM_DOMAIN
         # Save media to S3 only (we dont need an additional local copy)
         super(MediaFileStorage, self).__init__(*args, **kwargs)
-
-
-def get_salted_path(instance, filename):
-    return f'assets/{token_hex(16)[:15]}/{basename(filename)}'

app/assets/v2/scss/test1.scss

@@ -1,3 +1,3 @@
-$bg-color: green;
+$bg-color: rgb(49, 230, 49);
 
 @import './test2.scss';

app/assets/v2/scss/test2.scss

@@ -1,6 +1,7 @@
 body {
   background: $bg-color;
   & > div:after {
+    font-weight: bold;
     content: "working"
   }
-}
+}

app/retail/templates/home/sass_experiment.html

@@ -22,7 +22,7 @@
     <div></div>
     {% compress js file testing %}
       <script>
-        alert('testing');
+        alert('This has been changed: working');
       </script>
     {% endcompress %}
   </body>

requirements/base.txt

@@ -58,7 +58,7 @@ watchdog==0.9.0
 Werkzeug[watchdog]==0.15.5
 imageio
 boto3==1.7.81
-django-storages==1.6.6
+django-storages==1.11.1
 easy-thumbnails==2.5
 eth-account==0.2.2
 django-classy-tags==0.8.0