From c54a36a99d491d60da2c21527d450352e47a0107 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Tue, 19 Aug 2014 16:02:31 +0100 Subject: [PATCH] Add support for binary assets back in. Filters that expect binary input will need to declare a `binary_input` attribute. Otherwise they will be passed data in Unicode strings. Binary outputs will be converted to Unicode strings before each filter that doesn't declare itself as supporting binary input. That includes input files read from disk. Output files written to disk will be encoded in utf-8 if the last filter output Unicode, otherwise the binary data output by the last filter. --- src/webassets/filter/__init__.py | 2 +- src/webassets/filter/sass.py | 4 +-- src/webassets/merge.py | 52 ++++++++++++++++++++++------ src/webassets/version.py | 8 +++-- tests/test_hunks.py | 58 ++++++++++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 16 deletions(-) create mode 100644 tests/test_hunks.py diff --git a/src/webassets/filter/__init__.py b/src/webassets/filter/__init__.py index 05767c7b..877ee894 100644 --- a/src/webassets/filter/__init__.py +++ b/src/webassets/filter/__init__.py @@ -492,7 +492,7 @@ def created(self): try: data = (data.read() if hasattr(data, 'read') else data) - if data is not None: + if isinstance(data, unicode): data = data.encode('utf-8') if input_file.created: diff --git a/src/webassets/filter/sass.py b/src/webassets/filter/sass.py index 03911a77..bd4f4b1a 100644 --- a/src/webassets/filter/sass.py +++ b/src/webassets/filter/sass.py @@ -125,7 +125,7 @@ def _apply_sass(self, _in, out, cd=None): # shell: necessary on windows to execute # ruby files, but doesn't work on linux. shell=(os.name == 'nt')) - stdout, stderr = proc.communicate(_in.read().encode('utf-8')) + stdout, stderr = proc.communicate(_in.read()) if proc.returncode != 0: raise FilterError(('sass: subprocess had error: stderr=%s, '+ @@ -134,7 +134,7 @@ def _apply_sass(self, _in, out, cd=None): elif stderr: print("sass filter has warnings:", stderr) - out.write(stdout.decode('utf-8')) + out.write(stdout) finally: if cd: os.chdir(old_dir) diff --git a/src/webassets/merge.py b/src/webassets/merge.py index 646663ad..a290b9bc 100644 --- a/src/webassets/merge.py +++ b/src/webassets/merge.py @@ -10,7 +10,7 @@ from urllib2 import Request as URLRequest, urlopen from urllib2 import HTTPError import logging -from io import open +from io import open, BytesIO from webassets.six.moves import filter from .utils import cmp_debug_levels, StringIO, hash_func @@ -56,8 +56,11 @@ def data(self): raise NotImplementedError() def save(self, filename): - with open(filename, 'w') as f: - f.write(self.data()) + data = self.data() + if isinstance(data, unicode): + data = data.encode('utf-8') + with open(filename, 'wb') as f: + f.write(data) class FileHunk(BaseHunk): @@ -74,7 +77,7 @@ def mtime(self): pass def data(self): - f = open(self.filename, 'r', encoding='utf-8') + f = open(self.filename, 'rb') try: return f.read() finally: @@ -159,11 +162,11 @@ def data(self): return self._data def save(self, filename): - f = open(filename, 'w', encoding='utf-8') - try: - f.write(self.data()) - finally: - f.close() + data = self.data() + if isinstance(data, unicode): + data = data.encode('utf-8') + with open(filename, 'wb') as f: + f.write(data) def merge(hunks, separator=None): @@ -222,6 +225,32 @@ def _wrap_cache(self, key, func): self.cache.set(key, content) return MemoryHunk(content) + def create_input_buffer_for(self, data): + have_binary = isinstance(data, BytesIO) + + if isinstance(data, unicode): + return StringIO(data) + else: + return BytesIO(data) + + def convert_input_buffer_for(self, filter, data): + accepts_binary = getattr(filter, 'binary_input', False) + have_binary = isinstance(data, BytesIO) + + if accepts_binary and not have_binary: + return BytesIO(data.getvalue().encode('utf-8')) + elif not accepts_binary and have_binary: + return StringIO(data.getvalue().decode('utf-8')) + else: + return data + + def create_output_buffer_for(self, filter): + if getattr(filter, 'binary_output', False): + return BytesIO(b'') + else: + # For 2.x, StringIO().getvalue() returns str + return StringIO(u'') + def apply(self, hunk, filters, type, kwargs=None): """Apply the given list of filters to the hunk, returning a new ``MemoryHunk`` object. @@ -244,11 +273,12 @@ def apply(self, hunk, filters, type, kwargs=None): kwargs_final.update(kwargs or {}) def func(): - data = StringIO(hunk.data()) + data = self.create_input_buffer_for(hunk.data()) for filter in filters: + data = self.convert_input_buffer_for(filter, data) + out = self.create_output_buffer_for(filter) log.debug('Running method "%s" of %s with kwargs=%s', type, filter, kwargs_final) - out = StringIO(u'') # For 2.x, StringIO().getvalue() returns str getattr(filter, type)(data, out, **kwargs_final) data = out data.seek(0) diff --git a/src/webassets/version.py b/src/webassets/version.py index 5bb71607..90f58d96 100644 --- a/src/webassets/version.py +++ b/src/webassets/version.py @@ -6,8 +6,8 @@ import os import pickle -from webassets import six +from webassets import six from webassets.merge import FileHunk from webassets.utils import md5_constructor, RegistryMetaclass, is_url @@ -162,8 +162,12 @@ def determine_version(self, bundle, ctx, hunk=None): raise VersionIndeterminableError( 'output target has a placeholder') + data = hunk.data() + if isinstance(data, unicode): + data = data.encode('utf-8') + hasher = self.hasher() - hasher.update(hunk.data().encode('utf-8')) + hasher.update(data) return hasher.hexdigest()[:self.length] diff --git a/tests/test_hunks.py b/tests/test_hunks.py new file mode 100644 index 00000000..076ca3c7 --- /dev/null +++ b/tests/test_hunks.py @@ -0,0 +1,58 @@ +"""Test the versioners and manifest implementations. +""" +import hashlib +import codecs +import os + +from nose.tools import assert_raises + +from webassets.env import Environment +from webassets.merge import MemoryHunk, FileHunk +from webassets.test import TempEnvironmentHelper +from webassets.version import ( + FileManifest, JsonManifest, CacheManifest, TimestampVersion, + VersionIndeterminableError, HashVersion, get_versioner, get_manifest) + + +class TestBinaryHunks(TempEnvironmentHelper): + BINARY_DATA = b'\x80\x90\xa0\xff' + + def setup(self): + super(TestBinaryHunks, self).setup() + self.bundle = self.mkbundle('in', depends=('dep'), output='out') + + self.name = 'in' + dirs = os.path.dirname(self.path(self.name)) + if not os.path.exists(dirs): + os.makedirs(dirs) + f = codecs.open(self.path(self.name), 'wb') + f.write(self.BINARY_DATA) + f.close() + + def test_read_binary_file_hunk(self): + h = FileHunk(self.path(self.name)) + d = h.data() + assert isinstance(d, str) + assert d == self.BINARY_DATA + + def test_write_binary_file_hunk(self): + outfile = self.path('out') + h = FileHunk(self.path(self.name)) + h.save(outfile) + + with open(outfile, 'rb') as f: + d = f.read() + + assert isinstance(d, str) + assert d == self.BINARY_DATA + + def test_write_memory_hunk(self): + outfile = self.path('out') + h = MemoryHunk(self.BINARY_DATA) + h.save(outfile) + + with open(outfile, 'rb') as f: + d = f.read() + + assert isinstance(d, str) + assert d == self.BINARY_DATA