Add handling for pre and code blocks

This commit adds handling for pre and code blocks.

- Pre and code blocks use Courier font.
- Pre blocks retain white space and new line characters.
- Code blocks do not retain white space and new lines.

Changes
=======
- Add unittests for code and pre blocks, and inline code snippets.
- Add manual test for code blocks. Run this manually to validate
  visual representation of code and pre-formatted blocks.
- Add doctests for remove_whitespace function to help show exactly
  what the function does.
- Run h2d doctests in python-app workflow
- Run test_code module in python-app workflow
- Update remove_whitespace to allow removing leading and trailing
  new lines wrapped with white space. This prevents spaces appearing
  before or after code blocks.

Author: Jon Grace-Cox

.github/workflows/python-app.yml

@@ -35,6 +35,8 @@ jobs:
       run: |
         python -m tests.test
         python -m tests.test_tables
+        python -m tests.test_code
+        python -m doctest -v htmldocx/h2d.py
     - name: Upload test artifacts
       uses: actions/[email protected]
       with:

htmldocx/h2d.py

@@ -58,7 +58,67 @@ def fetch_image(url):
 def remove_last_occurence(ls, x):
     ls.pop(len(ls) - ls[::-1].index(x) - 1)
 
-def remove_whitespace(string):
+def remove_whitespace(string, leading=False, trailing=False):
+    """Remove white space from a string.
+
+    Args:
+        string(str): The string to remove white space from.
+        leading(bool, optional): Remove leading new lines when True.
+        trailing(bool, optional): Remove trailing new lines when False.
+
+    Returns:
+        str: The input string with new line characters removed and white space squashed.
+
+    Examples:
+
+        Single or multiple new line characters are replaced with space.
+
+            >>> remove_whitespace("abc\\ndef")
+            'abc def'
+            >>> remove_whitespace("abc\\n\\n\\ndef")
+            'abc def'
+
+        New line characters surrounded by white space are replaced with a single space.
+
+            >>> remove_whitespace("abc \\n \\n \\n def")
+            'abc def'
+            >>> remove_whitespace("abc  \\n  \\n  \\n  def")
+            'abc def'
+
+        Leading and trailing new lines are replaced with a single space.
+
+            >>> remove_whitespace("\\nabc")
+            ' abc'
+            >>> remove_whitespace("  \\n  abc")
+            ' abc'
+            >>> remove_whitespace("abc\\n")
+            'abc '
+            >>> remove_whitespace("abc  \\n  ")
+            'abc '
+
+        Use ``leading=True`` to remove leading new line characters, including any surrounding
+        white space:
+
+            >>> remove_whitespace("\\nabc", leading=True)
+            'abc'
+            >>> remove_whitespace("  \\n  abc", leading=True)
+            'abc'
+
+        Use ``trailing=True`` to remove trailing new line characters, including any surrounding
+        white space:
+
+            >>> remove_whitespace("abc  \\n  ", trailing=True)
+            'abc'
+    """
+    # Remove any leading new line characters along with any surrounding white space
+    if leading:
+        string = re.sub(r'^\s*\n+\s*', '', string)
+
+    # Remove any trailing new line characters along with any surrounding white space
+    if trailing:
+        string = re.sub(r'\s*\n+\s*$', '', string)
+
+    # Replace new line characters and absorb any surrounding space.
     string = re.sub(r'\s*\n\s*', ' ', string)
     return re.sub(r'>\s{2+}<', '><', string)
 
@@ -68,7 +128,7 @@ def delete_paragraph(paragraph):
     p.getparent().remove(p)
     p._p = p._element = None
 
-fonts = {
+font_styles = {
     'b': 'bold',
     'strong': 'bold',
     'em': 'italic',
@@ -80,6 +140,11 @@ def delete_paragraph(paragraph):
     'th': 'bold',
 }
 
+font_names = {
+    'code': 'Courier',
+    'pre': 'Courier',
+}
+
 class HtmlToDocx(HTMLParser):
 
     def __init__(self):
@@ -309,7 +374,7 @@ def handle_starttag(self, tag, attrs):
             return
 
         self.tags[tag] = current_attrs
-        if tag == 'p':
+        if tag in ['p', 'pre']:
             self.paragraph = self.doc.add_paragraph()
 
         elif tag == 'li':
@@ -331,7 +396,7 @@ def handle_starttag(self, tag, attrs):
             return
 
         # set new run reference point in case of leading line breaks
-        if tag == 'p' or tag == 'li':
+        if tag in ['p', 'li', 'pre']:
             self.run = self.paragraph.add_run()
 
         # add style
@@ -375,6 +440,18 @@ def handle_data(self, data):
         if self.skip:
             return
 
+        # Only remove white space if we're not in a pre block.
+        if 'pre' not in self.tags:
+
+            args = {}
+
+            # In a code block we want to strip leading and trailing new lines and white space.
+            # Without this we would have a leading space in the code block.
+            if 'code' in self.tags:
+                args['leading'] = True
+                args['trailing'] = True
+            data = remove_whitespace(data, **args)
+
         if not self.paragraph:
             self.paragraph = self.doc.add_paragraph()
 
@@ -393,13 +470,16 @@ def handle_data(self, data):
                     style = self.parse_dict_string(span['style'])
                     self.add_styles_to_run(style)
 
-
-            # add font style
+            # add font style and name
             for tag in self.tags:
-                if tag in fonts:
-                    font_style = fonts[tag]
+                if tag in font_styles:
+                    font_style = font_styles[tag]
                     setattr(self.run.font, font_style, True)
 
+                if tag in font_names:
+                    font_name = font_names[tag]
+                    self.run.font.name = font_name
+
     def ignore_nested_tables(self, tables_soup):
         """
         Returns array containing only the highest level tables
@@ -446,9 +526,7 @@ def get_tables(self):
     def run_process(self, html):
         if self.bs and BeautifulSoup:
             self.soup = BeautifulSoup(html, 'html.parser')
-            html = remove_whitespace(str(self.soup))
-        else:
-            html = remove_whitespace(html)
+            html = str(self.soup)
         if self.include_tables:
             self.get_tables()
         self.feed(html)

tests/code.html

@@ -0,0 +1,27 @@
+<h1>Code Test</h1>
+
+<p>This test validates handling of code blocks.</p>
+
+<h2>Pre-formatted block</h2>
+
+<pre>
+This is a pre-formatted block.
+  That should be pre-formatted.
+Retaining any carriage returns, and     all    white    space.
+
+And blank lines.
+</pre>
+
+<h2>Code block</h2>
+
+<p><code>
+This is a code block.
+  That should be NOT be pre-formatted.
+It should NOT retain carriage returns, or     all    white    space.
+
+or blank lines.
+</code></p>
+
+<h2>Code elements</h2>
+
+<p>This is a sentence that includes <code>code</code> elements.</p>

tests/test.py

@@ -95,5 +95,47 @@ def test_add_html_to_cells_method(self):
         html = '''<p>Line 0 with p tags</p>Line 1 without p tags'''
         self.parser.add_html_to_cell(html, cell)
 
+    def test_inline_code(self):
+        self.document.add_heading(
+            'Test: inline code block',
+            level=1
+        )
+
+        html = "<p>This is a sentence that contains <code>some code elements</code> that " \
+               "should appear as code.</p>"
+        self.parser.add_html_to_document(html, self.document)
+
+    def test_code_block(self):
+        self.document.add_heading(
+            'Test: code block',
+            level=1
+        )
+
+        html = """<p><code>
+This is a code block.
+  That should be NOT be pre-formatted.
+It should NOT retain carriage returns,
+
+or blank lines.
+</code></p>"""
+        self.parser.add_html_to_document(html, self.document)
+
+    def test_pre_block(self):
+        self.document.add_heading(
+            'Test: pre block',
+            level=1
+        )
+
+        html = """<pre>
+This is a pre-formatted block.
+  That should be pre-formatted.
+Retaining any carriage returns,
+
+and blank lines.
+</pre>
+"""
+        self.parser.add_html_to_document(html, self.document)
+
+
 if __name__ == '__main__':
     unittest.main()

tests/test_code.py

@@ -0,0 +1,9 @@
+import os
+from .context import HtmlToDocx, test_dir
+
+# Manual test (requires inspection of result) for converting code and pre blocks.
+
+filename = os.path.join(test_dir, 'code.html')
+d = HtmlToDocx()
+
+d.parse_html_file(filename)