From d558617cd769d5f01693adb929ea044e91d55b02 Mon Sep 17 00:00:00 2001 From: Igor Dvorkin Date: Sun, 15 Nov 2020 09:04:22 -0800 Subject: [PATCH 1/3] Add support for headings that include nested block elements --- markdownify/__init__.py | 64 +++++++++++++++++++++++++++------------ tests/test_conversions.py | 8 +++++ 2 files changed, 52 insertions(+), 20 deletions(-) diff --git a/markdownify/__init__.py b/markdownify/__init__.py index 2d5daf1..0a376a7 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -61,22 +61,28 @@ class MarkdownConverter(object): def convert(self, html): soup = BeautifulSoup(html, 'html.parser') - return self.process_tag(soup, children_only=True) + return self.process_tag(soup, convert_as_inline=False, children_only=True) - def process_tag(self, node, children_only=False): + def process_tag(self, node, convert_as_inline, children_only=False): text = '' + # markdown headings can't include block elements (elements w/newlines) + isHeading = node.name.startswith('h') + convert_children_as_inline = convert_as_inline + + if not children_only and isHeading: + convert_children_as_inline = True # Convert the children first for el in node.children: if isinstance(el, NavigableString): text += self.process_text(six.text_type(el)) else: - text += self.process_tag(el) + text += self.process_tag(el, convert_children_as_inline) if not children_only: convert_fn = getattr(self, 'convert_%s' % node.name, None) if convert_fn and self.should_convert_tag(node.name): - text = convert_fn(node, text) + text = convert_fn(node, text, convert_as_inline) return text @@ -89,8 +95,8 @@ class MarkdownConverter(object): if m: n = int(m.group(1)) - def convert_tag(el, text): - return self.convert_hn(n, el, text) + def convert_tag(el, text, convert_as_inline): + return self.convert_hn(n, el, text, convert_as_inline) convert_tag.__name__ = 'convert_h%s' % n setattr(self, convert_tag.__name__, convert_tag) @@ -116,10 +122,12 @@ class MarkdownConverter(object): text = (text or '').rstrip() return '%s\n%s\n\n' % (text, pad_char * len(text)) if text else '' - def convert_a(self, el, text): + def convert_a(self, el, text, convert_as_inline): prefix, suffix, text = chomp(text) if not text: return '' + if convert_as_inline: + return text href = el.get('href') title = el.get('title') if self.options['autolinks'] and text == href and not title: @@ -128,22 +136,32 @@ class MarkdownConverter(object): title_part = ' "%s"' % title.replace('"', r'\"') if title else '' return '%s[%s](%s%s)%s' % (prefix, text, href, title_part, suffix) if href else text - def convert_b(self, el, text): - return self.convert_strong(el, text) + def convert_b(self, el, text, convert_as_inline): + return self.convert_strong(el, text, convert_as_inline) + + def convert_blockquote(self, el, text, convert_as_inline): + + if convert_as_inline: + return text - def convert_blockquote(self, el, text): return '\n' + line_beginning_re.sub('> ', text) if text else '' - def convert_br(self, el, text): + def convert_br(self, el, text, convert_as_inline): + if convert_as_inline: + return "" + return ' \n' - def convert_em(self, el, text): + def convert_em(self, el, text, convert_as_inline): prefix, suffix, text = chomp(text) if not text: return '' return '%s*%s*%s' % (prefix, text, suffix) - def convert_hn(self, n, el, text): + def convert_hn(self, n, el, text, convert_as_inline): + if convert_as_inline: + return text + style = self.options['heading_style'] text = text.rstrip() if style == UNDERLINED and n <= 2: @@ -154,10 +172,14 @@ class MarkdownConverter(object): return '%s %s %s\n\n' % (hashes, text, hashes) return '%s %s\n\n' % (hashes, text) - def convert_i(self, el, text): - return self.convert_em(el, text) + def convert_i(self, el, text, convert_as_inline): + return self.convert_em(el, text, convert_as_inline) + + def convert_list(self, el, text, convert_as_inline): + + # Converting a list to inline is undefined. + # Ignoring convert_to_inline for list. - def convert_list(self, el, text): nested = False while el: if el.name == 'li': @@ -172,7 +194,7 @@ class MarkdownConverter(object): convert_ul = convert_list convert_ol = convert_list - def convert_li(self, el, text): + def convert_li(self, el, text, convert_as_inline): parent = el.parent if parent is not None and parent.name == 'ol': if parent.get("start"): @@ -190,16 +212,18 @@ class MarkdownConverter(object): bullet = bullets[depth % len(bullets)] return '%s %s\n' % (bullet, text or '') - def convert_p(self, el, text): + def convert_p(self, el, text, convert_as_inline): + if convert_as_inline: + return text return '%s\n\n' % text if text else '' - def convert_strong(self, el, text): + def convert_strong(self, el, text, convert_as_inline): prefix, suffix, text = chomp(text) if not text: return '' return '%s**%s**%s' % (prefix, text, suffix) - def convert_img(self, el, text): + def convert_img(self, el, text, convert_as_inline): alt = el.attrs.get('alt', None) or '' src = el.attrs.get('src', None) or '' title = el.attrs.get('title', None) or '' diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 12ce36b..65dbfd2 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -107,6 +107,14 @@ def test_hn(): assert md('
Hello
') == '###### Hello\n\n' +def test_hn_nested_tag(): + assert md('

A Bold C

') == '### A **Bold** C\n\n' + assert md('

A

P

C

') == '### A P C\n\n' + assert md('

A

P

C

', heading_style=ATX_CLOSED) == '# A P C #\n\n' + assert md('

A

P

C

', heading_style=ATX) == '# A P C\n\n' + assert md('

A
BQ
C

') == '### A BQ C\n\n' + + def test_atx_headings(): assert md('

Hello

', heading_style=ATX) == '# Hello\n\n' assert md('

Hello

', heading_style=ATX) == '## Hello\n\n' From 7780f82c302483a5537175f435d271d66cfc4d84 Mon Sep 17 00:00:00 2001 From: Igor Dvorkin Date: Fri, 11 Dec 2020 16:54:14 -0800 Subject: [PATCH 2/3] Using a regexp to determine if a tag is a heading. --- markdownify/__init__.py | 3 ++- tests/test_conversions.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/markdownify/__init__.py b/markdownify/__init__.py index 0a376a7..cb12d43 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -6,6 +6,7 @@ import six convert_heading_re = re.compile(r'convert_h(\d+)') line_beginning_re = re.compile(r'^', re.MULTILINE) whitespace_re = re.compile(r'[\r\n\s\t ]+') +html_heading_re = re.compile(r'h[1-6]') # Heading styles @@ -66,7 +67,7 @@ class MarkdownConverter(object): def process_tag(self, node, convert_as_inline, children_only=False): text = '' # markdown headings can't include block elements (elements w/newlines) - isHeading = node.name.startswith('h') + isHeading = html_heading_re.match(node.name) is not None convert_children_as_inline = convert_as_inline if not children_only and isHeading: diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 65dbfd2..ab1ce05 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -115,6 +115,14 @@ def test_hn_nested_tag(): assert md('

A
BQ
C

') == '### A BQ C\n\n' +def test_hr(): + assert md('
hr') == 'hr' + + +def test_head(): + assert md('head') == 'head' + + def test_atx_headings(): assert md('

Hello

', heading_style=ATX) == '# Hello\n\n' assert md('

Hello

', heading_style=ATX) == '## Hello\n\n' From 05ea8dc58ab35e1d3e8bae5b84df77bd4e3dc14d Mon Sep 17 00:00:00 2001 From: Igor Dvorkin Date: Sun, 13 Dec 2020 17:39:08 +0000 Subject: [PATCH 3/3] Add many tests and support image tag --- markdownify/__init__.py | 3 +++ tests/test_conversions.py | 38 ++++++++++++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/markdownify/__init__.py b/markdownify/__init__.py index cb12d43..2cd8fc8 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -229,6 +229,9 @@ class MarkdownConverter(object): src = el.attrs.get('src', None) or '' title = el.attrs.get('title', None) or '' title_part = ' "%s"' % title.replace('"', r'\"') if title else '' + if convert_as_inline: + return alt + return '![%s](%s%s)' % (alt, src, title_part) diff --git a/tests/test_conversions.py b/tests/test_conversions.py index ab1ce05..f5fc1c2 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -107,12 +107,42 @@ def test_hn(): assert md('
Hello
') == '###### Hello\n\n' -def test_hn_nested_tag(): - assert md('

A Bold C

') == '### A **Bold** C\n\n' - assert md('

A

P

C

') == '### A P C\n\n' +def test_hn_nested_tag_heading_style(): assert md('

A

P

C

', heading_style=ATX_CLOSED) == '# A P C #\n\n' assert md('

A

P

C

', heading_style=ATX) == '# A P C\n\n' - assert md('

A
BQ
C

') == '### A BQ C\n\n' + + +def test_hn_nested_simple_tag(): + tag_to_markdown = [ + ("strong", "**strong**"), + ("b", "**b**"), + ("em", "*em*"), + ("i", "*i*"), + ("p", "p"), + ("a", "a"), + ("div", "div"), + ("blockquote", "blockquote"), + ] + + for tag, markdown in tag_to_markdown: + assert md('

A <' + tag + '>' + tag + ' B

') == '### A ' + markdown + ' B\n\n' + + assert md('

A
B

', heading_style=ATX) == '### A B\n\n' + + # Nested lists not supported + # assert md('

A
  • li1
  • l2

', heading_style=ATX) == '### A li1 li2 B\n\n' + + +def test_hn_nested_img(): + assert md('Alt text') == '![Alt text](/path/to/img.jpg "Optional title")' + assert md('Alt text') == '![Alt text](/path/to/img.jpg)' + image_attributes_to_markdown = [ + ("", ""), + ("alt='Alt Text'", "Alt Text"), + ("alt='Alt Text' title='Optional title'", "Alt Text"), + ] + for image_attributes, markdown in image_attributes_to_markdown: + assert md('

A B

') == '### A ' + markdown + ' B\n\n' def test_hr():