From 2d654a6b7e822e1547199da855c9d304d162cb27 Mon Sep 17 00:00:00 2001 From: Vincent Kelleher Date: Sat, 29 Mar 2025 11:29:29 +0100 Subject: [PATCH 01/12] Add beautiful_soup_parser option (#206) * add beautiful_soup_parser option * add Beautiful Soup parser argument to command line --------- Co-authored-by: Vincent Kelleher Co-authored-by: AlexVonB --- README.rst | 7 +++++++ markdownify/__init__.py | 3 ++- markdownify/main.py | 10 +++++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c6c6d84..71f6b14 100644 --- a/README.rst +++ b/README.rst @@ -157,6 +157,13 @@ strip_document within the document are unaffected. Defaults to ``STRIP``. +beautiful_soup_parser + Specify the Beautiful Soup parser to be used for interpreting HTML markup. Parsers such + as `html5lib`, `lxml` or even a custom parser as long as it is installed on the execution + environment. Defaults to ``html.parser``. + +.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ + Options may be specified as kwargs to the ``markdownify`` function, or as a nested ``Options`` class in ``MarkdownConverter`` subclasses. diff --git a/markdownify/__init__.py b/markdownify/__init__.py index 7f69bfe..25196e6 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -154,6 +154,7 @@ def _next_block_content_sibling(el): class MarkdownConverter(object): class DefaultOptions: autolinks = True + beautiful_soup_parser = 'html.parser' bullets = '*+-' # An iterable of bullet types. code_language = '' code_language_callback = None @@ -191,7 +192,7 @@ class MarkdownConverter(object): self.convert_fn_cache = {} def convert(self, html): - soup = BeautifulSoup(html, 'html.parser') + soup = BeautifulSoup(html, self.options['beautiful_soup_parser']) return self.convert_soup(soup) def convert_soup(self, soup): diff --git a/markdownify/main.py b/markdownify/main.py index 432efb5..2fc02f8 100644 --- a/markdownify/main.py +++ b/markdownify/main.py @@ -55,7 +55,9 @@ def main(argv=sys.argv[1:]): parser.add_argument('--no-escape-underscores', dest='escape_underscores', action='store_false', help="Do not escape '_' to '\\_' in text.") - parser.add_argument('-i', '--keep-inline-images-in', nargs='*', + parser.add_argument('-i', '--keep-inline-images-in', + default=[], + nargs='*', help="Images are converted to their alt-text when the images are " "located inside headlines or table cells. If some inline images " "should be converted to markdown images instead, this option can " @@ -68,6 +70,12 @@ def main(argv=sys.argv[1:]): parser.add_argument('-w', '--wrap', action='store_true', help="Wrap all text paragraphs at --wrap-width characters.") parser.add_argument('--wrap-width', type=int, default=80) + parser.add_argument('-p', '--beautiful-soup-parser', + dest='beautiful_soup_parser', + default='html.parser', + help="Specify the Beautiful Soup parser to be used for interpreting HTML markup. Parsers such " + "as html5lib, lxml or even a custom parser as long as it is installed on the execution " + "environment.") args = parser.parse_args(argv) print(markdownify(**vars(args))) From e29de4e75314daf9267f30f47fd7a091ef5f1deb Mon Sep 17 00:00:00 2001 From: Chris Papademetrious Date: Sun, 20 Apr 2025 06:20:01 -0400 Subject: [PATCH 02/12] make convert_hn() public instead of internal (#213) Signed-off-by: chrispy --- markdownify/__init__.py | 10 +++++----- tests/test_conversions.py | 3 ++- tests/test_custom_converter.py | 10 ++++++++-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/markdownify/__init__.py b/markdownify/__init__.py index 25196e6..780761c 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -363,11 +363,11 @@ class MarkdownConverter(object): if not self.should_convert_tag(tag_name): return None - # Handle headings with _convert_hn() function + # Handle headings with convert_hN() function match = re_html_heading.match(tag_name) if match: n = int(match.group(1)) - return lambda el, text, parent_tags: self._convert_hn(n, el, text, parent_tags) + return lambda el, text, parent_tags: self.convert_hN(n, el, text, parent_tags) # For other tags, look up their conversion function by tag name convert_fn_name = "convert_%s" % re_make_convert_fn_name.sub('_', tag_name) @@ -510,12 +510,12 @@ class MarkdownConverter(object): return '\n\n%s\n' % text - def _convert_hn(self, n, el, text, parent_tags): - """ Method name prefixed with _ to prevent to call this """ + def convert_hN(self, n, el, text, parent_tags): + # convert_hN() converts tags, where N is any integer if '_inline' in parent_tags: return text - # prevent MemoryErrors in case of very large n + # Markdown does not support heading depths of n > 6 n = max(1, min(6, n)) style = self.options['heading_style'].lower() diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 0df8f57..27951ee 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -164,7 +164,8 @@ def test_hn(): assert md('
Hello
') == '\n\n##### Hello\n\n' assert md('
Hello
') == '\n\n###### Hello\n\n' assert md('Hello') == md('
Hello
') - assert md('Hello') == md('Hello') + assert md('Hello') == md('

Hello

') + assert md('Hello') == md('Hello') def test_hn_chained(): diff --git a/tests/test_custom_converter.py b/tests/test_custom_converter.py index f4734c9..26faf90 100644 --- a/tests/test_custom_converter.py +++ b/tests/test_custom_converter.py @@ -12,7 +12,11 @@ class UnitTestConverter(MarkdownConverter): def convert_custom_tag(self, el, text, parent_tags): """Ensure conversion function is found for tags with special characters in name""" - return "FUNCTION USED: %s" % text + return "convert_custom_tag(): %s" % text + + def convert_hN(self, n, el, text, parent_tags): + """Ensure conversion function is found for headings""" + return "convert_hN(%d): %s" % (n, text) def test_custom_conversion_functions(): @@ -23,7 +27,9 @@ def test_custom_conversion_functions(): assert md('Alt texttext') == '![Alt text](/path/to/img.jpg "Optional title")\n\ntext' assert md('Alt texttext') == '![Alt text](/path/to/img.jpg)\n\ntext' - assert md("text") == "FUNCTION USED: text" + assert md("text") == "convert_custom_tag(): text" + + assert md("

text

") == "convert_hN(3): text" def test_soup(): From 0e1a849346f24e9b15eefb11752feaf7b4b821fa Mon Sep 17 00:00:00 2001 From: Colin <144122160+colinrobinsonuib@users.noreply.github.com> Date: Mon, 28 Apr 2025 12:37:33 +0200 Subject: [PATCH 03/12] Add conversion support for tags (#217) --- markdownify/__init__.py | 3 +++ tests/test_conversions.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/markdownify/__init__.py b/markdownify/__init__.py index 780761c..a43f3a7 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -650,6 +650,9 @@ class MarkdownConverter(object): return '\n\n```%s\n%s\n```\n\n' % (code_language, text) + def convert_q(self, el, text, parent_tags): + return '"' + text + '"' + def convert_script(self, el, text, parent_tags): return '' diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 27951ee..6145411 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -305,6 +305,11 @@ def test_pre(): assert md("

foo

\n
bar
\n

baz

", sub_symbol="^") == "\n\nfoo\n\n```\nbar\n```\n\nbaz" +def test_q(): + assert md('foo quote bar') == 'foo "quote" bar' + assert md('foo quote bar') == 'foo "quote" bar' + + def test_script(): assert md('foo bar') == 'foo bar' From 016251e915a4cb44b2f21a94db85c733e12a665a Mon Sep 17 00:00:00 2001 From: Chris Papademetrious Date: Sat, 3 May 2025 10:57:09 -0400 Subject: [PATCH 04/12] ensure that explicitly provided heading conversion functions are used (#212) (#214) Signed-off-by: chrispy --- markdownify/__init__.py | 16 ++++++++++------ tests/test_custom_converter.py | 8 +++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/markdownify/__init__.py b/markdownify/__init__.py index a43f3a7..c732711 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -363,16 +363,20 @@ class MarkdownConverter(object): if not self.should_convert_tag(tag_name): return None - # Handle headings with convert_hN() function + # Look for an explicitly defined conversion function by tag name first + convert_fn_name = "convert_%s" % re_make_convert_fn_name.sub("_", tag_name) + convert_fn = getattr(self, convert_fn_name, None) + if convert_fn: + return convert_fn + + # If tag is any heading, handle with convert_hN() function match = re_html_heading.match(tag_name) if match: - n = int(match.group(1)) + n = int(match.group(1)) # get value of N from return lambda el, text, parent_tags: self.convert_hN(n, el, text, parent_tags) - # For other tags, look up their conversion function by tag name - convert_fn_name = "convert_%s" % re_make_convert_fn_name.sub('_', tag_name) - convert_fn = getattr(self, convert_fn_name, None) - return convert_fn + # No conversion function was found + return None def should_convert_tag(self, tag): """Given a tag name, return whether to convert based on strip/convert options.""" diff --git a/tests/test_custom_converter.py b/tests/test_custom_converter.py index 26faf90..00a83fc 100644 --- a/tests/test_custom_converter.py +++ b/tests/test_custom_converter.py @@ -14,8 +14,12 @@ class UnitTestConverter(MarkdownConverter): """Ensure conversion function is found for tags with special characters in name""" return "convert_custom_tag(): %s" % text + def convert_h1(self, el, text, parent_tags): + """Ensure explicit heading conversion function is used""" + return "convert_h1: %s" % (text) + def convert_hN(self, n, el, text, parent_tags): - """Ensure conversion function is found for headings""" + """Ensure general heading conversion function is used""" return "convert_hN(%d): %s" % (n, text) @@ -29,6 +33,8 @@ def test_custom_conversion_functions(): assert md("text") == "convert_custom_tag(): text" + assert md("

text

") == "convert_h1: text" + assert md("

text

") == "convert_hN(3): text" From 75ab3064dd8f10018d2fce09f62017943692a521 Mon Sep 17 00:00:00 2001 From: Chris Papademetrious Date: Sat, 14 Jun 2025 09:06:22 -0400 Subject: [PATCH 05/12] allow BeautifulSoup configuration kwargs to be specified (#224) Signed-off-by: chrispy --- README.rst | 14 +++++++++----- markdownify/__init__.py | 8 ++++++-- markdownify/main.py | 9 ++++----- tests/test_args.py | 6 ++++++ 4 files changed, 25 insertions(+), 12 deletions(-) mode change 100644 => 100755 markdownify/main.py diff --git a/README.rst b/README.rst index 71f6b14..946c83d 100644 --- a/README.rst +++ b/README.rst @@ -157,12 +157,16 @@ strip_document within the document are unaffected. Defaults to ``STRIP``. -beautiful_soup_parser - Specify the Beautiful Soup parser to be used for interpreting HTML markup. Parsers such - as `html5lib`, `lxml` or even a custom parser as long as it is installed on the execution - environment. Defaults to ``html.parser``. +bs4_options + Specify additional configuration options for the ``BeautifulSoup`` object + used to interpret the HTML markup. String and list values (such as ``lxml`` + or ``html5lib``) are treated as ``features`` arguments to control parser + selection. Dictionary values (such as ``{"from_encoding": "iso-8859-8"}``) + are treated as full kwargs to be used for the BeautifulSoup constructor, + allowing specification of any parameter. For parameter details, see the + Beautiful Soup documentation at: -.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ +.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/bs4/doc/ Options may be specified as kwargs to the ``markdownify`` function, or as a nested ``Options`` class in ``MarkdownConverter`` subclasses. diff --git a/markdownify/__init__.py b/markdownify/__init__.py index c732711..b219ca2 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -154,7 +154,7 @@ def _next_block_content_sibling(el): class MarkdownConverter(object): class DefaultOptions: autolinks = True - beautiful_soup_parser = 'html.parser' + bs4_options = 'html.parser' bullets = '*+-' # An iterable of bullet types. code_language = '' code_language_callback = None @@ -188,11 +188,15 @@ class MarkdownConverter(object): raise ValueError('You may specify either tags to strip or tags to' ' convert, but not both.') + # If a string or list is passed to bs4_options, assume it is a 'features' specification + if not isinstance(self.options['bs4_options'], dict): + self.options['bs4_options'] = {'features': self.options['bs4_options']} + # Initialize the conversion function cache self.convert_fn_cache = {} def convert(self, html): - soup = BeautifulSoup(html, self.options['beautiful_soup_parser']) + soup = BeautifulSoup(html, **self.options['bs4_options']) return self.convert_soup(soup) def convert_soup(self, soup): diff --git a/markdownify/main.py b/markdownify/main.py old mode 100644 new mode 100755 index 2fc02f8..ba70671 --- a/markdownify/main.py +++ b/markdownify/main.py @@ -70,12 +70,11 @@ def main(argv=sys.argv[1:]): parser.add_argument('-w', '--wrap', action='store_true', help="Wrap all text paragraphs at --wrap-width characters.") parser.add_argument('--wrap-width', type=int, default=80) - parser.add_argument('-p', '--beautiful-soup-parser', - dest='beautiful_soup_parser', + parser.add_argument('--bs4-options', default='html.parser', - help="Specify the Beautiful Soup parser to be used for interpreting HTML markup. Parsers such " - "as html5lib, lxml or even a custom parser as long as it is installed on the execution " - "environment.") + help="Specifies the parser that BeautifulSoup should use to parse " + "the HTML markup. Examples include 'html5.parser', 'lxml', and " + "'html5lib'.") args = parser.parse_args(argv) print(markdownify(**vars(args))) diff --git a/tests/test_args.py b/tests/test_args.py index 301c19f..1ba6482 100644 --- a/tests/test_args.py +++ b/tests/test_args.py @@ -32,3 +32,9 @@ def test_strip_document(): assert markdownify("

Hello

", strip_document=RSTRIP) == "\n\nHello" assert markdownify("

Hello

", strip_document=STRIP) == "Hello" assert markdownify("

Hello

", strip_document=None) == "\n\nHello\n\n" + + +def bs4_options(): + assert markdownify("

Hello

", bs4_options="html.parser") == "Hello" + assert markdownify("

Hello

", bs4_options=["html.parser"]) == "Hello" + assert markdownify("

Hello

", bs4_options={"features": "html.parser"}) == "Hello" From 9b1412aa5b5bca345806068b66d35086cbc88b25 Mon Sep 17 00:00:00 2001 From: Chris Papademetrious Date: Sat, 14 Jun 2025 16:37:47 -0400 Subject: [PATCH 06/12] implement a strip_pre configuration option (#218) (#222) Signed-off-by: chrispy --- README.rst | 6 ++++++ markdownify/__init__.py | 31 ++++++++++++++++++++++++++++++- tests/test_args.py | 9 ++++++++- tests/test_conversions.py | 2 +- 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 946c83d..9a0798f 100644 --- a/README.rst +++ b/README.rst @@ -157,6 +157,12 @@ strip_document within the document are unaffected. Defaults to ``STRIP``. +strip_pre + Controls whether leading/trailing blank lines are removed from ``
``
+  tags. Supported values are ``STRIP`` (all leading/trailing blank lines),
+  ``STRIP_ONE`` (one leading/trailing blank line), and ``None`` (neither).
+  Defaults to ``STRIP``.
+
 bs4_options
   Specify additional configuration options for the ``BeautifulSoup`` object
   used to interpret the HTML markup. String and list values (such as ``lxml``
diff --git a/markdownify/__init__.py b/markdownify/__init__.py
index b219ca2..72c5214 100644
--- a/markdownify/__init__.py
+++ b/markdownify/__init__.py
@@ -11,6 +11,10 @@ re_whitespace = re.compile(r'[\t ]+')
 re_all_whitespace = re.compile(r'[\t \r\n]+')
 re_newline_whitespace = re.compile(r'[\t \r\n]*[\r\n][\t \r\n]*')
 re_html_heading = re.compile(r'h(\d+)')
+re_pre_lstrip1 = re.compile(r'^ *\n')
+re_pre_rstrip1 = re.compile(r'\n *$')
+re_pre_lstrip = re.compile(r'^[ \n]*\n')
+re_pre_rstrip = re.compile(r'[ \n]*$')
 
 # Pattern for creating convert_ function names from tag names
 re_make_convert_fn_name = re.compile(r'[\[\]:-]')
@@ -51,10 +55,25 @@ BACKSLASH = 'backslash'
 ASTERISK = '*'
 UNDERSCORE = '_'
 
-# Document strip styles
+# Document/pre strip styles
 LSTRIP = 'lstrip'
 RSTRIP = 'rstrip'
 STRIP = 'strip'
+STRIP_ONE = 'strip_one'
+
+
+def strip1_pre(text):
+    """Strip one leading and trailing newline from a 
 string."""
+    text = re_pre_lstrip1.sub('', text)
+    text = re_pre_rstrip1.sub('', text)
+    return text
+
+
+def strip_pre(text):
+    """Strip all leading and trailing newlines from a 
 string."""
+    text = re_pre_lstrip.sub('', text)
+    text = re_pre_rstrip.sub('', text)
+    return text
 
 
 def chomp(text):
@@ -168,6 +187,7 @@ class MarkdownConverter(object):
         newline_style = SPACES
         strip = None
         strip_document = STRIP
+        strip_pre = STRIP
         strong_em_symbol = ASTERISK
         sub_symbol = ''
         sup_symbol = ''
@@ -656,6 +676,15 @@ class MarkdownConverter(object):
         if self.options['code_language_callback']:
             code_language = self.options['code_language_callback'](el) or code_language
 
+        if self.options['strip_pre'] == STRIP:
+            text = strip_pre(text)  # remove all leading/trailing newlines
+        elif self.options['strip_pre'] == STRIP_ONE:
+            text = strip1_pre(text)  # remove one leading/trailing newline
+        elif self.options['strip_pre'] is None:
+            pass  # leave leading and trailing newlines as-is
+        else:
+            raise ValueError('Invalid value for strip_pre: %s' % self.options['strip_pre'])
+
         return '\n\n```%s\n%s\n```\n\n' % (code_language, text)
 
     def convert_q(self, el, text, parent_tags):
diff --git a/tests/test_args.py b/tests/test_args.py
index 1ba6482..838ef9d 100644
--- a/tests/test_args.py
+++ b/tests/test_args.py
@@ -2,7 +2,7 @@
 Test whitelisting/blacklisting of specific tags.
 
 """
-from markdownify import markdownify, LSTRIP, RSTRIP, STRIP
+from markdownify import markdownify, LSTRIP, RSTRIP, STRIP, STRIP_ONE
 from .utils import md
 
 
@@ -34,6 +34,13 @@ def test_strip_document():
     assert markdownify("

Hello

", strip_document=None) == "\n\nHello\n\n" +def test_strip_pre(): + assert markdownify("
  \n  \n  Hello  \n  \n  
") == "```\n Hello\n```" + assert markdownify("
  \n  \n  Hello  \n  \n  
", strip_pre=STRIP) == "```\n Hello\n```" + assert markdownify("
  \n  \n  Hello  \n  \n  
", strip_pre=STRIP_ONE) == "```\n \n Hello \n \n```" + assert markdownify("
  \n  \n  Hello  \n  \n  
", strip_pre=None) == "```\n \n \n Hello \n \n \n```" + + def bs4_options(): assert markdownify("

Hello

", bs4_options="html.parser") == "Hello" assert markdownify("

Hello

", bs4_options=["html.parser"]) == "Hello" diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 6145411..825559b 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -370,4 +370,4 @@ def test_spaces(): assert md('test
text
after') == 'test\n> text\n\nafter' assert md('
  1. x
  2. y
') == '\n\n1. x\n2. y\n' assert md('
  • x
  • y
  • ') == '\n\n* x\n* y\n' - assert md('test
     foo 
    bar') == 'test\n\n```\n foo \n```\n\nbar' + assert md('test
     foo 
    bar') == 'test\n\n```\n foo\n```\n\nbar' From 48724e700270353b5bbf53f98e85d89796b7d0bf Mon Sep 17 00:00:00 2001 From: Chris Papademetrious Date: Sun, 29 Jun 2025 14:56:21 -0400 Subject: [PATCH 07/12] support backticks in spans (#226) (#230) Signed-off-by: chrispy --- markdownify/__init__.py | 23 ++++++++++++++++++++--- tests/test_conversions.py | 3 +++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/markdownify/__init__.py b/markdownify/__init__.py index 72c5214..e901d10 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -41,6 +41,9 @@ re_escape_misc_hashes = re.compile(r'(\s|^)(#{1,6}(?:\s|$))') # confused with a list item re_escape_misc_list_items = re.compile(r'((?:\s|^)[0-9]{1,9})([.)](?:\s|$))') +# Find consecutive backtick sequences in a string +re_backtick_runs = re.compile(r'`+') + # Heading styles ATX = 'atx' ATX_CLOSED = 'atx_closed' @@ -480,10 +483,24 @@ class MarkdownConverter(object): return ' \n' def convert_code(self, el, text, parent_tags): - if 'pre' in parent_tags: + if '_noformat' in parent_tags: return text - converter = abstract_inline_conversion(lambda self: '`') - return converter(self, el, text, parent_tags) + + prefix, suffix, text = chomp(text) + if not text: + return '' + + # Find the maximum number of consecutive backticks in the text, then + # delimit the code span with one more backtick than that + max_backticks = max((len(match) for match in re.findall(re_backtick_runs, text)), default=0) + markup_delimiter = '`' * (max_backticks + 1) + + # If the maximum number of backticks is greater than zero, add a space + # to avoid interpretation of inside backticks as literals + if max_backticks > 0: + text = " " + text + " " + + return '%s%s%s%s%s' % (prefix, markup_delimiter, text, markup_delimiter, suffix) convert_del = abstract_inline_conversion(lambda self: '~~') diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 825559b..dd99dfb 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -101,6 +101,9 @@ def test_code(): assert md('foo bar baz') == '`foo bar baz`' assert md('foobarbaz', sup_symbol='^') == '`foobarbaz`' assert md('foobarbaz', sub_symbol='^') == '`foobarbaz`' + assert md('foo`bar`baz') == 'foo`` `bar` ``baz' + assert md('foo``bar``baz') == 'foo``` ``bar`` ```baz' + assert md('foo `bar` baz') == 'foo `` `bar` `` baz' def test_dl(): From 76e5edb3575d9a67b70b022f898ea0d0accb6434 Mon Sep 17 00:00:00 2001 From: alheiveea Date: Wed, 9 Jul 2025 22:08:47 +0200 Subject: [PATCH 08/12] limit colspan values to range [1, 1000] (#232) --- markdownify/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/markdownify/__init__.py b/markdownify/__init__.py index e901d10..148d340 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -735,13 +735,13 @@ class MarkdownConverter(object): def convert_td(self, el, text, parent_tags): colspan = 1 if 'colspan' in el.attrs and el['colspan'].isdigit(): - colspan = int(el['colspan']) + colspan = max(1, min(1000, int(el['colspan']))) return ' ' + text.strip().replace("\n", " ") + ' |' * colspan def convert_th(self, el, text, parent_tags): colspan = 1 if 'colspan' in el.attrs and el['colspan'].isdigit(): - colspan = int(el['colspan']) + colspan = max(1, min(1000, int(el['colspan']))) return ' ' + text.strip().replace("\n", " ") + ' |' * colspan def convert_tr(self, el, text, parent_tags): @@ -762,7 +762,7 @@ class MarkdownConverter(object): full_colspan = 0 for cell in cells: if 'colspan' in cell.attrs and cell['colspan'].isdigit(): - full_colspan += int(cell["colspan"]) + full_colspan += max(1, min(1000, int(cell['colspan']))) else: full_colspan += 1 if ((is_headrow From 7edbc5a22b2ff90a3d9898903d6f598511553df6 Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Tue, 15 Jul 2025 07:52:04 +1200 Subject: [PATCH 09/12] ci: update `actions/checkout` to v4 (#233) * ci: update `actions/checkout` to v4 --- .github/workflows/python-app.yml | 2 +- .github/workflows/python-publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 481ade5..3e0f6fb 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v2 with: diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index c337bab..7341e30 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v2 with: From f7053e46ab505e68a1ea339945985f703f1f28f9 Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Sun, 3 Aug 2025 22:24:28 +1200 Subject: [PATCH 10/12] docs: fix typo (#234) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 9a0798f..059a68f 100644 --- a/README.rst +++ b/README.rst @@ -110,7 +110,7 @@ code_language_callback When the HTML code contains ``pre`` tags that in some way provide the code language, for example as class, this callback can be used to extract the language from the tag and prefix it to the converted ``pre`` tag. - The callback gets one single argument, an BeautifylSoup object, and returns + The callback gets one single argument, a BeautifulSoup object, and returns a string containing the code language, or ``None``. An example to use the class name as code language could be:: From 85ef82e083a9b9a3b4fe541324437bbd5c16e4bc Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Sun, 3 Aug 2025 22:35:46 +1200 Subject: [PATCH 11/12] Add basic type stubs (#221) (#215) * feat: add basic type stubs * feat: add types for constants * feat: add type for `MarkdownConverter` class * ci: add basic job for checking types * feat: add new constant * ci: install types as required * ci: install types package manually * test: add strict coverage for types * fix: allow `strip_document` to be `None` * feat: expand types for MarkdownConverter * fix: do not use `Unpack` as it requires Python 3.12 * feat: define `MarkdownConverter#convert_soup` * feat: improve type for `code_language_callback` * chore: add end-of-file newline * refactor: use `Union` for now --- .github/workflows/python-app.yml | 19 ++++++++ markdownify/__init__.pyi | 77 ++++++++++++++++++++++++++++++++ tests/types.py | 70 +++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 markdownify/__init__.pyi create mode 100644 tests/types.py diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 3e0f6fb..04d6a9a 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -30,3 +30,22 @@ jobs: - name: Build run: | python -m build -nwsx . + + types: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade setuptools setuptools_scm wheel build tox mypy types-beautifulsoup4 + - name: Check types + run: | + mypy . + mypy --strict tests/types.py diff --git a/markdownify/__init__.pyi b/markdownify/__init__.pyi new file mode 100644 index 0000000..5f9b852 --- /dev/null +++ b/markdownify/__init__.pyi @@ -0,0 +1,77 @@ +from _typeshed import Incomplete +from typing import Callable, Union + +ATX: str +ATX_CLOSED: str +UNDERLINED: str +SETEXT = UNDERLINED +SPACES: str +BACKSLASH: str +ASTERISK: str +UNDERSCORE: str +LSTRIP: str +RSTRIP: str +STRIP: str +STRIP_ONE: str + + +def markdownify( + html: str, + autolinks: bool = ..., + bs4_options: str = ..., + bullets: str = ..., + code_language: str = ..., + code_language_callback: Union[Callable[[Incomplete], Union[str, None]], None] = ..., + convert: Union[list[str], None] = ..., + default_title: bool = ..., + escape_asterisks: bool = ..., + escape_underscores: bool = ..., + escape_misc: bool = ..., + heading_style: str = ..., + keep_inline_images_in: list[str] = ..., + newline_style: str = ..., + strip: Union[list[str], None] = ..., + strip_document: Union[str, None] = ..., + strip_pre: str = ..., + strong_em_symbol: str = ..., + sub_symbol: str = ..., + sup_symbol: str = ..., + table_infer_header: bool = ..., + wrap: bool = ..., + wrap_width: int = ..., +) -> str: ... + + +class MarkdownConverter: + def __init__( + self, + autolinks: bool = ..., + bs4_options: str = ..., + bullets: str = ..., + code_language: str = ..., + code_language_callback: Union[Callable[[Incomplete], Union[str, None]], None] = ..., + convert: Union[list[str], None] = ..., + default_title: bool = ..., + escape_asterisks: bool = ..., + escape_underscores: bool = ..., + escape_misc: bool = ..., + heading_style: str = ..., + keep_inline_images_in: list[str] = ..., + newline_style: str = ..., + strip: Union[list[str], None] = ..., + strip_document: Union[str, None] = ..., + strip_pre: str = ..., + strong_em_symbol: str = ..., + sub_symbol: str = ..., + sup_symbol: str = ..., + table_infer_header: bool = ..., + wrap: bool = ..., + wrap_width: int = ..., + ) -> None: + ... + + def convert(self, html: str) -> str: + ... + + def convert_soup(self, soup: Incomplete) -> str: + ... diff --git a/tests/types.py b/tests/types.py new file mode 100644 index 0000000..7424978 --- /dev/null +++ b/tests/types.py @@ -0,0 +1,70 @@ +from markdownify import markdownify, ASTERISK, BACKSLASH, LSTRIP, RSTRIP, SPACES, STRIP, UNDERLINED, UNDERSCORE, MarkdownConverter +from bs4 import BeautifulSoup +from typing import Union + +markdownify("

    Hello

    ") == "Hello" # test default of STRIP +markdownify("

    Hello

    ", strip_document=LSTRIP) == "Hello\n\n" +markdownify("

    Hello

    ", strip_document=RSTRIP) == "\n\nHello" +markdownify("

    Hello

    ", strip_document=STRIP) == "Hello" +markdownify("

    Hello

    ", strip_document=None) == "\n\nHello\n\n" + +# default options +MarkdownConverter( + autolinks=True, + bs4_options='html.parser', + bullets='*+-', + code_language='', + code_language_callback=None, + convert=None, + default_title=False, + escape_asterisks=True, + escape_underscores=True, + escape_misc=False, + heading_style=UNDERLINED, + keep_inline_images_in=[], + newline_style=SPACES, + strip=None, + strip_document=STRIP, + strip_pre=STRIP, + strong_em_symbol=ASTERISK, + sub_symbol='', + sup_symbol='', + table_infer_header=False, + wrap=False, + wrap_width=80, +).convert("") + +# custom options +MarkdownConverter( + strip_document=None, + bullets="-", + escape_asterisks=True, + escape_underscores=True, + escape_misc=True, + autolinks=True, + default_title=True, + newline_style=BACKSLASH, + sup_symbol='^', + sub_symbol='^', + keep_inline_images_in=['h3'], + wrap=True, + wrap_width=80, + strong_em_symbol=UNDERSCORE, + code_language='python', + code_language_callback=None +).convert("") + +html = 'test' +soup = BeautifulSoup(html, 'html.parser') +MarkdownConverter().convert_soup(soup) == '**test**' + + +def callback(el: BeautifulSoup) -> Union[str, None]: + return el['class'][0] if el.has_attr('class') else None + + +MarkdownConverter(code_language_callback=callback).convert("") +MarkdownConverter(code_language_callback=lambda el: None).convert("") + +markdownify('
    test\n    foo\nbar
    ', code_language_callback=callback) +markdownify('
    test\n    foo\nbar
    ', code_language_callback=lambda el: None) From fbc135359313224e4e037925c7832c08a8149b2e Mon Sep 17 00:00:00 2001 From: AlexVonB Date: Sat, 9 Aug 2025 19:40:43 +0200 Subject: [PATCH 12/12] bump to version v1.2.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7306055..64eebba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "markdownify" -version = "1.1.0" +version = "1.2.0" authors = [{name = "Matthew Tretter", email = "m@tthewwithanm.com"}] description = "Convert HTML to markdown." readme = "README.rst"