diff --git a/scrapegraphai/utils/convert_to_md.py b/scrapegraphai/utils/convert_to_md.py index bd7c994c..c6bf0bb5 100644 --- a/scrapegraphai/utils/convert_to_md.py +++ b/scrapegraphai/utils/convert_to_md.py @@ -2,8 +2,6 @@ convert_to_md module """ -from urllib.parse import urlparse - import html2text @@ -29,8 +27,6 @@ def convert_to_md(html: str, url: str = None) -> str: h.body_width = 0 if url is not None: - parsed_url = urlparse(url) - domain = f"{parsed_url.scheme}://{parsed_url.netloc}" - h.baseurl = domain + h.baseurl = url return h.handle(html) diff --git a/tests/utils/convert_to_md_test.py b/tests/utils/convert_to_md_test.py index d2b64b48..1c02550f 100644 --- a/tests/utils/convert_to_md_test.py +++ b/tests/utils/convert_to_md_test.py @@ -11,6 +11,15 @@ def test_html_with_links_and_images(): assert convert_to_md(html) is not None +def test_relative_links_use_document_url_as_base(): + html = 'GuideLogo' + + markdown = convert_to_md(html, "https://example.com/docs/index.html") + + assert "[Guide](https://example.com/docs/guide.html)" in markdown + assert "![Logo](https://example.com/docs/images/logo.png)" in markdown + + def test_html_with_tables(): html = """