From a23acce5ae0e91387b2da98fa2ae6210208750a9 Mon Sep 17 00:00:00 2001
From: benoit74 <benoit74@users.noreply.github.com>
Date: Fri, 25 Oct 2024 15:20:14 +0000
Subject: [PATCH 1/4] Fix error message and move item to download handling to
 rewriter

---
 scraper/src/mindtouch2zim/processor.py | 28 +++++++++++++++-----------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/scraper/src/mindtouch2zim/processor.py b/scraper/src/mindtouch2zim/processor.py
index 1502de6..5103cba 100644
--- a/scraper/src/mindtouch2zim/processor.py
+++ b/scraper/src/mindtouch2zim/processor.py
@@ -582,7 +582,7 @@ def rewrite_href_src_attributes(
     if attr_name not in ("href", "src") or not attr_value:
         return
     if not isinstance(url_rewriter, HtmlUrlsRewriter):
-        raise Exception("Expecting MindtouchUrlRewriter")
+        raise Exception("Expecting HtmlUrlsRewriter")
     new_attr_value = None
     if tag == "a":
         rewrite_result = url_rewriter(
@@ -601,17 +601,7 @@ def rewrite_href_src_attributes(
         )
         # add 'content/' to the URL since all assets will be stored in the sub.-path
         new_attr_value = f"content/{rewrite_result.rewriten_url}"
-        if rewrite_result.zim_path is not None:
-            # if item is expected to be inside the ZIM, store asset information so that
-            # we can download it afterwards
-            if rewrite_result.zim_path in url_rewriter.items_to_download:
-                url_rewriter.items_to_download[rewrite_result.zim_path].add(
-                    HttpUrl(rewrite_result.absolute_url)
-                )
-            else:
-                url_rewriter.items_to_download[rewrite_result.zim_path] = {
-                    HttpUrl(rewrite_result.absolute_url)
-                }
+        url_rewriter.add_item_to_download(rewrite_result)
     if not new_attr_value:
         # we do not (yet) support other tags / attributes so we fail the scraper
         raise ValueError(
@@ -659,6 +649,20 @@ def __call__(
         result = super().__call__(item_url, base_href, rewrite_all_url=rewrite_all_url)
         return result
 
+    def add_item_to_download(self, rewrite_result: RewriteResult):
+        """Add item to download based on rewrite result"""
+        if rewrite_result.zim_path is not None:
+            # if item is expected to be inside the ZIM, store asset information so that
+            # we can download it afterwards
+            if rewrite_result.zim_path in self.items_to_download:
+                self.items_to_download[rewrite_result.zim_path].add(
+                    HttpUrl(rewrite_result.absolute_url)
+                )
+            else:
+                self.items_to_download[rewrite_result.zim_path] = {
+                    HttpUrl(rewrite_result.absolute_url)
+                }
+
 
 class CssUrlsRewriter(ArticleUrlRewriter):
     """A rewriter for CSS processing, storing items to download as URL as processed"""

From 51ae41ee7c639ef51c1abc75a219c4e6691933ff Mon Sep 17 00:00:00 2001
From: benoit74 <benoit74@users.noreply.github.com>
Date: Fri, 25 Oct 2024 15:22:13 +0000
Subject: [PATCH 2/4] Add icon indicating external URLs

---
 zimui/index.html               |  1 +
 zimui/public/custom.css        | 15 +++++++++++++++
 zimui/public/external-link.svg | 15 +++++++++++++++
 3 files changed, 31 insertions(+)
 create mode 100644 zimui/public/custom.css
 create mode 100644 zimui/public/external-link.svg

diff --git a/zimui/index.html b/zimui/index.html
index 90659e5..c481be2 100644
--- a/zimui/index.html
+++ b/zimui/index.html
@@ -6,6 +6,7 @@
     <link rel="stylesheet" type="text/css" media="screen" href="./content/screen.css" />
     <link rel="stylesheet" type="text/css" media="print" href="./content/print.css" />
     <link rel="stylesheet" type="text/css" href="./content/inline.css" />
+    <link rel="stylesheet" type="text/css" href="./custom.css" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>Vite App</title>
   </head>
diff --git a/zimui/public/custom.css b/zimui/public/custom.css
new file mode 100644
index 0000000..d57f80f
--- /dev/null
+++ b/zimui/public/custom.css
@@ -0,0 +1,15 @@
+a[href^="http://"]:after,
+a[href^="https://"]:after
+{
+  content: '';
+  display: inline-block;
+  width: 10px;
+  height: 10px;
+  background-image: url('external-link.svg');
+  background-size: contain;
+  background-repeat: no-repeat;
+  margin-left: 5px;
+  position: relative;
+  bottom: 0px;
+  right: 0px;
+}
diff --git a/zimui/public/external-link.svg b/zimui/public/external-link.svg
new file mode 100644
index 0000000..f06102f
--- /dev/null
+++ b/zimui/public/external-link.svg
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg height="800px" width="800px" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
+	 viewBox="0 0 26 26" xml:space="preserve">
+<g>
+	<path style="fill:#db0000;" d="M18,17.759v3.366C18,22.159,17.159,23,16.125,23H4.875C3.841,23,3,22.159,3,21.125V9.875
+		C3,8.841,3.841,8,4.875,8h3.429l3.001-3h-6.43C2.182,5,0,7.182,0,9.875v11.25C0,23.818,2.182,26,4.875,26h11.25
+		C18.818,26,21,23.818,21,21.125v-6.367L18,17.759z"/>
+	<g>
+		<path style="fill:#db0000;" d="M22.581,0H12.322c-1.886,0.002-1.755,0.51-0.76,1.504l3.22,3.22l-5.52,5.519
+			c-1.145,1.144-1.144,2.998,0,4.141l2.41,2.411c1.144,1.141,2.996,1.142,4.14-0.001l5.52-5.52l3.16,3.16
+			c1.101,1.1,1.507,1.129,1.507-0.757L26,3.419C25.999-0.018,26.024-0.001,22.581,0z"/>
+	</g>
+</g>
+</svg>

From f118f3a29d2f44cdbb249225781a0cbc5cdd83ba Mon Sep 17 00:00:00 2001
From: benoit74 <benoit74@users.noreply.github.com>
Date: Fri, 25 Oct 2024 15:25:01 +0000
Subject: [PATCH 3/4] Replace Youtube and Vimeo videos with their thumbnails
 and a link to open them

---
 scraper/src/mindtouch2zim/client.py    |  9 ++--
 scraper/src/mindtouch2zim/constants.py |  3 ++
 scraper/src/mindtouch2zim/processor.py | 63 +++++++++++++++++++++++++-
 scraper/src/mindtouch2zim/vimeo.py     | 30 ++++++++++++
 zimui/public/custom.css                | 26 +++++++++++
 zimui/public/play-button.svg           | 14 ++++++
 6 files changed, 140 insertions(+), 5 deletions(-)
 create mode 100644 scraper/src/mindtouch2zim/vimeo.py
 create mode 100644 zimui/public/play-button.svg

diff --git a/scraper/src/mindtouch2zim/client.py b/scraper/src/mindtouch2zim/client.py
index 17d3a53..9c06fc4 100644
--- a/scraper/src/mindtouch2zim/client.py
+++ b/scraper/src/mindtouch2zim/client.py
@@ -7,10 +7,11 @@
 from bs4 import BeautifulSoup, NavigableString
 from pydantic import BaseModel
 
-from mindtouch2zim.constants import logger
-
-HTTP_TIMEOUT_NORMAL_SECONDS = 15
-HTTP_TIMEOUT_LONG_SECONDS = 30
+from mindtouch2zim.constants import (
+    HTTP_TIMEOUT_LONG_SECONDS,
+    HTTP_TIMEOUT_NORMAL_SECONDS,
+    logger,
+)
 
 
 class MindtouchParsingError(Exception):
diff --git a/scraper/src/mindtouch2zim/constants.py b/scraper/src/mindtouch2zim/constants.py
index 28f4a9d..7c7838a 100644
--- a/scraper/src/mindtouch2zim/constants.py
+++ b/scraper/src/mindtouch2zim/constants.py
@@ -14,4 +14,7 @@
 # As of 2024-09-24, all libraries appears to be in English.
 LANGUAGE_ISO_639_3 = "eng"
 
+HTTP_TIMEOUT_NORMAL_SECONDS = 15
+HTTP_TIMEOUT_LONG_SECONDS = 30
+
 logger = getLogger(NAME, level=logging.DEBUG)
diff --git a/scraper/src/mindtouch2zim/processor.py b/scraper/src/mindtouch2zim/processor.py
index 5103cba..544811f 100644
--- a/scraper/src/mindtouch2zim/processor.py
+++ b/scraper/src/mindtouch2zim/processor.py
@@ -15,7 +15,7 @@
 from zimscraperlib.image.conversion import convert_svg2png
 from zimscraperlib.image.probing import format_for
 from zimscraperlib.rewriting.css import CssRewriter
-from zimscraperlib.rewriting.html import HtmlRewriter
+from zimscraperlib.rewriting.html import AttrsList, HtmlRewriter, get_attr_value_from
 from zimscraperlib.rewriting.html import rules as html_rules
 from zimscraperlib.rewriting.url_rewriting import (
     ArticleUrlRewriter,
@@ -41,6 +41,7 @@
     PageModel,
     SharedModel,
 )
+from mindtouch2zim.vimeo import get_vimeo_thumbnail_url
 from mindtouch2zim.zimconfig import ZimConfig
 
 
@@ -624,6 +625,66 @@ def refuse_unsupported_tags(tag: str):
     raise UnsupportedTagError(f"Tag {tag} is not yet supported in this scraper")
 
 
+YOUTUBE_IFRAME_RE = re.compile(r".*youtube(?:-\w+)*\.\w+\/embed\/(?P<id>.*?)(?:\?.*)*$")
+VIMEO_IFRAME_RE = re.compile(r".*vimeo(?:-\w+)*\.\w+\/video\/(?:.*?)(?:\?.*)*$")
+
+
+@html_rules.rewrite_tag()
+def rewrite_iframe_tags(
+    tag: str,
+    attrs: AttrsList,
+    base_href: str | None,
+    url_rewriter: ArticleUrlRewriter,
+):
+    """Rewrite youtube and vimeo iframes to remove player until video is included"""
+    if tag not in ["iframe"]:
+        return
+    if not isinstance(url_rewriter, HtmlUrlsRewriter):
+        raise Exception("Expecting HtmlUrlsRewriter")
+    src = get_attr_value_from(attrs=attrs, name="src")
+    if not src:
+        raise UnsupportedTagError("Unsupported empty src in iframe")
+    image_rewriten_url = None
+    try:
+        if ytb_match := YOUTUBE_IFRAME_RE.match(src):
+            rewrite_result = url_rewriter(
+                f'https://i.ytimg.com/vi/{ytb_match.group("id")}/hqdefault.jpg',
+                base_href=base_href,
+            )
+            url_rewriter.add_item_to_download(rewrite_result)
+            image_rewriten_url = rewrite_result.rewriten_url
+        if VIMEO_IFRAME_RE.match(src):
+            rewrite_result = url_rewriter(
+                get_vimeo_thumbnail_url(src),
+                base_href=base_href,
+            )
+            url_rewriter.add_item_to_download(rewrite_result)
+            image_rewriten_url = rewrite_result.rewriten_url
+    except Exception as exc:
+        logger.warning(f"Failed to rewrite iframe with src {src}", exc_info=exc)
+        return (
+            f'<a href="{src}" target="_blank">'
+            f"<div>"
+            f"{src}"
+            "</div>"
+            "</a>"
+            '<iframe style="display: none;">'  # fake opening tag just to remove iframe
+        )
+
+    if image_rewriten_url:
+        return (
+            f'<a href="{src}" target="_blank">'
+            f'<div class="zim-removed-video">'
+            f'<img src="content/{image_rewriten_url}">'
+            "</img>"
+            "</div>"
+            "</a>"
+            '<iframe style="display: none;">'  # fake opening tag just to remove iframe
+        )
+
+    raise UnsupportedTagError(f"Unsupported src {src} in iframe")
+
+
 class HtmlUrlsRewriter(ArticleUrlRewriter):
     """A rewriter for HTML processing
 
diff --git a/scraper/src/mindtouch2zim/vimeo.py b/scraper/src/mindtouch2zim/vimeo.py
new file mode 100644
index 0000000..2776fbd
--- /dev/null
+++ b/scraper/src/mindtouch2zim/vimeo.py
@@ -0,0 +1,30 @@
+import requests
+
+from mindtouch2zim.constants import (
+    HTTP_TIMEOUT_NORMAL_SECONDS,
+    logger,
+)
+
+
+class VimeoThumbnailError(Exception):
+    """Error raised when there is a problem with a vimeo video"""
+
+    pass
+
+
+def get_vimeo_thumbnail_url(video_url: str) -> str:
+    """From a vimeo URL - player or normal - retrieve corresponding thumbnail URL"""
+    resp = requests.get(
+        f"https://vimeo.com/api/oembed.json?url={video_url}",
+        timeout=HTTP_TIMEOUT_NORMAL_SECONDS,
+    )
+    resp.raise_for_status()
+    json_doc = resp.json()
+    if "thumbnail_url" not in json_doc:
+        logger.warning(f"Failed to find thumbnail_url in response:\n{resp.text}")
+        raise VimeoThumbnailError("API response misses the thumbnail_url")
+    thumbnail_url = json_doc["thumbnail_url"]
+    if not thumbnail_url:
+        logger.warning(f"Emtpy thumbnail_url in response:\n{resp.text}")
+        raise VimeoThumbnailError("API response has empty thumbnail_url")
+    return thumbnail_url
diff --git a/zimui/public/custom.css b/zimui/public/custom.css
index d57f80f..ad075b4 100644
--- a/zimui/public/custom.css
+++ b/zimui/public/custom.css
@@ -13,3 +13,29 @@ a[href^="https://"]:after
   bottom: 0px;
   right: 0px;
 }
+
+.zim-removed-video {
+  position: relative;
+  display: inline-block;
+  width: 100%;
+  height: auto;
+}
+
+.zim-removed-video::before {
+  content: '';
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  background-image: url('play-button.svg');
+  background-repeat: no-repeat;
+  background-position: center;
+  background-size: 30%;
+}
+
+.zim-removed-video img {
+  width: 100%;
+  height: auto;
+  display: block;
+}
diff --git a/zimui/public/play-button.svg b/zimui/public/play-button.svg
new file mode 100644
index 0000000..58c7c3c
--- /dev/null
+++ b/zimui/public/play-button.svg
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<!-- License at retrieval from https://www.svgrepo.com/svg/13672/play-button: CC0-->
+<!-- Adapted to remove background + add color-->
+<svg height="800px" width="800px" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
+	 viewBox="0 0 60 60" xml:space="preserve">
+<g>
+	<path fill="#83878D" d="M45.563,29.174l-22-15c-0.307-0.208-0.703-0.231-1.031-0.058C22.205,14.289,22,14.629,22,15v30
+		c0,0.371,0.205,0.711,0.533,0.884C22.679,45.962,22.84,46,23,46c0.197,0,0.394-0.059,0.563-0.174l22-15
+		C45.836,30.64,46,30.331,46,30S45.836,29.36,45.563,29.174z M24,43.107V16.893L43.225,30L24,43.107z"/>
+	<path fill="#83878D" d="M30,0C13.458,0,0,13.458,0,30s13.458,30,30,30s30-13.458,30-30S46.542,0,30,0z M30,58C14.561,58,2,45.439,2,30
+		S14.561,2,30,2s28,12.561,28,28S45.439,58,30,58z"/>
+</g>
+</svg>

From 52559e6c2df7dfc9eedbbaf0c635058ad1c981bc Mon Sep 17 00:00:00 2001
From: benoit74 <benoit74@users.noreply.github.com>
Date: Mon, 28 Oct 2024 10:21:17 +0000
Subject: [PATCH 4/4] Always indicate iframe link when we do not know how to
 fetch image placeholder

---
 scraper/pyproject.toml                 |  2 +-
 scraper/src/mindtouch2zim/processor.py | 23 ++++++++++++++---------
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/scraper/pyproject.toml b/scraper/pyproject.toml
index 801877d..28b5e54 100644
--- a/scraper/pyproject.toml
+++ b/scraper/pyproject.toml
@@ -11,7 +11,7 @@ dependencies = [
   "yt-dlp",                      # youtube-dl should be updated as frequently as possible
   "jinja2==3.1.4",
   # use zimscraperlib pinned version once content rewriting functions have been released
-  "zimscraperlib @ git+https://github.com/openzim/python-scraperlib@mindtouch_changes",
+  "zimscraperlib @ git+https://github.com/openzim/python-scraperlib@main",
   "requests==2.32.3",
   "types-requests==2.32.0.20240914",
   "kiwixstorage==0.9.0",
diff --git a/scraper/src/mindtouch2zim/processor.py b/scraper/src/mindtouch2zim/processor.py
index 544811f..37d94e9 100644
--- a/scraper/src/mindtouch2zim/processor.py
+++ b/scraper/src/mindtouch2zim/processor.py
@@ -653,23 +653,17 @@ def rewrite_iframe_tags(
             )
             url_rewriter.add_item_to_download(rewrite_result)
             image_rewriten_url = rewrite_result.rewriten_url
-        if VIMEO_IFRAME_RE.match(src):
+        elif VIMEO_IFRAME_RE.match(src):
             rewrite_result = url_rewriter(
                 get_vimeo_thumbnail_url(src),
                 base_href=base_href,
             )
             url_rewriter.add_item_to_download(rewrite_result)
             image_rewriten_url = rewrite_result.rewriten_url
+        else:
+            logger.debug(f"iframe pointing to {src} will not have any preview")
     except Exception as exc:
         logger.warning(f"Failed to rewrite iframe with src {src}", exc_info=exc)
-        return (
-            f'<a href="{src}" target="_blank">'
-            f"<div>"
-            f"{src}"
-            "</div>"
-            "</a>"
-            '<iframe style="display: none;">'  # fake opening tag just to remove iframe
-        )
 
     if image_rewriten_url:
         return (
@@ -681,6 +675,17 @@ def rewrite_iframe_tags(
             "</a>"
             '<iframe style="display: none;">'  # fake opening tag just to remove iframe
         )
+    else:
+        # replace iframe with text indicating the online URL which has not been ZIMed
+        return (
+            f"This content is not inside the ZIM. "
+            f'View content online at <a href="{src}" target="_blank">'
+            f"<div>"
+            f"{src}"
+            "</div>"
+            "</a>"
+            '<iframe style="display: none;">'  # fake opening tag just to remove iframe
+        )
 
     raise UnsupportedTagError(f"Unsupported src {src} in iframe")