fix(api): Tree images passed to Generators as AsyncGenerator
This commit is contained in:
@@ -1,5 +1,4 @@
|
|||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from typing import Generator, List
|
|
||||||
|
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from ebooklib import epub
|
from ebooklib import epub
|
||||||
@@ -12,9 +11,9 @@ class EPUBGenerator(AbstractGenerator):
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
metadata: Story,
|
metadata: Story,
|
||||||
part_trees: List[BeautifulSoup],
|
part_trees: list[BeautifulSoup],
|
||||||
cover: bytes,
|
cover: bytes,
|
||||||
images: List[Generator[bytes]] | None,
|
images: list[list[bytes | None]],
|
||||||
):
|
):
|
||||||
self.story = metadata
|
self.story = metadata
|
||||||
self.parts = part_trees
|
self.parts = part_trees
|
||||||
|
|||||||
@@ -2,9 +2,8 @@ from base64 import b64encode
|
|||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from tempfile import NamedTemporaryFile, _TemporaryFileWrapper
|
from tempfile import NamedTemporaryFile, _TemporaryFileWrapper
|
||||||
from typing import Generator, List, cast
|
|
||||||
|
|
||||||
from bs4 import BeautifulSoup, Tag
|
from bs4 import BeautifulSoup
|
||||||
from exiftool import ExifTool
|
from exiftool import ExifTool
|
||||||
from jinja2 import Template
|
from jinja2 import Template
|
||||||
from weasyprint import CSS, HTML
|
from weasyprint import CSS, HTML
|
||||||
@@ -87,9 +86,9 @@ class PDFGenerator(AbstractGenerator):
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
metadata: Story,
|
metadata: Story,
|
||||||
part_trees: List[BeautifulSoup],
|
part_trees: list[BeautifulSoup],
|
||||||
cover: bytes,
|
cover: bytes,
|
||||||
images: List[Generator[bytes]] | None,
|
images: list[list[bytes | None]],
|
||||||
author_image: bytes,
|
author_image: bytes,
|
||||||
):
|
):
|
||||||
self.story = metadata
|
self.story = metadata
|
||||||
@@ -109,6 +108,9 @@ class PDFGenerator(AbstractGenerator):
|
|||||||
for img_idx, (img_data, img_tag) in enumerate(
|
for img_idx, (img_data, img_tag) in enumerate(
|
||||||
zip(self.images[idx], tree.find_all("img"))
|
zip(self.images[idx], tree.find_all("img"))
|
||||||
):
|
):
|
||||||
|
if not img_data:
|
||||||
|
continue
|
||||||
|
|
||||||
img_tag["src"] = (
|
img_tag["src"] = (
|
||||||
f"data:image/jpg;base64,{b64encode(img_data).decode()}"
|
f"data:image/jpg;base64,{b64encode(img_data).decode()}"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from tempfile import _TemporaryFileWrapper
|
from tempfile import _TemporaryFileWrapper
|
||||||
from typing import Generator, List, Literal
|
from typing import Literal
|
||||||
|
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from ebooklib.epub import EpubBook
|
from ebooklib.epub import EpubBook
|
||||||
@@ -15,15 +15,15 @@ class AbstractGenerator:
|
|||||||
metadata (Story): Story Metadata.
|
metadata (Story): Story Metadata.
|
||||||
part_trees (List[BeautifulSoup]): Parsed part trees.
|
part_trees (List[BeautifulSoup]): Parsed part trees.
|
||||||
cover (bytes): Cover image.
|
cover (bytes): Cover image.
|
||||||
images (List[List[bytes]] | None): An array of images for each chapter, if images have been downloaded.
|
images (List[List[bytes | None]]): An array of images for each chapter, if images have been downloaded.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
metadata: Story,
|
metadata: Story,
|
||||||
part_trees: List[BeautifulSoup],
|
part_trees: list[BeautifulSoup],
|
||||||
cover: bytes,
|
cover: bytes,
|
||||||
images: List[Generator[bytes]] | None,
|
images: list[list[bytes | None]],
|
||||||
):
|
):
|
||||||
self.story = metadata
|
self.story = metadata
|
||||||
self.parts = part_trees
|
self.parts = part_trees
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
from itertools import batched, chain
|
from itertools import batched
|
||||||
from typing import Generator, List, Tuple, cast
|
from typing import cast
|
||||||
|
|
||||||
from aiohttp import ClientSession
|
from aiohttp import ClientSession
|
||||||
from bs4 import BeautifulSoup, Tag
|
from bs4 import BeautifulSoup, Tag
|
||||||
@@ -74,11 +74,13 @@ async def fetch_image(url: str) -> bytes | None:
|
|||||||
return body
|
return body
|
||||||
|
|
||||||
|
|
||||||
async def download_tree_images(tree: BeautifulSoup) -> Generator[bytes]:
|
async def fetch_tree_images(tree: BeautifulSoup):
|
||||||
"""Return a Generator of bytes containing image data for all images referenced in the tree."""
|
"""Return a Generator of bytes containing image data for all images referenced in the tree."""
|
||||||
image_urls = [img["src"] for img in tree.find_all("img")]
|
image_urls = [img["src"] for img in tree.find_all("img")]
|
||||||
downloaded_images: Generator[bytes] = chain(
|
|
||||||
await asyncio.gather(*[fetch_image(url) for url in chunk])
|
images = []
|
||||||
for chunk in batched(image_urls, 3)
|
for chunk in batched(image_urls, 3):
|
||||||
)
|
for image_data in await asyncio.gather(*[fetch_image(url) for url in chunk]):
|
||||||
return downloaded_images
|
images.append(image_data)
|
||||||
|
|
||||||
|
return images
|
||||||
|
|||||||
+6
-3
@@ -31,7 +31,7 @@ from create_book import (
|
|||||||
logger,
|
logger,
|
||||||
slugify,
|
slugify,
|
||||||
)
|
)
|
||||||
from create_book.parser import clean_tree, download_tree_images
|
from create_book.parser import clean_tree, fetch_tree_images
|
||||||
|
|
||||||
app = FastAPI()
|
app = FastAPI()
|
||||||
BUILD_PATH = Path(__file__).parent / "build"
|
BUILD_PATH = Path(__file__).parent / "build"
|
||||||
@@ -181,8 +181,11 @@ async def handle_download(
|
|||||||
for part in metadata["parts"]
|
for part in metadata["parts"]
|
||||||
]
|
]
|
||||||
|
|
||||||
# download_images:
|
images = (
|
||||||
images = [await download_tree_images(tree) for tree in part_trees] if download_images else None
|
[await fetch_tree_images(tree) for tree in part_trees]
|
||||||
|
if download_images
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
|
||||||
match format:
|
match format:
|
||||||
case DownloadFormat.epub:
|
case DownloadFormat.epub:
|
||||||
|
|||||||
Reference in New Issue
Block a user