Move scraper to subfolder and initialize new Vue.js project

.github/workflows/Publish.yml

@@ -16,18 +16,19 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v4
  with:
- python-version-file: pyproject.toml
+ python-version-file: scraper/pyproject.toml
  architecture: x64
 
  - name: Build packages
+ working-directory: scraper
  run: |
  pip install -U pip build
  python -m build --sdist --wheel
 
  - name: Upload to PyPI
  uses: pypa/gh-action-pypi-publish@release/v1.8
  with:
- packages-dir: dist/
+ packages-dir: scraper/dist/
 
  - name: Build and push Docker image
  uses: openzim/docker-publish-action@v10

.github/workflows/QA.yml

@@ -16,19 +16,47 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v4
  with:
- python-version-file: pyproject.toml
+ python-version-file: scraper/pyproject.toml
  architecture: x64
 
  - name: Install dependencies
+ working-directory: scraper
  run: |
  pip install -U pip
  pip install -e .[lint,check,scripts,test]
 
  - name: Check black formatting
+ working-directory: scraper
  run: inv lint-black
 
  - name: Check ruff
+ working-directory: scraper
  run: inv lint-ruff
 
  - name: Check pyright
- run: inv check-pyright
+ working-directory: scraper
+ run: inv check-pyright
+
+ check-zimui-qa:
+ runs-on: ubuntu-22.04
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set up Node.js
+ uses: actions/setup-node@v3
+ with:
+ node-version-file: zimui/.node-version
+
+ - name: Install JS dependencies
+ working-directory: zimui
+ run: |
+ yarn install
+ - name: Check prettier
+ working-directory: zimui
+ run: |
+ yarn format
+ - name: Check eslint
+ working-directory: zimui
+ run: |
+ yarn lint

.github/workflows/Tests.yml

@@ -16,15 +16,17 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v4
  with:
- python-version-file: pyproject.toml
+ python-version-file: scraper/pyproject.toml
  architecture: x64
 
  - name: Install dependencies (and project)
+ working-directory: scraper
  run: |
  pip install -U pip
  pip install -e .[test,scripts]
 
  - name: Run the tests
+ working-directory: scraper
  run: inv coverage --args "-vvv"
 
  - name: Upload coverage report to codecov
@@ -40,14 +42,34 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v4
  with:
- python-version-file: pyproject.toml
+ python-version-file: scraper/pyproject.toml
  architecture: x64
 
  - name: Ensure we can build Python targets
+ working-directory: scraper
  run: |
  pip install -U pip build
  python3 -m build --sdist --wheel
 
+ build-zimui:
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set up Node.js
+ uses: actions/setup-node@v3
+ with:
+ node-version-file: zimui/.node-version
+
+ - name: Install dependencies
+ working-directory: zimui
+ run: |
+ yarn install
+ - name: Build
+ working-directory: zimui
+ run: |
+ yarn build
+
  build-docker:
  runs-on: ubuntu-22.04
  steps:

.gitignore

@@ -366,13 +366,13 @@ pyrightconfig.json
 # End of https://www.toptal.com/developers/gitignore/api/linux,macos,python,database,visualstudiocode,intellij
 
 # JS deps
-src/youtube2zim/templates/assets/chosen/
-src/youtube2zim/templates/assets/jquery.min.js
-src/youtube2zim/templates/assets/ogvjs/
-src/youtube2zim/templates/assets/videojs-ogvjs.js
-src/youtube2zim/templates/assets/videojs/
-src/youtube2zim/templates/assets/polyfills.js
-src/youtube2zim/templates/assets/webp-hero.bundle.js
+scraper/src/youtube2zim/templates/assets/chosen/
+scraper/src/youtube2zim/templates/assets/jquery.min.js
+scraper/src/youtube2zim/templates/assets/ogvjs/
+scraper/src/youtube2zim/templates/assets/videojs-ogvjs.js
+scraper/src/youtube2zim/templates/assets/videojs/
+scraper/src/youtube2zim/templates/assets/polyfills.js
+scraper/src/youtube2zim/templates/assets/webp-hero.bundle.js
 
 # output dir
 output

.pre-commit-config.yaml

@@ -6,6 +6,22 @@ repos:
  hooks:
  - id: trailing-whitespace
  - id: end-of-file-fixer
+- repo: https:/pre-commit/mirrors-prettier
+ rev: v3.0.3
+ hooks:
+ - id: prettier
+ files: zimui\/.*$ # files in zimui folder
+- repo: https:/pre-commit/mirrors-eslint
+ rev: v8.51.0
+ hooks:
+ - id: eslint
+ types: [file]
+ files: zimui\/src\/.*(?:\.[jt]sx?|\.vue)$ # *.js, *.jsx, *.ts, *.tsx, *.vue in zimui/src folder
+ args:
+ - --ignore-path
+ - zimui/.eslintignore
+ - --config
+ - zimui/.eslintrc.cjs
 - repo: https:/psf/black
  rev: "24.4.2"
  hooks:

CHANGELOG

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Move scraper files to `scraper` subfolder and update workflows
+- Bump `requests` package from 2.32.0 to 2.32.2
+- Initialize new Vue.js project in `zimui` subfolder
+
 ## [2.3.0] - 2024-05-22
 
 ### Added

Dockerfile

@@ -1,3 +1,10 @@
+FROM node:20-alpine as zimui
+
+WORKDIR /src
+COPY zimui /src
+RUN yarn install --frozen-lockfile
+RUN yarn build
+
 FROM python:3.12-bookworm
 LABEL org.opencontainers.image.source https:/openzim/youtube
 
@@ -14,24 +21,30 @@ RUN apt-get update \
  pip
 
 # Custom entrypoint
-COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+COPY scraper/entrypoint.sh /usr/local/bin/entrypoint.sh
 ENTRYPOINT ["entrypoint.sh"]
 RUN mkdir -p /output
 WORKDIR /output
 
 # Copy pyproject.toml and its dependencies
-COPY pyproject.toml README.md openzim.toml /src/
-COPY src/youtube2zim/__about__.py /src/src/youtube2zim/__about__.py
+COPY README.md /src/
+COPY scraper/pyproject.toml scraper/openzim.toml /src/scraper/
+COPY scraper/src/youtube2zim/__about__.py /src/scraper/src/youtube2zim/__about__.py
 
 # Install Python dependencies
-RUN pip install --no-cache-dir /src
+RUN pip install --no-cache-dir /src/scraper
 
 # Copy code + associated artifacts
-COPY src /src/src
+COPY scraper/src /src/scraper/src
 COPY *.md LICENSE CHANGELOG /src/
 
 # Install + cleanup
-RUN pip install --no-cache-dir /src \
- && rm -rf /src
+RUN pip install --no-cache-dir /src/scraper \
+ && rm -rf /src/scraper
+
+# Copy zimui build output
+COPY --from=zimui /src/dist /src/zimui
+
+ENV KOLIBRI_ZIMUI_DIST=/src/zimui
 
 CMD ["youtube2zim", "--help"]

README.md

@@ -50,6 +50,7 @@ pip3 install hatch
 Start a hatch shell: this will install software including dependencies in an isolated virtual environment.
 
 ``` bash
+cd scraper
 hatch shell
 ```
 

pyproject.toml → scraper/pyproject.toml

@@ -6,14 +6,14 @@ build-backend = "hatchling.build"
 name = "youtube2zim"
 requires-python = ">=3.12,<3.13"
 description = "Make ZIM file from a Youtube channel, user or playlist(s)"
-readme = "README.md"
+readme = "../README.md"
 dependencies = [
  "python-slugify==3.0.3",
  "yt-dlp", # youtube-dl should be updated as frequently as possible
  "python-dateutil==2.9.0.post0",
  "jinja2==3.1.4",
  "zimscraperlib==3.3.2",
- "requests==2.32.0",
+ "requests==2.32.2",
  "kiwixstorage==0.8.3",
  "pif==0.8.2",
 ]

src/youtube2zim/entrypoint.py → scraper/src/youtube2zim/entrypoint.py

@@ -3,6 +3,7 @@
 
 import argparse
 import logging
+import os
 import sys
 
 from youtube2zim.constants import CHANNEL, NAME, PLAYLIST, SCRAPER, USER, logger
@@ -86,6 +87,15 @@ def main():
  default=False,
  )
 
+ parser.add_argument(
+ "--zimui-dist",
+ type=str,
+ help=(
+ "Directory containing Vite build output from the Zim UI Vue.JS application"
+ ),
+ default=os.getenv("KOLIBRI_ZIMUI_DIST", "../zimui/dist"),
+ )
+
  parser.add_argument(
  "--zim-file",
  help="ZIM file name (based on --name if not provided). "

src/youtube2zim/locale/fr/LC_MESSAGES/messages.po → scraper/src/youtube2zim/locale/fr/LC_MESSAGES/messages.po

@@ -33,4 +33,3 @@ msgstr "Page {current}/{total}"
 #: youtube2zim/scraper.py:872
 msgid "Back to top"
 msgstr "Haut"
-

src/youtube2zim/playlists/scraper.py → scraper/src/youtube2zim/playlists/scraper.py

@@ -198,7 +198,7 @@ def handle_single_zim(self):
 
  @staticmethod
  def compute_format(playlist, fmt):
- return fmt.format(**playlist.__dict__(), **{"period": "{period}"})
+ return fmt.format(**playlist.to_dict(), **{"period": "{period}"})
 
  def fetch_metadata(self):
  """retrieves and loads metadata from --metadata-from"""