bamboo-cms · YogiLiu · Dec 27, 2023 · Dec 28, 2023 · Dec 28, 2023 · Dec 28, 2023
diff --git a/.env.template b/.env.template
@@ -18,5 +18,10 @@ DATABASE_NAME=bamboo # database name
 # REDIS
 REDIS_PORT=6379
 
+
 # WEB
 PORT=8000
+
+# SSG
+SSG_SYNC_INTERVAL=3 # in minutes, optional, default 3 minutes
+SSG_GH_TOKEN=github_token # github token, optional, default empty
diff --git a/backend/bamboo/__init__.py b/backend/bamboo/__init__.py
@@ -1,8 +1,11 @@
 from apiflask import APIFlask
 from flask import redirect, url_for
 
-from bamboo import blueprints, database
+from bamboo import blueprints, database, jobs
 from bamboo.settings import config
+from bamboo.ssg import SSG
+
+ssg = SSG()
 
 
 def create_app(config_name: str) -> APIFlask:
@@ -13,6 +16,10 @@ def create_app(config_name: str) -> APIFlask:
     blueprints.init_app(app)
     # database
     database.init_app(app)
+    # jobs
+    jobs.init_app(app)
+    # SSG
+    ssg.init_app(app)
 
     # TODO: direct it to the dashboard when it's ready.
     @app.get("/")

diff --git a/backend/bamboo/blueprints/__init__.py b/backend/bamboo/blueprints/__init__.py
@@ -6,6 +6,7 @@
 from bamboo.blueprints.command import command
 from bamboo.blueprints.error import error
 from bamboo.blueprints.page import page
+from bamboo.blueprints.ssg import ssg
 from bamboo.blueprints.talk import talk
 
 
@@ -17,3 +18,4 @@ def init_app(app: APIFlask) -> None:
     app.register_blueprint(blog, url_prefix="/blog")
     app.register_blueprint(page, url_prefix="/page")
     app.register_blueprint(talk, url_prefix="/talk")
+    app.register_blueprint(ssg, url_prefix="/ssg")
diff --git a/backend/bamboo/blueprints/ssg.py b/backend/bamboo/blueprints/ssg.py
@@ -0,0 +1,17 @@
+from apiflask import APIBlueprint
+from flask import current_app
+
+from bamboo.database import db
+from bamboo.database.models import Site
+from bamboo.ssg import SSG
+
+ssg = APIBlueprint("ssg", __name__)
+
+
+@ssg.get("/<int:site_id>/<path:file>")
+def render(site_id: int, file: str):
+    site = db.session.get(Site, site_id)
+    if not site:
+        return "Site not found.", 404
+    generator: SSG = current_app.extensions["ssg"]
+    return generator.render_page(site, file)
diff --git a/backend/bamboo/jobs.py b/backend/bamboo/jobs.py
@@ -0,0 +1,84 @@
+import logging
+import re
+import shutil
+import tempfile
+import zipfile
+from datetime import timedelta
+from pathlib import Path
+from typing import Optional
+
+import httpx
+from flask import Flask
+from flask_rq2 import RQ
+
+from bamboo.database.models import Site
+
+rq = RQ()
+logger = logging.getLogger(__name__)
+
+gh_pattern = re.compile(r"https://github\.com/(?P<owner>\S+)/(?P<repo>\S+)")
+
+
+@rq.job
+def sync_templates(store_dir: Path, sync_interval: timedelta, gh_token: Optional[str] = None):
+    logger.info("Syncing templates")
+    for site in Site.query.all():
+        if not site.config.get("sync") or not site.template_url:
+            continue
+        elif site.template_url.startswith("https://github.com"):
+            name = f"{site.id}_{site.name}"
+            fetch_template(name, site.template_url, store_dir, gh_token)
+        else:
+            logger.warning(f"Unsupported template URL: {site.template_url}, skipping.")
+    sync_templates.schedule(sync_interval)
+
+
+@rq.job
+def fetch_template(name: str, url: str, store_dir: Path, gh_token: Optional[str] = None):
+    logger.info(f"Fetching {name} from {url}")
+    match = gh_pattern.match(url)
+    if match is None or "owner" not in match.groupdict() or "repo" not in match.groupdict():
+        logger.warning(f"Invalid GitHub URL: {url}, skipping.")
+        return
+    owner = match.group("owner")
+    repo = match.group("repo")
+    file_url = f"https://api.github.com/repos/{owner}/{repo}/zipball"
+    headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}
+    if gh_token:
+        headers["Authorization"] = f"Bearer {gh_token}"
+    with httpx.Client() as client:
+        try:
+            res = client.get(file_url, headers=headers, follow_redirects=True, timeout=30)
+            res.raise_for_status()
+        except httpx.HTTPError:
+            logger.warning(f"Failed to fetch {file_url}, skipping.")
+            return
+    with tempfile.NamedTemporaryFile(suffix=".zip") as tmp:
+        chunk_size = 128 * 1024  # 128 KB
+        for chunk in res.iter_bytes(chunk_size):
+            tmp.write(chunk)
+        tmp.seek(0)
+        try:
+            zip_file = zipfile.ZipFile(tmp)
+        except zipfile.BadZipFile:
+            logger.warning(f"Bad zip file: {file_url}, skipping.")
+            return
+        if zip_file.testzip() is not None:
+            logger.warning(f"Can not read zip file: {file_url}, skipping.")
+            return
+        if len(zip_file.namelist()) == 0:
+            logger.warning(f"Empty zip file: {file_url}, skipping.")
+            return
+        dir_name = zip_file.namelist()[0].split("/")[0]
+        dest = store_dir / name
+        if dest.exists():
+            shutil.rmtree(dest)
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            zip_file.extractall(tmp_dir)
+            shutil.copytree(Path(tmp_dir) / dir_name, dest)
+        zip_file.close()
+        logger.info(f"Stored {name} at {dest.absolute()}")
+
+
+def init_app(app: Flask) -> None:
+    rq.init_app(app)
diff --git a/backend/bamboo/settings.py b/backend/bamboo/settings.py
@@ -11,6 +11,8 @@
 
 class BaseConfig:
     SECRET_KEY = os.getenv("SECRET_KEY", "dev key")
+    SSG_SYNC_INTERVAL = os.getenv("SSG_SYNC_INTERVAL", "3")
+    SSG_GH_TOKEN = os.getenv("SSG_GH_TOKEN", "")
 
 
 class DevelopmentConfig(BaseConfig):
@@ -20,6 +22,7 @@ class DevelopmentConfig(BaseConfig):
 class TestingConfig(BaseConfig):
     TESTING = True
     SQLALCHEMY_DATABASE_URI = "sqlite:///:memory:"  # in-memory database
+    RQ_CONNECTION_CLASS = "fakeredis.FakeStrictRedis"
 
 
 class ProductionConfig(BaseConfig):

diff --git a/backend/bamboo/ssg.py b/backend/bamboo/ssg.py
@@ -0,0 +1,157 @@
+import logging
+import tempfile
+import threading
+import zipfile
+from contextlib import contextmanager
+from datetime import timedelta
+from pathlib import Path
+from typing import Generator, Optional
+
+import jinja2
+from flask import Flask, Response, send_from_directory
+from flask_rq2 import RQ
+
+from bamboo.database.models import Site
+from bamboo.jobs import sync_templates
+
+logger = logging.getLogger(__name__)
+
+
+reserved_dir = "jinja"  # Reserved for Jinja2, can not be accessed from web.
+static_dir = "static"  # JS, CSS, Image, etc. Send to browser directly.
+
+
+class SSG:
+    """
+    Static site generator.
+    """
+
+    fetcher: "Fetcher"
+
+    def __init__(self, app: Optional[Flask] = None):
+        if app is not None:
+            self.init_app(app)
+        self.tpl_dir = (
+            Path(__file__).parent.parent.parent / "data" / "templates"
+        )  # 'data' in root of project
+
+    def init_app(self, app: Flask):
+        if not hasattr(app, "extensions"):
+            app.extensions = {}
+        app.extensions["ssg"] = self
+        sync_interval = app.config.get("SSG_SYNC_INTERVAL", "3")
+        minutes = timedelta(minutes=float(sync_interval))
+        if hasattr(app, "extensions") and "rq2" in app.extensions:
+            rq2: RQ = app.extensions["rq2"]
+        else:
+            raise RuntimeError("RQ is not initialized.")
+        gh_token = app.config.get("SSG_GH_TOKEN")
+        config = {}
+        if gh_token:
+            config["GH_TOKEN"] = gh_token
+        self.fetcher = Fetcher(minutes, rq2, self.tpl_dir, config)
+        if not app.config.get("TESTING"):
+            self.fetcher.schedule()
+
+    def render_page(self, site: Site, path: str) -> Response:
+        """
+        Renders a page.
+
+        :param site: Site to render.
+        :param path: Path to render, it **must be not** startswith "/".
+        """
+        if path.startswith(reserved_dir):
+            return Response(f"Reserved path: {path}", status=403, mimetype="text/plain")
+        elif path.startswith(static_dir):
+            asset_path = Path(f"{site.id}_{site.name}") / path
+            return send_from_directory(self.tpl_dir, asset_path)
+        try:
+            jinja_env = self.fetcher.get_jinja_env(site)
+            template = jinja_env.get_template(path)
+            page = template.render(site=site)
+        except (jinja2.TemplateNotFound, FileNotFoundError) as e:
+            logger.warning(f"Template not found: {e}")
+            return Response(f"Template not found: {e}", status=404, mimetype="text/plain")
+        return Response(page, mimetype="text/html")
+
+    @contextmanager
+    def pack_site(self, site: Site) -> Generator[zipfile.ZipFile, None, None]:
+        """
+        Packs a site into a zip file.
+
+        :param site: Site to pack.
+        :raises jinja2.TemplateNotFound: If template not found.
+        """
+        site_dir = self.tpl_dir / f"{site.id}_{site.name}"
+        if not site_dir.exists():
+            raise FileNotFoundError(f"Site not found: {site.id}_{site.name}")
+        with tempfile.NamedTemporaryFile(suffix=".zip") as tmp:
+            with zipfile.ZipFile(tmp, "w") as zip_file:
+                # render page
+                for tpl_path in site_dir.glob("**/*.html"):
+                    if tpl_path.is_dir():
+                        continue
+                    # ignore reserved dir and static dir
+                    if tpl_path.relative_to(site_dir).parts[0] in (reserved_dir, static_dir):
+                        continue
+                    tpl_rel_path = tpl_path.relative_to(site_dir)
+                    try:
+                        jinja_env = self.fetcher.get_jinja_env(site)
+                        tpl = jinja_env.get_template(str(tpl_rel_path))
+                        page = tpl.render(site=site)
+                    except (jinja2.TemplateNotFound, FileNotFoundError) as e:
+                        logger.error(f"Template not found: {e}")
+                        raise
+                    p = tpl_path.relative_to(site_dir)
+                    zip_file.writestr(str(p), page)
+                # copy static files
+                for static_path in site_dir.glob(f"{static_dir}/**/*"):
+                    if static_path.is_dir():
+                        continue
+                    p = static_path.relative_to(site_dir)
+                    zip_file.write(static_path, str(p))
+                yield zip_file
+
+
+class Fetcher:
+    """
+    Fetches templates from remote source and stores them locally.
+
+    :param sync_interval: Interval between syncs with remote source.
+    :param config: Configuration of the fetcher. For example, GH-TOKEN.
+    """
+
+    def __init__(
+        self,
+        sync_interval: timedelta,
+        rq: RQ,
+        store_dir: Path,
+        config: Optional[dict] = None,
+    ):
+        self.sync_interval = sync_interval
+        self.store_dir = store_dir
+        self.rq = rq
+        self.config = config or {}
+        self._jinja_envs: dict[str, jinja2.Environment] = {}
+        self._jinja_envs_lock = threading.Lock()
+
+    def schedule(self):
+        """
+        Schedules the fetcher.
+        """
+        logger.info(f"Scheduling sync with interval {self.sync_interval}")
+        sync_templates(self.store_dir, self.sync_interval, self.config.get("GH_TOKEN", None))
+
+    def get_jinja_env(self, site: Site) -> jinja2.Environment:
+        name = f"{site.id}_{site.name}"
+        path = self.store_dir / name
+        with self._jinja_envs_lock:
+            if not path.exists():
+                if name in self._jinja_envs:
+                    del self._jinja_envs[name]
+                raise FileNotFoundError(f"Site {site.id} not found.")
+            if name not in self._jinja_envs:
+                self._jinja_envs[name] = jinja2.Environment(
+                    loader=jinja2.FileSystemLoader(str(path))
+                )
+            return self._jinja_envs[name]