#!/usr/bin/env python3 # maravento.com """ gitfolder.py ------------ Downloads a specific folder or file from a public GitHub repository using the GitHub API. Supports recursive download of subfolders and preserves the original directory structure locally. Usage: python3 gitfolder.py Example: python3 gitfolder.py https://github.com/maravento/vault/project_name """ import requests import os from urllib.parse import urlparse import sys GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "") REQUEST_TIMEOUT = 15 HEADERS = { "User-Agent": "gitfolder/1.0", } if GITHUB_TOKEN: HEADERS["Authorization"] = f"token {GITHUB_TOKEN}" def sanitize_path_component(name): """Sanitize a path component to prevent path traversal attacks.""" name = os.path.normpath(name).lstrip(os.sep) if name.startswith(".."): return "" return name def sanitize_output_dir(path): """Sanitize an output directory path to prevent path traversal attacks.""" parts = [sanitize_path_component(p) for p in path.split("/") if p and p != ".."] return os.path.join(*parts) if parts else "." def fetch_all_pages(url): """Fetch every page of a GitHub Contents API listing (max 1000 entries per page), following the Link: rel="next" header. Returns (items, ok).""" items = [] next_url = url while next_url: try: response = requests.get(next_url, headers=HEADERS, timeout=REQUEST_TIMEOUT) except requests.exceptions.RequestException as e: print(f"Request error: {next_url} → {e}") return items, False if response.status_code != 200: print(f"Failed to retrieve folder contents [{response.status_code}]: {next_url}") return items, False page = response.json() if isinstance(page, dict): # single-file response, not a paginated listing return page, True items.extend(page) next_url = response.links.get("next", {}).get("url") return items, True def download_folder_from_github(repo_owner, repo_name, folder_path, output_dir, branch=""): ref_param = f"?ref={branch}" if branch else "" url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/contents/{folder_path}{ref_param}" contents, fetch_ok = fetch_all_pages(url) if not fetch_ok and not contents: return False os.makedirs(output_dir, exist_ok=True) if isinstance(contents, dict) and contents.get("type") == "file": return download_item(contents, output_dir) and fetch_ok ok = fetch_ok for item in contents: if item["type"] == "file": ok = download_item(item, output_dir) and ok elif item["type"] == "dir": safe_name = sanitize_path_component(item["name"]) if not safe_name: print(f"Skipped unsafe directory name: {item['name']!r}") ok = False continue subfolder_path = folder_path + '/' + item["name"] subfolder_output_dir = os.path.join(output_dir, safe_name) ok = download_folder_from_github(repo_owner, repo_name, subfolder_path, subfolder_output_dir, branch) and ok else: print(f"Skipped unsupported type '{item['type']}': {item['name']}") ok = False return ok def download_item(item, output_dir): file_url = item.get("download_url") if not file_url: print(f"Skipped (no download_url): {item.get('name', '?')}") return False safe_name = sanitize_path_component(item["name"]) if not safe_name: print(f"Skipped unsafe filename: {item.get('name', '?')!r}") return False try: response = requests.get(file_url, headers=HEADERS, timeout=REQUEST_TIMEOUT) except requests.exceptions.RequestException as e: print(f"Request error: {file_url} → {e}") return False if response.status_code != 200: print(f"Failed to download file [{response.status_code}]: {file_url}") return False os.makedirs(output_dir, exist_ok=True) file_path = os.path.join(output_dir, safe_name) tmp_path = file_path + ".part" with open(tmp_path, "wb") as file: file.write(response.content) os.replace(tmp_path, file_path) print(f"Downloaded file: {file_path}") return True if __name__ == "__main__": if len(sys.argv) < 2: print("Usage: python3 gitfolder.py ") sys.exit(1) url = sys.argv[1] parsed_url = urlparse(url) path_parts = parsed_url.path.strip("/").split("/") if len(path_parts) < 2: print("Error: URL must include at least owner and repository name.") sys.exit(1) repo_owner = path_parts[0] repo_name = path_parts[1] branch = "" if len(path_parts) < 3: folder_path = "" output_dir = repo_name elif path_parts[2] in ("tree", "blob"): if len(path_parts) < 4: folder_path = "" output_dir = repo_name elif len(path_parts) < 5: branch = path_parts[3] folder_path = "" output_dir = repo_name else: branch = path_parts[3] folder_path = "/".join(path_parts[4:]) output_dir = sanitize_output_dir(folder_path) else: folder_path = "/".join(path_parts[2:]) output_dir = sanitize_output_dir(folder_path) print(f""" Owner: {repo_owner} Repository: {repo_name} Branch: {branch or "(default)"} Directory: {folder_path or "(root)"} """) if not download_folder_from_github(repo_owner, repo_name, folder_path, output_dir, branch): sys.exit(1)