chore: include node_modules in api slackware package (#1415)

pujitm · web-flow · commit 184b76de1cc3 · 2025-06-11T12:07:32.000-04:00
instead of vendoring &amp; uploading separately.

## Summary by CodeRabbit

- **Chores**
- Updated build process to retain the `node_modules` directory, removing
compression and archiving steps.
- Improved plugin installation by cleaning up outdated dependency
archives before reinstalling, enhancing system stability.
- Removed vendor store file references and related bundling steps from
the plugin build and installation process.
- Enhanced dependency restoration during service start to log warnings
without aborting on failure.
- Simplified dependency management scripts by removing vendor store URL
handling and download functionality.
- Streamlined build workflows by removing artifact upload/download and
validation steps related to node modules archives.
- Updated Docker Compose configuration to remove unused volume mounts
for node modules archives.
- Added repository cleanup commands to remove top-level `node_modules`
directories and common build artifact folders for easier maintenance.
diff --git a/.github/workflows/build-plugin.yml b/.github/workflows/build-plugin.yml
@@ -100,11 +100,6 @@ jobs:
         with:
           name: unraid-api
           path: ${{ github.workspace }}/plugin/api/
-      - name: Download PNPM Store
-        uses: actions/download-artifact@v4
-        with:
-          name: packed-node-modules
-          path: ${{ github.workspace }}/plugin/node-modules-archive/
       - name: Extract Unraid API
         run: |
           mkdir -p ${{ github.workspace }}/plugin/source/dynamix.unraid.net/usr/local/unraid-api
@@ -129,11 +124,6 @@ jobs:
             exit 1
           fi
 
-          if [ ! -f ./deploy/*.tar.xz ]; then
-            echo "Error: .tar.xz file not found in plugin/deploy/"
-            exit 1
-          fi
-
       - name: Upload to GHA
         uses: actions/upload-artifact@v4
         with:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -185,11 +185,6 @@ jobs:
         with:
           name: unraid-api
           path: ${{ github.workspace }}/api/deploy/unraid-api.tgz
-      - name: Upload Node Modules to Github artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: packed-node-modules
-          path: ${{ github.workspace }}/api/deploy/node-modules-archive/packed-node-modules.tar.xz
 
   build-unraid-ui-webcomponents:
     name: Build Unraid UI Library (Webcomponent Version)
diff --git a/api/scripts/build.ts b/api/scripts/build.ts
@@ -120,13 +120,6 @@ try {
         }
     }
 
-    const compressionLevel = process.env.WATCH_MODE ? '-1' : '-5';
-    await $`XZ_OPT=${compressionLevel} tar -cJf packed-node-modules.tar.xz node_modules`;
-    // Create a subdirectory for the node modules archive
-    await mkdir('../node-modules-archive', { recursive: true });
-    await $`mv packed-node-modules.tar.xz ../node-modules-archive/`;
-    await $`rm -rf node_modules`;
-
     // Clean the release directory
     await $`rm -rf ../release/*`;
 
diff --git a/justfile b/justfile
@@ -15,6 +15,29 @@ setup:
     
     pnpm install
 
+# Removes all node_modules directories in the repository
+clean-modules:
+    #!/usr/bin/env bash
+    echo "Finding and removing all node_modules directories..."
+    find . -name "node_modules" -type d -not -path "*/node_modules/*" | while read dir; do
+        echo "Removing: $dir"
+        rm -rf "$dir"
+    done
+    echo "All node_modules directories have been removed."
+
+# Removes all build artifacts (dist, .nuxt, .output, coverage, etc.)
+clean-build:
+    #!/usr/bin/env bash
+    echo "Finding and removing build artifacts..."
+    build_dirs=("dist" ".nuxt" ".output" "coverage" "deploy")
+    for build_dir in "${build_dirs[@]}"; do
+        find . -name "$build_dir" -type d | while read dir; do
+            echo "Removing: $dir"
+            rm -rf "$dir"
+        done
+    done
+    echo "All build artifacts have been removed."
+
 # restore notification files under api/dev
 restore-notifications:
     git checkout ./api/dev/notifications
diff --git a/plugin/builder/build-plugin.ts b/plugin/builder/build-plugin.ts
@@ -3,7 +3,7 @@ import { $ } from "zx";
 import { escape as escapeHtml } from "html-sloppy-escaper";
 import { dirname, join } from "node:path";
 import { getTxzName, pluginName, startingDir, defaultArch, defaultBuild } from "./utils/consts";
-import { getAssetUrl, getPluginUrl } from "./utils/bucket-urls";
+import { getPluginUrl } from "./utils/bucket-urls";
 import { getMainTxzUrl } from "./utils/bucket-urls";
 import {
   deployDir,
@@ -12,7 +12,6 @@ import {
 } from "./utils/paths";
 import { PluginEnv, setupPluginEnv } from "./cli/setup-plugin-environment";
 import { cleanupPluginFiles } from "./utils/cleanup";
-import { bundleVendorStore, getVendorBundleName } from "./build-vendor-store";
 
 /**
  * Check if git is available
@@ -76,8 +75,6 @@ const buildPlugin = async ({
     txz_url: getMainTxzUrl({ baseUrl, apiVersion, tag }),
     txz_sha256: txzSha256,
     txz_name: getTxzName(apiVersion),
-    vendor_store_url: getAssetUrl({ baseUrl, tag }, getVendorBundleName(apiVersion)),
-    vendor_store_filename: getVendorBundleName(apiVersion),
     ...(tag ? { tag } : {}),
   };
 
@@ -123,7 +120,6 @@ const main = async () => {
 
     await buildPlugin(validatedEnv);
     await moveTxzFile(validatedEnv);
-    await bundleVendorStore(validatedEnv.apiVersion);
   } catch (error) {
     console.error(error);
     process.exit(1);
diff --git a/plugin/builder/build-txz.ts b/plugin/builder/build-txz.ts
@@ -73,7 +73,7 @@ const storeVendorArchiveInfo = async (version: string, vendorUrl: string, vendor
     
     // Create a JSON config file with vendor information
     const configData = {
-      vendor_store_url: vendorUrl,
+      // vendor_store_url: vendorUrl,
       vendor_store_path: vendorFullPath,
       api_version: version
     };
diff --git a/plugin/builder/build-vendor-store.ts b/plugin/builder/build-vendor-store.ts
@@ -54,6 +54,7 @@ async function createNodeModulesTarball(outputPath: string): Promise<void> {
  * After this operation, the vendored node_modules will be available inside the `deployDir`.
  * 
  * @param apiVersion Required API version to use for the vendor bundle
+ * @deprecated vendored node_modules are now included in the API slackware package
  */
 export async function bundleVendorStore(apiVersion: string): Promise<void> {
     // Ensure deploy directory exists
diff --git a/plugin/docker-compose.yml b/plugin/docker-compose.yml
@@ -14,7 +14,6 @@ services:
       - ../unraid-ui/dist-wc:/app/source/dynamix.unraid.net/usr/local/emhttp/plugins/dynamix.my.servers/unraid-components/uui
       - ../web/.nuxt/nuxt-custom-elements/dist/unraid-components:/app/source/dynamix.unraid.net/usr/local/emhttp/plugins/dynamix.my.servers/unraid-components/nuxt
       - ../api/deploy/release/:/app/source/dynamix.unraid.net/usr/local/unraid-api # Use the release dir instead of pack to allow watcher to not try to build with node_modules
-      - ../api/deploy/node-modules-archive:/app/node-modules-archive
     stdin_open: true # equivalent to -i
     tty: true        # equivalent to -t
     environment:
diff --git a/plugin/plugins/dynamix.unraid.net.plg b/plugin/plugins/dynamix.unraid.net.plg
@@ -9,8 +9,6 @@
   <!ENTITY txz_sha256 "">
   <!ENTITY txz_url "">
   <!ENTITY txz_name "">
-  <!ENTITY vendor_store_url "">
-  <!ENTITY vendor_store_filename "">
   <!ENTITY arch "x86_64">
   <!ENTITY build "1">
   <!ENTITY tag "">
@@ -47,10 +45,6 @@ exit 0
 ]]>
     </INLINE>
   </FILE>
-  
-  <FILE Name="/boot/config/plugins/dynamix.my.servers/&vendor_store_filename;">
-    <URL>&vendor_store_url;</URL>
-  </FILE>
 
   <!-- download main txz -->
   <FILE Name="&source;">
@@ -223,7 +217,6 @@ exit 0
       PKG_FILE="&source;"      # Full path to the package file including .txz extension
       PKG_URL="&txz_url;"      # URL where package was downloaded from
       PKG_NAME="&txz_name;"    # Name of the package file
-      VENDOR_ARCHIVE="/boot/config/plugins/dynamix.my.servers/&vendor_store_filename;"
     <![CDATA[
 # Install the Slackware package
 echo "Installing package..."
@@ -235,6 +228,9 @@ for txz_file in /boot/config/plugins/dynamix.my.servers/dynamix.unraid.net-*.txz
   fi
 done
 
+# Clean up any old node_modules archives (on the boot drive) that don't match our current version
+/etc/rc.d/rc.unraid-api cleanup-dependencies
+
 # Remove existing node_modules directory
 echo "Cleaning up existing node_modules directory..."
 if [ -d "/usr/local/unraid-api/node_modules" ]; then
diff --git a/plugin/source/dynamix.unraid.net/etc/rc.d/rc.unraid-api b/plugin/source/dynamix.unraid.net/etc/rc.d/rc.unraid-api
@@ -29,11 +29,10 @@ uninstall() {
 start() {
   echo "Starting Unraid API service..."
   
-  # Ensure dependencies are installed
+  # Restore vendored API plugins if they were installed
   if [ -x "$scripts_dir/dependencies.sh" ]; then
-    "$scripts_dir/dependencies.sh" ensure || {
-      echo "Failed to install dependencies – aborting start."
-      return 1
+    "$scripts_dir/dependencies.sh" restore || {
+      echo "Failed to restore API plugin dependencies! Continuing with start, but API plugins may be unavailable."
     }
   else
     echo "Warning: dependencies.sh script not found or not executable"
diff --git a/plugin/source/dynamix.unraid.net/usr/local/share/dynamix.unraid.net/scripts/api_utils.sh b/plugin/source/dynamix.unraid.net/usr/local/share/dynamix.unraid.net/scripts/api_utils.sh
@@ -28,7 +28,6 @@ get_api_version() {
 get_archive_information() {
   # Define all local variables at the top
   local api_version=""
-  local vendor_store_url=""
   local vendor_store_path=""
   
   if [ ! -f "$CONFIG_FILE" ]; then
@@ -39,7 +38,6 @@ get_archive_information() {
   # Read values from JSON config using jq
   if command -v jq >/dev/null 2>&1; then
     api_version=$(jq -r '.api_version' "$CONFIG_FILE")
-    vendor_store_url=$(jq -r '.vendor_store_url' "$CONFIG_FILE")
     vendor_store_path=$(jq -r '.vendor_store_path' "$CONFIG_FILE")
   else
     echo "jq not found, can't parse config file" >&2
@@ -52,19 +50,13 @@ get_archive_information() {
     return 1
   fi
   
-  if [ -z "$vendor_store_url" ] || [ "$vendor_store_url" = "null" ]; then
-    echo "Invalid or missing vendor_store_url in config file" >&2
-    return 1
-  fi
-  
   if [ -z "$vendor_store_path" ] || [ "$vendor_store_path" = "null" ]; then
     echo "Invalid or missing vendor_store_path in config file" >&2
     return 1
   fi
   
   # Return the values
   echo "$api_version"
-  echo "$vendor_store_url"
   echo "$vendor_store_path"
   return 0
 } 
diff --git a/plugin/source/dynamix.unraid.net/usr/local/share/dynamix.unraid.net/scripts/dependencies.sh b/plugin/source/dynamix.unraid.net/usr/local/share/dynamix.unraid.net/scripts/dependencies.sh
@@ -26,7 +26,7 @@ cleanup() {
   fi
   
   api_version="${info[0]}"
-  vendor_store_path="${info[2]}"
+  vendor_store_path="${info[1]}"
   
   echo "Cleaning up node_modules archives that don't match current API version: $api_version"
   
@@ -62,48 +62,16 @@ cleanup() {
 # Args:
 #   $1 - Path to vendor archive to download (ignored, kept for backward compatibility)
 redownload_vendor_archive() {
-  # Define all local variables at the top
-  local info
-  local api_version=""
-  local vendor_store_url=""
-  local vendor_store_path=""
-  
-  # Get archive information
-  if ! mapfile -t info < <(get_archive_information); then
-    echo "Error: Failed to get vendor archive information. Cannot proceed." >&2
-    return 1
-  fi
-  
-  api_version="${info[0]}"
-  vendor_store_url="${info[1]}"
-  vendor_store_path="${info[2]}"
-  
-  echo "Attempting to download vendor archive for version $api_version"
-  
-  # Create directory if it doesn't exist
-  mkdir -p "$(dirname "$vendor_store_path")"
-  
-  # Attempt to download the vendor archive
-  echo "Downloading vendor archive from $vendor_store_url to $vendor_store_path"
-  if curl -f -L "$vendor_store_url" -o "$vendor_store_path"; then
-    echo "Successfully downloaded vendor archive to $vendor_store_path"
-    # Return the path to the downloaded archive
-    echo "$vendor_store_path"
-    return 0
-  else
-    echo "Failed to download vendor archive from URL"
-    return 1
-  fi
+  echo "Error: Download functionality not available - vendor store URL not configured" >&2
+  return 1
 }
 
 # Function to ensure vendor archive is available
-# This tries to locate or download the appropriate vendor archive
 # Returns the path to the vendor archive or empty string if not available
 ensure_vendor_archive() {
   # Define all local variables at the top
   local info
   local api_version=""
-  local vendor_store_url=""
   local vendor_store_path=""
   
   # Get archive information
@@ -113,8 +81,7 @@ ensure_vendor_archive() {
   fi
   
   api_version="${info[0]}"
-  vendor_store_url="${info[1]}"
-  vendor_store_path="${info[2]}"
+  vendor_store_path="${info[1]}"
   
   echo "Looking for vendor archive at $vendor_store_path" >&2
   
@@ -124,28 +91,41 @@ ensure_vendor_archive() {
     return 0
   fi
   
-  # Expected archive is missing, attempt to download
-  echo "Expected vendor archive missing at $vendor_store_path. Attempting to download..." >&2
-  downloaded_archive=$(redownload_vendor_archive)
-  if [ -n "$downloaded_archive" ] && [ -f "$downloaded_archive" ]; then
-    echo "$downloaded_archive"
-    return 0
-  fi
-  
   # No vendor archive available
-  echo "No vendor archive available" >&2
+  echo "No vendor archive available at $vendor_store_path" >&2
   return 1
 }
 
 # Restores the node_modules directory from a backup file
 # Args:
-#   $1 - Path to the backup file (tar.xz format)
+#   $1 - Path to the backup file (tar.xz format) [optional - if not provided, uses vendor store path]
 # Returns:
 #   0 on success, 1 on failure
 # Note: Requires 1.5x the backup size in free space for safe extraction
 restore_dependencies() {
   backup_file="$1"
   
+  # If no backup file provided, get it from vendor store path
+  if [ -z "$backup_file" ]; then
+    local info
+    local vendor_store_path=""
+
+    # Get archive information
+    if ! mapfile -t info < <(get_archive_information); then
+      echo "Error: Failed to get vendor archive information. Skipping restore." >&2
+      return 0
+    fi
+
+    vendor_store_path="${info[1]}"
+
+    if [ -z "$vendor_store_path" ]; then
+      echo "Vendor store path is undefined. Skipping restore."
+      return 0
+    fi
+
+    backup_file="$vendor_store_path"
+  fi
+  
   # Check if backup file exists
   if [ ! -f "$backup_file" ]; then
     echo "Backup file not found at '$backup_file'. Skipping restore."
@@ -191,7 +171,6 @@ archive_dependencies() {
   # Define all local variables at the top
   local info
   local api_version=""
-  local vendor_store_url=""
   local vendor_store_path=""
   local source_dir="$DEPENDENCIES_DIR"
   local archive_file=""
@@ -202,8 +181,7 @@ archive_dependencies() {
     return 1
   else
     api_version="${info[0]}"
-    vendor_store_url="${info[1]}"
-    vendor_store_path="${info[2]}"
+    vendor_store_path="${info[1]}"
     
     archive_file="$vendor_store_path"
   fi
@@ -260,13 +238,8 @@ ensure() {
 # Main logic
 case "$1" in
   'restore')
-    if [ -n "$2" ]; then
-      restore_dependencies "$2"
-      exit $?
-    else
-      echo "Usage: $0 restore <archive_path>"
-      exit 1
-    fi
+    restore_dependencies "$2"
+    exit $?
     ;;
   'archive')
     archive_dependencies
@@ -291,7 +264,7 @@ case "$1" in
     fi
     ;;
   *)
-    echo "Usage: $0 {restore|archive|ensure|cleanup|redownload}"
+    echo "Usage: $0 {restore [archive_path]|archive|ensure|cleanup|redownload}"
     exit 1
     ;;
 esac 

Original file line number	Diff line number	Diff line change
`@@ -120,13 +120,6 @@ try {`
`120`	`120`	`}`
`121`	`121`	`}`
`122`	`122`
`123`		`- const compressionLevel = process.env.WATCH_MODE ? '-1' : '-5';`
`124`		- await $`XZ_OPT=${compressionLevel} tar -cJf packed-node-modules.tar.xz node_modules`;
`125`		`- // Create a subdirectory for the node modules archive`
`126`		`- await mkdir('../node-modules-archive', { recursive: true });`
`127`		- await $`mv packed-node-modules.tar.xz ../node-modules-archive/`;
`128`		- await $`rm -rf node_modules`;
`129`		`-`
`130`	`123`	`// Clean the release directory`
`131`	`124`	await $`rm -rf ../release/*`;
`132`	`125`
Original file line number	Diff line number	Diff line change
`@@ -54,6 +54,7 @@ async function createNodeModulesTarball(outputPath: string): Promise<void> {`
`54`	`54`	* After this operation, the vendored node_modules will be available inside the `deployDir`.
`55`	`55`	`*`
`56`	`56`	`* @param apiVersion Required API version to use for the vendor bundle`
	`57`	`+ * @deprecated vendored node_modules are now included in the API slackware package`
`57`	`58`	`*/`
`58`	`59`	`export async function bundleVendorStore(apiVersion: string): Promise<void> {`
`59`	`60`	`// Ensure deploy directory exists`