updates

2025-11-26 08:15:00 -06:00
parent 3cbd4525a0
commit bb829c9b63
18 changed files with 2440 additions and 349 deletions
--- a/docs/manual-steps.md
+++ b/docs/manual-steps.md
@@ -0,0 +1,431 @@
+# Manual Test Run - Step by Step
+
+This guide provides the exact commands Jenkins runs, so you can execute them manually for testing.
+
+## Quick Start - Automated Script
+
+I've created an interactive script that walks through all steps:
+
+```bash
+cd /Users/ssimpson/Documents/git/resf-test-jenkins
+
+# Edit the configuration in the script first:
+vim docs/manual-test-run.sh
+# Update: TEST_REPO_URL, TEST_REPO_BRANCH, TEST_NAME
+
+# Run the interactive script:
+./docs/manual-test-run.sh
+```
+
+The script will pause at each step so you can see what's happening.
+
+---
+
+## Manual Step-by-Step Commands
+
+If you prefer to run each command manually, here's the exact sequence:
+
+### Prerequisites Check
+
+```bash
+# Verify required tools
+which qemu-img virsh virt-install virt-customize sparrowdo git curl
+
+# Verify libvirt is running
+sudo systemctl status libvirtd
+
+# Verify SSH keys exist
+ls -la ~/.ssh/id_rsa ~/.ssh/id_rsa.pub
+```
+
+---
+
+### Step 1: Set Variables
+
+```bash
+# Set your configuration
+export BUILD_ID="manual-$(date +%s)"
+export IMAGES_DIR="/var/lib/libvirt/images"
+export WORK_DIR="/Users/ssimpson/Documents/git/resf-test-jenkins/manual-test-${BUILD_ID}"
+export SCRIPT_DIR="/Users/ssimpson/Documents/git/resf-test-jenkins"
+
+# QCOW2 Image URL
+export QCOW2_URL="https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2"
+
+# SSH Keys
+export SSH_PRIVATE_KEY="${HOME}/.ssh/id_rsa"
+export SSH_PUBLIC_KEY="${HOME}/.ssh/id_rsa.pub"
+
+# Test Configuration - EDIT THESE
+export TEST_NAME="my-test"
+export TEST_REPO_URL="https://github.com/your-org/your-test-repo.git"
+export TEST_REPO_BRANCH="main"
+
+# Image download behavior (set to "true" to force re-download)
+export REDOWNLOAD_IMAGE="false"
+
+echo "Configuration set for build: ${BUILD_ID}"
+```
+
+---
+
+### Step 2: Initialize Environment
+
+```bash
+# Create working directory
+mkdir -p "${WORK_DIR}"
+cd "${WORK_DIR}"
+
+# Ensure images directory exists and is writable
+sudo mkdir -p "${IMAGES_DIR}"
+sudo chown ${USER}:${USER} "${IMAGES_DIR}"
+
+# Verify scripts are executable
+chmod +x "${SCRIPT_DIR}/scripts"/*.sh
+
+echo "Environment initialized"
+```
+
+---
+
+### Step 3: Download Base QCOW2 Image
+
+```bash
+# Extract filename from URL for caching
+export IMAGE_FILENAME=$(basename "${QCOW2_URL}")
+export CACHED_IMAGE="${IMAGES_DIR}/${IMAGE_FILENAME}"
+export BASE_IMAGE="${IMAGES_DIR}/base-${BUILD_ID}.qcow2"
+
+echo "Image URL: ${QCOW2_URL}"
+echo "Cached image: ${CACHED_IMAGE}"
+echo "Build image: ${BASE_IMAGE}"
+
+if [ "${REDOWNLOAD_IMAGE}" = "true" ]; then
+    echo "REDOWNLOAD_IMAGE is enabled - forcing fresh download"
+    curl -L --progress-bar -o "${BASE_IMAGE}" "${QCOW2_URL}"
+else
+    if [ -f "${CACHED_IMAGE}" ]; then
+        echo "Found cached image, creating copy for this build..."
+        cp "${CACHED_IMAGE}" "${BASE_IMAGE}"
+    else
+        echo "No cached image found, downloading..."
+        curl -L --progress-bar -o "${CACHED_IMAGE}" "${QCOW2_URL}"
+        echo "Creating copy for this build..."
+        cp "${CACHED_IMAGE}" "${BASE_IMAGE}"
+    fi
+fi
+
+# Verify the image
+echo "Verifying image..."
+qemu-img info "${BASE_IMAGE}" | head -10
+
+echo "Base image ready: ${BASE_IMAGE}"
+```
+
+---
+
+### Step 4: Prepare Golden Image
+
+```bash
+# Set golden image path
+export GOLDEN_IMAGE="${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"
+
+# Set prep script (you can customize this)
+export PREP_SCRIPT="${SCRIPT_DIR}/docs/default-prep.sh"
+
+echo "Creating golden image..."
+echo "Base: ${BASE_IMAGE}"
+echo "Golden: ${GOLDEN_IMAGE}"
+echo "Prep Script: ${PREP_SCRIPT}"
+echo "SSH Key: ${SSH_PUBLIC_KEY}"
+
+# Run the setup script
+"${SCRIPT_DIR}/scripts/setup_base.sh" \
+    "${BASE_IMAGE}" \
+    "${PREP_SCRIPT}" \
+    "${GOLDEN_IMAGE}" \
+    "${SSH_PUBLIC_KEY}"
+
+echo "Golden image ready: ${GOLDEN_IMAGE}"
+```
+
+---
+
+### Step 5: Provision Test VM
+
+```bash
+# Set VM name
+export VM_NAME="${TEST_NAME}-${BUILD_ID}"
+
+echo "Provisioning VM: ${VM_NAME}"
+
+# Provision the VM and capture its IP
+export VM_IP=$("${SCRIPT_DIR}/scripts/provision_vm.sh" "${VM_NAME}" "${GOLDEN_IMAGE}" 60)
+
+if [ "$VM_IP" = "ERROR" ] || [ -z "$VM_IP" ]; then
+    echo "ERROR: Failed to provision VM"
+    exit 1
+fi
+
+echo "VM provisioned successfully"
+echo "VM Name: ${VM_NAME}"
+echo "IP Address: ${VM_IP}"
+
+# Wait for SSH to be ready
+echo "Waiting for SSH to be ready..."
+for i in {1..30}; do
+    if ssh -i "${SSH_PRIVATE_KEY}" \
+           -o StrictHostKeyChecking=no \
+           -o ConnectTimeout=5 \
+           -o UserKnownHostsFile=/dev/null \
+           root@${VM_IP} 'echo "SSH ready"' 2>/dev/null; then
+        echo "SSH connection established!"
+        break
+    fi
+    echo "Attempt $i/30..."
+    sleep 2
+done
+
+# Test SSH connection manually
+echo "Testing SSH connection..."
+ssh -i "${SSH_PRIVATE_KEY}" \
+    -o StrictHostKeyChecking=no \
+    -o UserKnownHostsFile=/dev/null \
+    root@${VM_IP} 'hostname && cat /etc/rocky-release'
+```
+
+---
+
+### Step 6: Clone Test Repository
+
+```bash
+# Create test workspace
+export TEST_WORKSPACE="${WORK_DIR}/test-workspace"
+mkdir -p "${TEST_WORKSPACE}"
+cd "${TEST_WORKSPACE}"
+
+echo "Cloning test repository..."
+echo "URL: ${TEST_REPO_URL}"
+echo "Branch: ${TEST_REPO_BRANCH}"
+
+# Clone the repository
+git clone -b "${TEST_REPO_BRANCH}" "${TEST_REPO_URL}" test-repo
+
+# Find the sparrowfile
+export SPARROWFILE=$(find test-repo -name sparrowfile -type f | head -1)
+
+if [ -z "$SPARROWFILE" ]; then
+    echo "ERROR: No sparrowfile found in repository"
+    exit 1
+fi
+
+echo "Found sparrowfile: ${SPARROWFILE}"
+
+# Show sparrowfile contents
+echo "Sparrowfile contents:"
+cat "${SPARROWFILE}"
+```
+
+---
+
+### Step 7: Run Sparrowdo Test
+
+```bash
+# Create logs directory
+mkdir -p "${TEST_WORKSPACE}/logs"
+
+echo "=========================================="
+echo "Running Sparrowdo Test"
+echo "=========================================="
+echo "Target: root@${VM_IP}"
+echo "Sparrowfile: ${SPARROWFILE}"
+echo "Log: ${TEST_WORKSPACE}/logs/test.log"
+echo ""
+
+# Run the test
+timeout 900 sparrowdo \
+    --host="${VM_IP}" \
+    --ssh_user=root \
+    --ssh_private_key="${SSH_PRIVATE_KEY}" \
+    --ssh_args="-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null" \
+    --no_sudo \
+    --sparrowfile="${SPARROWFILE}" 2>&1 | tee "${TEST_WORKSPACE}/logs/test.log"
+
+echo ""
+echo "Test completed!"
+echo "Logs saved to: ${TEST_WORKSPACE}/logs/test.log"
+```
+
+---
+
+### Step 8: Cleanup
+
+```bash
+echo "=========================================="
+echo "Cleanup"
+echo "=========================================="
+
+# Destroy the VM
+echo "Destroying VM: ${VM_NAME}"
+"${SCRIPT_DIR}/scripts/cleanup_vm.sh" "${VM_NAME}"
+
+# Optional: Remove temporary images
+echo ""
+echo "Temporary images:"
+echo "  Base: ${BASE_IMAGE}"
+echo "  Golden: ${GOLDEN_IMAGE}"
+echo ""
+read -p "Remove temporary images? (y/n): " -n 1 -r
+echo
+
+if [[ $REPLY =~ ^[Yy]$ ]]; then
+    echo "Removing images..."
+    sudo rm -f "${BASE_IMAGE}"
+    sudo rm -f "${GOLDEN_IMAGE}"
+    echo "Images removed"
+else
+    echo "Images preserved at:"
+    echo "  ${BASE_IMAGE}"
+    echo "  ${GOLDEN_IMAGE}"
+fi
+```
+
+---
+
+### Step 9: View Results
+
+```bash
+echo "=========================================="
+echo "Test Run Summary"
+echo "=========================================="
+echo "Build ID: ${BUILD_ID}"
+echo "Test Name: ${TEST_NAME}"
+echo "VM Name: ${VM_NAME}"
+echo "Working Directory: ${WORK_DIR}"
+echo "Test Logs: ${TEST_WORKSPACE}/logs/test.log"
+echo ""
+
+# View the test log
+echo "=== Test Log ==="
+cat "${TEST_WORKSPACE}/logs/test.log"
+```
+
+---
+
+## Troubleshooting During Manual Run
+
+### VM Won't Start
+```bash
+# Check libvirt status
+sudo systemctl status libvirtd
+
+# List all VMs
+virsh list --all
+
+# Check VM details
+virsh dominfo "${VM_NAME}"
+
+# View VM console
+virsh console "${VM_NAME}"  # Ctrl+] to exit
+```
+
+### Can't Get IP Address
+```bash
+# Check network
+virsh net-list --all
+virsh net-info default
+
+# View DHCP leases
+virsh net-dhcp-leases default
+
+# Try alternative IP detection
+virsh domifaddr "${VM_NAME}" --source lease
+virsh domifaddr "${VM_NAME}" --source agent
+virsh domifaddr "${VM_NAME}" --source arp
+```
+
+### SSH Connection Fails
+```bash
+# Test SSH manually with verbose output
+ssh -vvv -i "${SSH_PRIVATE_KEY}" \
+    -o StrictHostKeyChecking=no \
+    -o UserKnownHostsFile=/dev/null \
+    root@${VM_IP}
+
+# Check if SSH key was injected
+sudo virt-cat -a "${GOLDEN_IMAGE}" /root/.ssh/authorized_keys
+
+# Check VM network from inside
+virsh console "${VM_NAME}"
+# Then inside VM:
+# ip addr
+# systemctl status sshd
+# firewall-cmd --list-all
+```
+
+### Sparrowdo Fails
+```bash
+# Test connectivity first
+ssh -i "${SSH_PRIVATE_KEY}" \
+    -o StrictHostKeyChecking=no \
+    root@${VM_IP} 'which perl && perl --version'
+
+# Run sparrowdo with debug output
+sparrowdo \
+    --host="${VM_IP}" \
+    --ssh_user=root \
+    --ssh_private_key="${SSH_PRIVATE_KEY}" \
+    --verbose \
+    --sparrowfile="${SPARROWFILE}"
+```
+
+### Clean Up Failed VMs
+```bash
+# List all VMs
+virsh list --all
+
+# Force cleanup
+for vm in $(virsh list --all --name | grep "${BUILD_ID}"); do
+    virsh destroy "$vm" 2>/dev/null || true
+    virsh undefine "$vm" 2>/dev/null || true
+    sudo rm -f "/var/lib/libvirt/images/${vm}.qcow2"
+done
+```
+
+---
+
+## Quick Cleanup Script
+
+If something goes wrong and you need to clean up everything:
+
+```bash
+# Save this as cleanup-all.sh
+#!/bin/bash
+
+BUILD_ID="${1:-manual}"
+
+echo "Cleaning up build: ${BUILD_ID}"
+
+# Stop and remove VMs
+for vm in $(virsh list --all --name | grep "${BUILD_ID}"); do
+    echo "Removing VM: $vm"
+    virsh destroy "$vm" 2>/dev/null || true
+    virsh undefine "$vm" 2>/dev/null || true
+    sudo rm -f "/var/lib/libvirt/images/${vm}.qcow2"
+done
+
+# Remove images
+sudo rm -f "/var/lib/libvirt/images/base-${BUILD_ID}.qcow2"
+sudo rm -f "/var/lib/libvirt/images/golden-${BUILD_ID}.qcow2"
+
+# Remove working directory
+rm -rf "/Users/ssimpson/Documents/git/resf-test-jenkins/manual-test-${BUILD_ID}"
+
+echo "Cleanup complete"
+```
+
+Then run:
+```bash
+chmod +x cleanup-all.sh
+./cleanup-all.sh manual-1234567890  # Use your BUILD_ID
+```