Refactor golden image role: update variable names and remove obsolete customize script

Signed-off-by: Stephen Simpson <ssimpson89@users.noreply.github.com>
Implement Ansible roles for Rocky Linux Testing Framework
2025-12-29 16:07:28 -06:00 · 2025-12-29 16:02:39 -06:00
45 changed files with 1991 additions and 2055 deletions
--- a/.ansible/.lock
+++ b/.ansible/.lock
--- a/444
+++ b/444
@@ -1,444 +0,0 @@
-pipeline {
-    agent {
-        label 'fedora-testing'
-        customWorkspace "/home/jenkins/jenkins-workspace/${BUILD_NUMBER}"
-    }
-
-    parameters {
-        // QCOW2 Image Source
-        string(
-            name: 'QCOW2_URL',
-            defaultValue: 'https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2',
-            description: 'URL to the base QCOW2 image to use for testing'
-        )
-
-        // Test Matrix - JSON array format
-        text(
-            name: 'TEST_MATRIX',
-            defaultValue: '''[
-  {
-    "name": "example-test",
-    "url": "https://github.com/your-org/sparrowdo-tests.git",
-    "branch": "main",
-    "description": "Example Sparrowdo test suite"
-  }
-]''',
-            description: 'JSON array of test definitions (name, url, branch, description)'
-        )
-
-        // Custom Golden Image Preparation
-        text(
-            name: 'GOLDEN_PREP_SCRIPT',
-            defaultValue: '''#!/bin/bash
-set -e
-
-echo "Preparing standard golden image..."
-
-# NOTE: This runs inside virt-customize (offline mode)
-# Cannot use firewall-cmd, hostnamectl, or systemctl start/restart
-# Only systemctl enable works (creates symlinks)
-
-# Update system (optional - can be slow, commented out by default)
-# dnf update -y
-
-# Install common testing dependencies including Raku/Sparrowdo
-dnf install -y \\
-    perl \\
-    git \\
-    wget \\
-    tar \\
-    openssh-server \\
-    vim \\
-    rakudo \\
-    rakudo-zef
-
-# Enable services (works offline)
-systemctl enable sshd
-
-# Create rocky user (standard non-root user)
-useradd -m rocky 2>/dev/null || true
-echo "rocky:rockypass" | chpasswd
-
-# Add rocky to sudoers (needed for sparrowdo --bootstrap)
-echo "rocky ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/rocky
-chmod 0440 /etc/sudoers.d/rocky
-
-# Create testuser for backward compatibility
-useradd -m testuser 2>/dev/null || true
-echo "testuser:testpass" | chpasswd
-
-# Add to sudoers
-echo "testuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/testuser
-chmod 0440 /etc/sudoers.d/testuser
-
-echo "Golden image preparation complete!"
-''',
-            description: 'Shell script to run inside the golden image (virt-customize offline mode)'
-        )
-
-        // Test Selection
-        string(
-            name: 'TEST_FILTER',
-            defaultValue: '.*',
-            description: 'Regex pattern to filter tests (e.g., "login.*" or ".*")'
-        )
-
-        // Concurrency Control
-        choice(
-            name: 'MAX_CONCURRENT',
-            choices: ['1', '2', '3', '5', '10'],
-            description: 'Maximum concurrent VMs'
-        )
-
-        // Image Download Control
-        booleanParam(
-            name: 'REDOWNLOAD_IMAGE',
-            defaultValue: false,
-            description: 'Force re-download of base QCOW2 image even if it exists'
-        )
-
-        // Cleanup Control
-        booleanParam(
-            name: 'KEEP_GOLDEN_IMAGE',
-            defaultValue: false,
-            description: 'Keep golden image after testing'
-        )
-
-        // SSH Configuration
-        string(
-            name: 'SSH_PRIVATE_KEY_PATH',
-            defaultValue: '${HOME}/.ssh/id_rsa',
-            description: 'Path to SSH private key for VM access'
-        )
-
-        string(
-            name: 'SSH_PUBLIC_KEY_PATH',
-            defaultValue: '${HOME}/.ssh/id_rsa.pub',
-            description: 'Path to SSH public key to inject into VMs'
-        )
-    }
-
-    environment {
-        BUILD_ID = "${BUILD_NUMBER}"
-        IMAGES_DIR = "/var/lib/libvirt/images"
-        MAX_PARALLEL = "${params.MAX_CONCURRENT}"
-    }
-
-    options {
-        disableConcurrentBuilds()
-        timeout(time: 2, unit: 'HOURS')
-        buildDiscarder(logRotator(numToKeepStr: '10'))
-    }
-
-    stages {
-        stage('Initialize') {
-            steps {
-                script {
-                    echo "=========================================="
-                    echo "Build ${BUILD_ID}"
-                    echo "Node: ${env.NODE_NAME}"
-                    echo "Max Concurrent: ${params.MAX_CONCURRENT}"
-                    echo "Test Filter: ${params.TEST_FILTER}"
-                    echo "=========================================="
-
-                    sh '''
-                        sudo mkdir -p ${IMAGES_DIR}
-                        sudo chown ${USER}:${USER} ${IMAGES_DIR}
-                        chmod +x scripts/*.sh
-                        echo "Environment initialized"
-                    '''
-                }
-            }
-        }
-
-        stage('Download Base Image') {
-            steps {
-                script {
-                    dir(IMAGES_DIR) {
-                        // Extract filename from URL for caching
-                        def imageFilename = params.QCOW2_URL.split('/').last()
-                        def cachedImagePath = "${IMAGES_DIR}/${imageFilename}"
-                        def buildImagePath = "${IMAGES_DIR}/base-${BUILD_ID}.qcow2"
-
-                        if (params.REDOWNLOAD_IMAGE) {
-                            echo "REDOWNLOAD_IMAGE is enabled - forcing fresh download"
-                            sh """
-                                echo "Downloading QCOW2 image from: ${QCOW2_URL}"
-                                curl -L --progress-bar -o "${buildImagePath}" "${QCOW2_URL}"
-                                echo ""
-                                echo "Image downloaded successfully:"
-                                qemu-img info "${buildImagePath}" | head -5
-                            """
-                        } else {
-                            sh """
-                                if [ -f "${cachedImagePath}" ]; then
-                                    echo "Found cached image: ${cachedImagePath}"
-                                    echo "Creating copy for build ${BUILD_ID}..."
-                                    cp "${cachedImagePath}" "${buildImagePath}"
-                                    qemu-img info "${buildImagePath}" | head -5
-                                else
-                                    echo "No cached image found at: ${cachedImagePath}"
-                                    echo "Downloading QCOW2 image from: ${QCOW2_URL}"
-                                    curl -L --progress-bar -o "${cachedImagePath}" "${QCOW2_URL}"
-                                    echo ""
-                                    echo "Image downloaded successfully:"
-                                    qemu-img info "${cachedImagePath}" | head -5
-                                    echo "Creating copy for build ${BUILD_ID}..."
-                                    cp "${cachedImagePath}" "${buildImagePath}"
-                                fi
-                            """
-                        }
-
-                        env.BASE_IMAGE_PATH = buildImagePath
-                        echo "Base image ready at: ${buildImagePath}"
-                    }
-                }
-            }
-        }
-
-        stage('Prepare Golden Image') {
-            steps {
-                script {
-                    def goldenImage = "${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"
-                    def prepScript = "prep-${BUILD_ID}.sh"
-
-                    // Expand SSH key paths
-                    def sshPubKey = params.SSH_PUBLIC_KEY_PATH.replace('${HOME}', env.HOME)
-
-                    // Write prep script
-                    writeFile file: prepScript, text: params.GOLDEN_PREP_SCRIPT
-                    sh "chmod +x ${prepScript}"
-
-                    env.PREP_SCRIPT_PATH = pwd() + "/" + prepScript
-                    env.GOLDEN_IMAGE_PATH = goldenImage
-                    env.SSH_PUB_KEY_PATH = sshPubKey
-
-                    echo "Creating golden image: ${goldenImage}"
-                    echo "Using SSH public key: ${sshPubKey}"
-
-                    sh '''
-                        ./scripts/setup_base.sh \\
-                            ${BASE_IMAGE_PATH} \\
-                            ${PREP_SCRIPT_PATH} \\
-                            ${GOLDEN_IMAGE_PATH} \\
-                            ${SSH_PUB_KEY_PATH}
-                    '''
-
-                    sh "rm -f ${prepScript}"
-                }
-            }
-        }
-
-        stage('Bootstrap Golden Image') {
-            steps {
-                script {
-                    echo "=========================================="
-                    echo "Bootstrapping Sparrowdo in Golden Image"
-                    echo "=========================================="
-                    echo "This installs Sparrowdo ONCE in the golden image"
-                    echo "All test VMs will inherit this and skip bootstrap"
-                    echo ""
-
-                    // Expand SSH private key path
-                    def sshPrivateKey = params.SSH_PRIVATE_KEY_PATH.replace('${HOME}', env.HOME)
-
-                    sh """
-                        ./scripts/bootstrap_golden.sh \\
-                            ${GOLDEN_IMAGE_PATH} \\
-                            ${sshPrivateKey}
-                    """
-
-                    echo "Golden image successfully bootstrapped"
-                }
-            }
-        }
-
-        stage('Parse Test Matrix') {
-            steps {
-                script {
-                    def testMatrix = readJSON text: params.TEST_MATRIX.trim()
-                    def filtered = testMatrix.findAll { it.name =~ params.TEST_FILTER }
-
-                    if (filtered.isEmpty()) {
-                        error "No tests match filter: ${params.TEST_FILTER}"
-                    }
-
-                    echo "=========================================="
-                    echo "Tests to run (${filtered.size()}):"
-                    filtered.each { echo "  - ${it.name}: ${it.description}" }
-                    echo "=========================================="
-
-                    env.FILTERED_TESTS = groovy.json.JsonOutput.toJson(filtered)
-                }
-            }
-        }
-
-        stage('Run Tests') {
-            steps {
-                script {
-                    def tests = readJSON text: env.FILTERED_TESTS
-                    def tasks = [:]
-
-                    // Expand SSH key path
-                    def sshPrivateKey = params.SSH_PRIVATE_KEY_PATH.replace('${HOME}', env.HOME)
-
-                    tests.each { test ->
-                        def testName = test.name
-                        def testUrl = test.url
-                        def testBranch = test.branch ?: 'main'
-
-                        tasks["${testName}"] = {
-                            lock(resource: 'kvm-slots', quantity: 1) {
-                                stage("Test: ${testName}") {
-                                    def vmId = "${testName}-${BUILD_ID}"
-                                    def targetIp = ""
-
-                                    try {
-                                        dir("ws-${testName}") {
-                                            echo "[${testName}] Provisioning VM..."
-                                            def ipOutput = sh(
-                                                script: "${WORKSPACE}/scripts/provision_vm.sh ${vmId} ${GOLDEN_IMAGE_PATH} 60",
-                                                returnStdout: true
-                                            ).trim()
-
-                                            if (ipOutput.contains("ERROR")) {
-                                                error "Failed to provision VM"
-                                            }
-
-                                            // Extract IP from output (last line)
-                                            targetIp = ipOutput.split('\n').last()
-                                            echo "[${testName}] VM ready at: ${targetIp}"
-
-                                            // Wait for SSH to be ready
-                                            echo "[${testName}] Waiting for SSH..."
-                                            sh """
-                                                for i in {1..30}; do
-                                                    if ssh -i ${sshPrivateKey} -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@${targetIp} 'echo SSH ready' 2>/dev/null; then
-                                                        echo "SSH connection established"
-                                                        break
-                                                    fi
-                                                    echo "Attempt \$i/30..."
-                                                    sleep 2
-                                                done
-                                            """
-
-                                            // Clone test repository
-                                            echo "[${testName}] Cloning test repository..."
-                                            sh "git clone -b ${testBranch} ${testUrl} test-repo || true"
-
-                                            // Verify sparrowfile exists (check for main.raku or sparrowfile)
-                                            def sparrowfilePath = sh(
-                                                script: 'find test-repo -name main.raku -type f | head -1',
-                                                returnStdout: true
-                                            ).trim()
-
-                                            if (!sparrowfilePath) {
-                                                sparrowfilePath = sh(
-                                                    script: 'find test-repo -name sparrowfile -type f | head -1',
-                                                    returnStdout: true
-                                                ).trim()
-                                            }
-
-                                            if (!sparrowfilePath) {
-                                                error "No sparrowfile or main.raku found in test repository"
-                                            }
-
-                                            echo "[${testName}] Found sparrowfile: ${sparrowfilePath}"
-
-                                            // Run test (bootstrap was already done on golden image)
-                                            echo "[${testName}] Running Sparrowdo test..."
-                                            sh """
-                                                mkdir -p logs
-                                                timeout 900 sparrowdo \\
-                                                    --host=${targetIp} \\
-                                                    --ssh_user=rocky \\
-                                                    --ssh_private_key=${sshPrivateKey} \\
-                                                    --ssh_args='-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null' \\
-                                                    --no_sudo \\
-                                                    --sparrowfile=${sparrowfilePath} \\
-                                                    --verbose \\
-                                                    --color 2>&1 | tee logs/test.log
-                                            """
-
-                                            echo "[${testName}] Test completed successfully"
-                                            archiveArtifacts artifacts: "logs/**", allowEmptyArchive: true
-                                        }
-
-                                    } catch (e) {
-                                        echo "[${testName}] FAILED: ${e.message}"
-
-                                        // Archive any logs that were generated
-                                        dir("ws-${testName}") {
-                                            archiveArtifacts artifacts: "logs/**", allowEmptyArchive: true
-                                        }
-
-                                        currentBuild.result = 'UNSTABLE'
-
-                                    } finally {
-                                        echo "[${testName}] Cleaning up VM..."
-                                        sh "${WORKSPACE}/scripts/cleanup_vm.sh ${vmId} || true"
-                                    }
-                                }
-                            }
-                        }
-                    }
-
-                    // Run all tests in parallel (respecting lock quantity)
-                    parallel tasks
-                }
-            }
-        }
-    }
-
-    post {
-        always {
-            script {
-                echo "=========================================="
-                echo "Post-build cleanup"
-                echo "=========================================="
-
-                if (!params.KEEP_GOLDEN_IMAGE) {
-                    sh '''
-                        echo "Removing temporary images..."
-                        sudo rm -f ${IMAGES_DIR}/base-${BUILD_ID}.qcow2 || true
-                        sudo rm -f ${IMAGES_DIR}/golden-${BUILD_ID}.qcow2 || true
-                        echo "Temporary images cleaned up"
-                    '''
-                } else {
-                    echo "Golden image preserved: ${GOLDEN_IMAGE_PATH}"
-                    echo "Base image preserved: ${BASE_IMAGE_PATH}"
-                }
-
-                // Clean up any orphaned VMs
-                sh '''
-                    echo "Checking for orphaned VMs from build ${BUILD_ID}..."
-                    for vm in $(virsh list --all --name 2>/dev/null | grep -E "-${BUILD_ID}"); do
-                        echo "Cleaning orphaned VM: $vm"
-                        sudo virsh destroy "$vm" 2>/dev/null || true
-                        sudo virsh undefine "$vm" 2>/dev/null || true
-                        sudo rm -f "/var/lib/libvirt/images/${vm}.qcow2" 2>/dev/null || true
-                    done
-                    echo "Orphan cleanup complete"
-                '''
-            }
-        }
-
-        success {
-            echo "=========================================="
-            echo "Build ${BUILD_ID} completed successfully!"
-            echo "=========================================="
-        }
-
-        failure {
-            echo "=========================================="
-            echo "Build ${BUILD_ID} failed!"
-            echo "=========================================="
-        }
-
-        unstable {
-            echo "=========================================="
-            echo "Build ${BUILD_ID} completed with test failures"
-            echo "=========================================="
-        }
-    }
-}
--- a/Jenkinsfile.simple
+++ b/Jenkinsfile.simple
@@ -1,61 +0,0 @@
-pipeline {
-    agent { label 'fedora-testing' }
-
-    parameters {
-        string(name: 'QCOW2_URL', defaultValue: 'https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2')
-        text(name: 'TEST_REPOS', defaultValue: 'https://github.com/org/test1\nhttps://github.com/org/test2')
-        choice(name: 'MAX_PARALLEL', choices: ['1', '2', '3', '5'], description: 'Concurrent tests')
-    }
-
-    environment {
-        IMAGES_DIR = '/var/lib/libvirt/images'
-        BUILD_ID = "${env.BUILD_NUMBER}"
-    }
-
-    stages {
-        stage('Download') {
-            steps {
-                sh './scripts/download-image.sh "${QCOW2_URL}" "${IMAGES_DIR}" > base.txt'
-                script {
-                    env.BASE_IMAGE = readFile('base.txt').trim()
-                }
-            }
-        }
-
-        stage('Build Golden') {
-            steps {
-                sh './scripts/build-golden.sh "${BASE_IMAGE}" "${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"'
-                script {
-                    env.GOLDEN_IMAGE = "${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"
-                }
-            }
-        }
-
-        stage('Run Tests') {
-            steps {
-                script {
-                    def repos = params.TEST_REPOS.split('\n').findAll { it.trim() }
-                    def tasks = [:]
-
-                    repos.each { repo ->
-                        def testName = repo.tokenize('/').last().replace('.git', '')
-                        tasks[testName] = {
-                            lock(resource: 'vm-slots', quantity: 1) {
-                                sh "./scripts/run-test.sh '${testName}' '${repo}' '${GOLDEN_IMAGE}'"
-                            }
-                        }
-                    }
-
-                    parallel tasks
-                }
-            }
-        }
-    }
-
-    post {
-        always {
-            sh "./scripts/cleanup-all.sh '.*-${BUILD_ID}' || true"
-            sh "rm -f '${IMAGES_DIR}/golden-${BUILD_ID}.qcow2' || true"
-        }
-    }
-}
--- a/README.md
+++ b/README.md
@@ -6,129 +6,154 @@ Simple, portable Sparrowdo testing framework for Rocky Linux.

 This framework provides automated testing for Rocky Linux:

- **Simple Scripts**: 4 standalone bash scripts, easily portable
 - **VM Isolation**: Each test runs in a fresh VM
 - **Parallel Execution**: Run multiple tests concurrently
 - **Fast Provisioning**: Linked clones (copy-on-write) for speed
- **Jenkins Ready**: Simple Jenkinsfile orchestrates scripts
+- **Bootstrap Once**: Golden image approach saves time
+
+Two implementations available:
+- **Ansible**: Provider-aware, structured roles, better for automation (recommended)
+- **Shell Scripts**: Simple scripts, easy to integrate anywhere (libvirt only)

 ## Prerequisites

 ```bash
-sudo dnf install -y qemu-kvm libvirt virt-install guestfs-tools rakudo
+sudo dnf install -y qemu-kvm libvirt virt-install guestfs-tools rakudo ansible
 sudo systemctl enable --now libvirtd
 sudo usermod -a -G libvirt $(whoami)
-
 ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N ""
 ```

 ## Quick Start

-### Manual Test Run
+### Ansible (Recommended)

 ```bash
-# 1. Download base image (cached automatically)
+cd ansible
+
+# 1. Add your tests to vars/test-definitions.yml
+# 2. Build golden image (interactive prompt for Rocky version)
+ansible-playbook playbooks/libvirt/build-golden-image.yml
+
+# 3. Run all tests
+ansible-playbook playbooks/run-tests.yml
+
+# 4. Run specific test
+ansible-playbook playbooks/run-tests.yml -e "test_filter=Sparky_Knot"
+```
+
+### Shell Scripts
+
+```bash
+# Download base image
 BASE=$(./scripts/download-image.sh \
  https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2)

-# 2. Build golden image (includes Sparrowdo bootstrap)
+# Build golden image
 ./scripts/build-golden.sh "$BASE" /var/lib/libvirt/images/golden.qcow2

-# 3. Run test
+# Run test
 ./scripts/run-test.sh my-test \
  https://github.com/your-org/test-repo.git \
  /var/lib/libvirt/images/golden.qcow2
-
-# 4. Cleanup
-./scripts/cleanup-all.sh
 ```

-### Jenkins
-
-1. Create new Pipeline job in Jenkins
-2. Point to this repository
-3. Use `Jenkinsfile.simple`
-4. Configure parameters:
-   - **QCOW2_URL**: Rocky Linux base image URL
-   - **TEST_REPOS**: One test repo URL per line
-   - **MAX_PARALLEL**: Number of concurrent tests
-5. Build with Parameters
-
 ## How It Works

 ```
-Download QCOW2 → Build Golden Image → Run Tests in Parallel → Cleanup
-                 (includes bootstrap)   (isolated VMs)
+Download QCOW2 → Build Golden Image → Run Tests → Cleanup
+                 (bootstrap once)     (parallel)
 ```

 **Golden Image Creation:**
 1. Copy base QCOW2 image
-2. Install Raku, Sparrowdo, dependencies via virt-customize
+2. Install Raku, Sparrowdo via virt-customize
 3. Create `rocky` user with sudo and SSH keys
 4. Boot VM and run `sparrowdo --bootstrap` (once)
-5. Shutdown VM, golden image ready
+5. Shutdown, save as golden image

 **Test Execution:**
-1. Provision VM as linked clone (1 second)
+1. Provision VM as linked clone (~1 second)
 2. Clone test repository
-3. Run `sparrowdo --no_sudo` with test
+3. Run `sparrowdo --no_sudo`
 4. Cleanup VM

-## Scripts
-
-### download-image.sh
-```bash
-./scripts/download-image.sh <url> [output_dir] [force]
-```
-Downloads and caches QCOW2 images. Returns path to cached image.
-
-### build-golden.sh
-```bash
-./scripts/build-golden.sh <base_image> <golden_image> [ssh_pub_key]
-```
-Creates golden image from base QCOW2. Installs Raku/Sparrowdo and bootstraps.
-
-### run-test.sh
-```bash
-./scripts/run-test.sh <test_name> <test_repo_url> <golden_image> [ssh_key]
-```
-Provisions VM, clones test repo, runs Sparrowdo test, cleans up.
-
-### provision_vm.sh
-```bash
-./scripts/provision_vm.sh <vm_name> <golden_image> [timeout]
-```
-Creates VM as linked clone, starts it, returns IP address.
-
-### cleanup_vm.sh
-```bash
-./scripts/cleanup_vm.sh <vm_name>
-```
-Destroys VM and removes disk image.
-
-### cleanup-all.sh
-```bash
-./scripts/cleanup-all.sh [pattern]
-```
-Emergency cleanup for orphaned VMs matching pattern.
-
 ## Directory Structure

 ```
 .
-├── Jenkinsfile.simple       # Simple Jenkins pipeline
-├── README.md                # This file
-├── scripts/
-│   ├── download-image.sh    # Download/cache images
-│   ├── build-golden.sh      # Create golden image
-│   ├── run-test.sh          # Run single test
-│   ├── provision_vm.sh      # Provision VM
-│   ├── cleanup_vm.sh        # Cleanup VM
-│   └── cleanup-all.sh       # Emergency cleanup
-└── docs/                    # Additional documentation
+├── ansible/                      # Ansible implementation (provider-aware)
+│   ├── inventory/
+│   │   ├── hosts.yml            # Provider groups (libvirt, aws, azure)
+│   │   └── group_vars/          # Provider-specific settings
+│   ├── vars/
+│   │   └── test-definitions.yml # Centralized test list
+│   ├── playbooks/
+│   │   ├── libvirt/             # Libvirt-specific playbooks
+│   │   │   └── build-golden-image.yml
+│   │   ├── aws/                 # AWS playbooks (future)
+│   │   ├── azure/               # Azure playbooks (future)
+│   │   └── run-tests.yml        # Provider-agnostic test runner
+│   ├── tasks/
+│   │   └── run-test-with-infrastructure.yml
+│   └── roles/                   # Reusable roles
+│       ├── download_image/
+│       ├── golden_image/
+│       ├── bootstrap_sparrowdo/
+│       ├── provision_vm/
+│       ├── run_test/
+│       └── cleanup_vm/
+├── scripts/                     # Shell scripts (libvirt only)
+│   ├── download-image.sh
+│   ├── build-golden.sh
+│   ├── run-test.sh
+│   └── cleanup_vm.sh
+└── docs/
+    ├── ANSIBLE-GUIDE.md
+    └── BOOTSTRAP-APPROACH.md
 ```

-## Test Repository Format
+## Ansible Roles
+
+| Role | Purpose |
+|------|---------|
+| `download_image` | Download and cache QCOW2 images |
+| `golden_image` | Build golden image with virt-customize |
+| `bootstrap_sparrowdo` | Bootstrap Sparrowdo on golden image |
+| `provision_vm` | Provision VM as linked clone |
+| `run_test` | Execute test workflow |
+| `cleanup_vm` | Clean up VMs and disks |
+
+## Shell Scripts
+
+| Script | Purpose |
+|--------|---------|
+| `download-image.sh` | Download and cache QCOW2 images |
+| `build-golden.sh` | Build golden image |
+| `bootstrap_golden.sh` | Bootstrap Sparrowdo |
+| `run-test.sh` | Run single test |
+| `provision_vm.sh` | Create VM |
+| `cleanup_vm.sh` | Cleanup single VM |
+| `cleanup-all.sh` | Cleanup multiple VMs |
+
+## Test Definitions
+
+### Ansible
+Edit `ansible/vars/test-definitions.yml` to add tests:
+
+```yaml
+tests:
+  - name: "Sparky_Knot"
+    repo_url: "https://git.resf.org/testing/Sparky_Knot.git"
+    # Optional: branch, timeout
+
+  - name: "ssh-test"
+    repo_url: "https://github.com/your-org/ssh-test.git"
+    branch: "develop"
+    timeout: 600
+```
+
+### Test Repository Format

 Your test repository needs `sparrowfile` or `main.raku`:

@@ -140,183 +165,66 @@ task-run 'check-sshd', %(
    plugin => 'systemd-service',
    args => ['sshd', 'running']
 );
-
-task-run 'verify-rocky-version', %(
-    plugin => 'shell-command',
-    args => 'cat /etc/rocky-release'
-);
 ```

 ## Configuration

-### Golden Image Users
+### Ansible

- **rocky**: Primary test user (password: `rockypass`, sudo: yes)
- **root**: Root user (password: `rockytesting`)
+**Provider Groups:**
+- `inventory/hosts.yml` - Provider groups (libvirt, aws, azure)
+- `inventory/group_vars/all.yml` - Settings for all providers
+- `inventory/group_vars/libvirt.yml` - Libvirt-specific (VM resources, packages)
+- `inventory/group_vars/aws.yml` - AWS-specific (future)
+- `inventory/group_vars/azure.yml` - Azure-specific (future)

-Both have SSH keys injected from `~/.ssh/id_rsa.pub`.
+**Test Definitions:**
+- `vars/test-definitions.yml` - Centralized test repository list

-### VM Resources
-
-Default per VM (edit `provision_vm.sh` to change):
- Memory: 2048 MB
- CPUs: 2
- Disk: Linked clone (only diffs stored)
-
-## Portability
-
-All logic is in standalone shell scripts. Easy to integrate with:
-
-**GitHub Actions:**
-```yaml
- name: Run test
-  run: ./scripts/run-test.sh my-test $REPO $GOLDEN
-```
-
-**GitLab CI:**
-```yaml
-test:
-  script:
-    - ./scripts/run-test.sh my-test $REPO $GOLDEN
-```
-
-**Windmill:**
-```typescript
-await bash.run(`./scripts/run-test.sh ${name} ${repo} ${golden}`)
-```
-
-**Manually:**
+**Example override:**
 ```bash
-./scripts/run-test.sh my-test https://github.com/org/test.git golden.qcow2
+ansible-playbook playbooks/run-tests.yml -e "vm_memory=4096"
 ```

+### Shell Scripts
+Pass parameters directly:
+```bash
+./scripts/provision_vm.sh <vm_name> <golden_image> [timeout]
+```
+
+## Documentation
+
+- [Ansible Guide](docs/ANSIBLE-GUIDE.md) - Complete Ansible documentation
+- [Bootstrap Approach](docs/BOOTSTRAP-APPROACH.md) - How golden image works
+
+## Key Features
+
+### Ansible Implementation
+- **Provider-aware** - Supports libvirt, AWS, Azure (cloud providers in progress)
+- **Interactive prompts** - No need to remember image URLs
+- **Test definitions** - Centralized test list in `vars/test-definitions.yml`
+- **Result tracking** - Shows passed/failed tests with summary report
+- **Automatic cleanup** - Infrastructure cleaned up after each test
+
 ## Troubleshooting

-### List VMs
 ```bash
+# List VMs
 virsh -c qemu:///system list --all
-```

-### Get VM IP
-```bash
+# Get VM IP
 virsh -c qemu:///system domifaddr <vm-name>
-```
-
-### SSH to VM
-```bash
-ssh -i ~/.ssh/id_rsa rocky@<vm-ip>
-```
-
-### View Console
-```bash
-virsh -c qemu:///system console <vm-name>
-# Press Ctrl+] to exit
-```
-
-### Check Network
-```bash
-virsh -c qemu:///system net-list --all
-virsh -c qemu:///system net-start default  # If stopped
-```
-
-### Force Cleanup
-```bash
-./scripts/cleanup-all.sh
-rm -f /var/lib/libvirt/images/golden-*.qcow2
-```
-
-### Bootstrap Fails
-```bash
-# Check if VM is accessible
-ssh -i ~/.ssh/id_rsa rocky@<vm-ip>
-
-# Manually bootstrap
-sparrowdo --host <vm-ip> --ssh_user rocky --bootstrap --color
-```
-
-## Performance
-
-### Image Caching
- First download: ~5-10 minutes (2GB)
- Subsequent builds: 2 seconds (cached)
-
-### Golden Image Build
- Base preparation: ~2-3 minutes (virt-customize)
- Bootstrap: ~5-10 minutes (sparrowdo --bootstrap)
- Total: ~7-13 minutes (once per build)
-
-### Test Execution
- VM provision: ~30 seconds (boot + IP)
- Test runtime: Varies by test
- VM cleanup: ~2 seconds
-
-### Example: 70 Tests
- Golden image: 10 minutes (once)
- 70 tests @ 3 concurrent: ~15 minutes
- **Total: 25 minutes**
-
-Compare to bootstrapping each VM: 70 × 7 min = 490 minutes (8+ hours)!
-
-## Advanced Usage
-
-### Custom Golden Image
-```bash
-# Create custom prep script
-cat > custom-prep.sh << 'EOF'
-#!/bin/bash
-dnf install -y postgresql-server nginx
-systemctl enable postgresql nginx
-EOF
-
-# Use virt-customize directly
-sudo virt-customize -a golden.qcow2 \
-    --run custom-prep.sh \
-    --selinux-relabel
-```
-
-### Testing Beta Images
-```bash
-BASE=$(./scripts/download-image.sh \
-  https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base-9.5-beta.x86_64.qcow2 \
-  /var/lib/libvirt/images true)  # Force re-download
-```
-
-### Running Subset of Tests
-```bash
-# In Jenkins, edit TEST_REPOS parameter
-https://github.com/org/test1.git
-https://github.com/org/test3.git
-# (omit test2)
-```
-
-### Debugging Test Failures
-```bash
-# Run test manually and keep VM alive
-VM_NAME="debug-vm-$$"
-VM_IP=$(./scripts/provision_vm.sh "$VM_NAME" golden.qcow2 60 | tail -1)

 # SSH to VM
-ssh -i ~/.ssh/id_rsa rocky@$VM_IP
+ssh -i ~/.ssh/id_rsa rocky@<vm-ip>

-# Manually run test steps
-cd /tmp
-git clone https://github.com/org/test.git
-cd test
-sparrowdo --host localhost --ssh_user rocky --no_sudo --sparrowfile main.raku
+# Ansible verbose mode
+ansible-playbook playbooks/run-tests.yml -vvv

-# Cleanup when done
-./scripts/cleanup_vm.sh "$VM_NAME"
+# Run specific test
+ansible-playbook playbooks/run-tests.yml -e "test_filter=Sparky_Knot"
 ```

-## Contributing
-
-This framework is designed to be simple and portable. Keep it that way:
-
- Scripts should be standalone bash
- Minimal dependencies
- Easy to read, no over-commenting
- Portable across CI/CD platforms
-
 ## License

 [Your License Here]
--- a/README.simple.md
+++ b/README.simple.md
@@ -28,15 +28,6 @@ BASE=$(./scripts/download-image.sh https://download.rockylinux.org/pub/rocky/9/i
 ./scripts/cleanup-all.sh
 ```

-### Jenkins
-
-1. Create Pipeline job
-2. Point to `Jenkinsfile.simple`
-3. Set parameters:
-   - `QCOW2_URL`: Rocky Linux image URL
-   - `TEST_REPOS`: One repo URL per line
-   - `MAX_PARALLEL`: Number of concurrent tests
-
 ## Scripts

 - `download-image.sh` - Download/cache QCOW2 images
--- a/ansible/README.md
+++ b/ansible/README.md
@@ -0,0 +1,194 @@
+# Ansible Implementation
+
+This directory contains the Ansible-based implementation of the Rocky Linux Testing Framework with provider-aware infrastructure support.
+
+## Quick Start
+
+```bash
+cd ansible
+
+# 1. Build golden image (libvirt only)
+ansible-playbook playbooks/libvirt/build-golden-image.yml
+
+# 2. Run all tests
+ansible-playbook playbooks/run-tests.yml
+
+# 3. Run specific test
+ansible-playbook playbooks/run-tests.yml -e "test_filter=Sparky_Knot"
+```
+
+## Documentation
+
+See [ANSIBLE-GUIDE.md](../docs/ANSIBLE-GUIDE.md) for complete documentation.
+
+## Structure
+
+```
+ansible/
+├── ansible.cfg              # Ansible configuration
+├── inventory/
+│   ├── hosts.yml           # Provider groups (libvirt, aws, azure)
+│   └── group_vars/
+│       ├── all.yml         # Provider-agnostic variables
+│       ├── libvirt.yml     # Libvirt-specific settings
+│       ├── aws.yml         # AWS-specific settings (placeholder)
+│       └── azure.yml       # Azure-specific settings (placeholder)
+├── vars/
+│   └── test-definitions.yml # Test repository list
+├── tasks/
+│   └── run-test-with-infrastructure.yml # Provider-aware orchestration
+├── playbooks/
+│   ├── libvirt/            # Libvirt-specific playbooks
+│   │   └── build-golden-image.yml
+│   ├── aws/                # AWS-specific playbooks (future)
+│   ├── azure/              # Azure-specific playbooks (future)
+│   └── run-tests.yml       # Provider-agnostic test runner
+└── roles/
+    ├── download_image/     # Download QCOW2 images
+    ├── golden_image/       # Build golden image
+    ├── bootstrap_sparrowdo/# Bootstrap Sparrowdo
+    ├── provision_vm/       # Provision VMs (libvirt)
+    ├── run_test/          # Execute Sparrowdo tests
+    └── cleanup_vm/        # Cleanup VMs (libvirt)
+```
+
+## Roles
+
+| Role | Purpose |
+|------|---------|
+| `download_image` | Download and cache QCOW2 images |
+| `golden_image` | Create golden image with virt-customize |
+| `bootstrap_sparrowdo` | Bootstrap Sparrowdo on golden image |
+| `provision_vm` | Provision VM as linked clone |
+| `run_test` | Execute Sparrowdo test |
+| `cleanup_vm` | Clean up VMs and disks |
+
+## Playbooks
+
+### Provider-Agnostic
+| Playbook | Purpose |
+|----------|---------|
+| `run-tests.yml` | Run tests from test-definitions.yml (works on any provider) |
+
+### Libvirt-Specific
+| Playbook | Purpose |
+|----------|---------|
+| `libvirt/build-golden-image.yml` | Build golden image with interactive prompts |
+
+### Future
+- `aws/` - AWS-specific setup playbooks
+- `azure/` - Azure-specific setup playbooks
+
+## Configuration
+
+### Provider Groups
+
+The inventory is organized by provider type:
+- **libvirt** - Local VM testing (requires golden image)
+- **aws** - AWS EC2 testing (uses AMIs, ready for implementation)
+- **azure** - Azure VM testing (uses VM images, ready for implementation)
+
+### Test Definitions
+
+Edit `vars/test-definitions.yml` to add/modify tests:
+```yaml
+tests:
+  - name: "Sparky_Knot"
+    repo_url: "https://git.resf.org/testing/Sparky_Knot.git"
+    # Optional: branch, timeout
+```
+
+### Provider-Specific Settings
+
+- **all.yml** - Settings for all providers (SSH, Sparrowdo, etc.)
+- **libvirt.yml** - VM resources, image paths, packages
+- **aws.yml** - AMI IDs, instance types (when implemented)
+- **azure.yml** - Image references, VM sizes (when implemented)
+
+## Examples
+
+### Build Golden Image (Libvirt Only)
+
+```bash
+# Interactive prompt for Rocky image URL
+ansible-playbook playbooks/libvirt/build-golden-image.yml
+```
+
+### Run All Tests
+
+```bash
+ansible-playbook playbooks/run-tests.yml
+```
+
+### Run Specific Test
+
+```bash
+ansible-playbook playbooks/run-tests.yml -e "test_filter=Sparky_Knot"
+```
+
+### Run Tests on Specific Provider
+
+```bash
+ansible-playbook playbooks/run-tests.yml --limit libvirt
+# Future: --limit aws or --limit azure
+```
+
+### Override VM Resources
+
+```bash
+ansible-playbook playbooks/run-tests.yml -e "vm_memory=4096" -e "vm_vcpus=4"
+```
+
+## Troubleshooting
+
+### Verbose Output
+
+```bash
+ansible-playbook playbooks/run-tests.yml -vvv
+```
+
+### Check Syntax
+
+```bash
+ansible-playbook playbooks/libvirt/build-golden-image.yml --syntax-check
+```
+
+### List Tasks
+
+```bash
+ansible-playbook playbooks/run-tests.yml --list-tasks
+```
+
+## Key Features
+
+- **Provider-aware** - Supports multiple infrastructure providers (libvirt, AWS, Azure)
+- **Test definitions** - Centralized test repository list in `vars/test-definitions.yml`
+- **Interactive prompts** - No need to remember image URLs or flags
+- **Result tracking** - Shows passed/failed tests with summary report
+- **Automatic cleanup** - Infrastructure cleaned up after each test
+- **Idempotent** - Safe to re-run
+- **Integration with Ascender** - Works with Ansible automation platform
+
+### Script Mapping
+
+| Shell Script | Ansible Role |
+|--------------|--------------|
+| `download-image.sh` | `download_image` |
+| `setup_base.sh` + `build-golden.sh` | `golden_image` |
+| `bootstrap_golden.sh` | `bootstrap_sparrowdo` |
+| `provision_vm.sh` | `provision_vm` |
+| `run-test.sh` | `run_test` |
+| `cleanup_vm.sh` | `cleanup_vm` |
+
+## Prerequisites
+
+```bash
+sudo dnf install -y ansible qemu-kvm libvirt virt-install guestfs-tools rakudo
+sudo systemctl enable --now libvirtd
+sudo usermod -a -G libvirt $(whoami)
+ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N ""
+```
+
+## License
+
+[Your License Here]
--- a/ansible/ansible.cfg
+++ b/ansible/ansible.cfg
@@ -0,0 +1,20 @@
+[defaults]
+inventory = inventory/hosts.yml
+roles_path = roles
+host_key_checking = False
+retry_files_enabled = False
+gathering = smart
+fact_caching = jsonfile
+fact_caching_connection = /tmp/ansible_facts
+fact_caching_timeout = 3600
+stdout_callback = default
+callbacks_enabled = timer
+
+[privilege_escalation]
+become_method = sudo
+become_user = root
+become_ask_pass = False
+
+[ssh_connection]
+pipelining = True
+ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
--- a/ansible/inventory/group_vars/all.yml
+++ b/ansible/inventory/group_vars/all.yml
@@ -0,0 +1,13 @@
+---
+# Global configuration
+
+# SSH defaults
+ssh_user: "rocky"
+
+# Default image password
+default_image_password: "rocky"
+
+# Rocky Linux image URLs
+rocky9_qcow2_url: "https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2"
+rocky8_qcow2_url: "https://download.rockylinux.org/pub/rocky/8/images/x86_64/Rocky-8-GenericCloud-Base.latest.x86_64.qcow2"
+qcow2_url: "{{ rocky9_qcow2_url }}"
--- a/ansible/inventory/group_vars/aws.yml
+++ b/ansible/inventory/group_vars/aws.yml
@@ -0,0 +1,29 @@
+---
+# AWS provider-specific configuration
+# TODO: Implement AWS-specific settings
+
+# Provider identification
+provider: aws
+
+# AWS-specific variables (to be implemented)
+# aws_region: "us-east-1"
+# aws_instance_type: "t3.medium"
+# aws_vpc_id: ""
+# aws_subnet_id: ""
+# aws_security_group_id: ""
+
+# Rocky Linux AMI IDs (to be populated)
+# rocky9_ami: "ami-xxxxxxxxx"  # Rocky 9 AMI for your region
+# rocky8_ami: "ami-xxxxxxxxx"  # Rocky 8 AMI for your region
+
+# Default AMI (override with -e "ami_id=ami-xxx")
+# ami_id: "{{ rocky9_ami }}"
+
+# Instance settings
+# instance_name_prefix: "test"
+# instance_tags:
+#   Environment: "testing"
+#   ManagedBy: "ansible"
+
+# Note: AWS does not require golden image creation
+# Workflow: Select AMI -> Launch instance -> Run Sparrowdo tests -> Terminate
--- a/ansible/inventory/group_vars/azure.yml
+++ b/ansible/inventory/group_vars/azure.yml
@@ -0,0 +1,36 @@
+---
+# Azure provider-specific configuration
+# TODO: Implement Azure-specific settings
+
+# Provider identification
+provider: azure
+
+# Azure-specific variables (to be implemented)
+# azure_location: "eastus"
+# azure_resource_group: ""
+# azure_vm_size: "Standard_B2s"
+# azure_virtual_network: ""
+# azure_subnet: ""
+# azure_security_group: ""
+
+# Rocky Linux image references (to be populated)
+# rocky9_image:
+#   publisher: "resf"
+#   offer: "rockylinux-9"
+#   sku: "rockylinux-9"
+#   version: "latest"
+
+# rocky8_image:
+#   publisher: "resf"
+#   offer: "rockylinux-8"
+#   sku: "rockylinux-8"
+#   version: "latest"
+
+# VM settings
+# vm_name_prefix: "test"
+# vm_tags:
+#   Environment: "testing"
+#   ManagedBy: "ansible"
+
+# Note: Azure does not require golden image creation
+# Workflow: Select image -> Create VM -> Run Sparrowdo tests -> Delete VM
--- a/ansible/inventory/group_vars/libvirt.yml
+++ b/ansible/inventory/group_vars/libvirt.yml
@@ -0,0 +1,21 @@
+---
+# Libvirt provider configuration
+
+provider: libvirt
+
+# Image directories
+images_dir: "/var/lib/libvirt/images"
+base_images_dir: "{{ images_dir }}/base"
+golden_images_dir: "{{ images_dir }}/golden"
+golden_image_path: "{{ golden_images_dir }}/rocky-golden.qcow2"
+
+# Download settings
+force_download: false
+download_timeout: 1800
+
+# Golden image settings
+root_password: "{{ default_image_password }}"
+
+# VM settings
+vm_memory: 4096
+vm_vcpus: 4
--- a/ansible/inventory/hosts.yml
+++ b/ansible/inventory/hosts.yml
@@ -0,0 +1,16 @@
+---
+all:
+  children:
+
+    libvirt:
+      hosts:
+        ssimpson-pc:
+          ansible_host: 172.16.1.141
+          ansible_user: jenkins
+          ansible_python_interpreter: /usr/bin/python3
+
+    aws:
+      hosts: {}
+
+    azure:
+      hosts: {}
--- a/ansible/playbooks/aws/README.md
+++ b/ansible/playbooks/aws/README.md
@@ -0,0 +1,19 @@
+# AWS Playbooks
+
+This directory will contain AWS-specific playbooks for the Rocky Linux Testing Framework.
+
+## Future Playbooks
+
+- `setup.yml` - Configure AWS resources (VPC, security groups, etc.)
+- `create-ami.yml` - Build custom Rocky Linux AMIs
+- `cleanup.yml` - Clean up AWS resources
+
+## Workflow
+
+AWS testing doesn't require golden image creation. Instead:
+1. Select Rocky Linux AMI
+2. Launch EC2 instance
+3. Run Sparrowdo tests
+4. Terminate instance
+
+All tests use the main `playbooks/run-tests.yml` playbook, which automatically handles AWS-specific infrastructure provisioning.
--- a/ansible/playbooks/azure/README.md
+++ b/ansible/playbooks/azure/README.md
@@ -0,0 +1,19 @@
+# Azure Playbooks
+
+This directory will contain Azure-specific playbooks for the Rocky Linux Testing Framework.
+
+## Future Playbooks
+
+- `setup.yml` - Configure Azure resources (resource groups, networks, etc.)
+- `create-image.yml` - Build custom Rocky Linux VM images
+- `cleanup.yml` - Clean up Azure resources
+
+## Workflow
+
+Azure testing doesn't require golden image creation. Instead:
+1. Select Rocky Linux VM image
+2. Create Azure VM
+3. Run Sparrowdo tests
+4. Delete VM
+
+All tests use the main `playbooks/run-tests.yml` playbook, which automatically handles Azure-specific infrastructure provisioning.
--- a/ansible/playbooks/libvirt/build-golden-image.yml
+++ b/ansible/playbooks/libvirt/build-golden-image.yml
@@ -0,0 +1,78 @@
+---
+# Build a golden image: Download -> Customize -> Bootstrap Sparrowdo
+
+- name: Build Golden Image
+  hosts: libvirt
+  gather_facts: true
+
+  vars_prompt:
+    - name: qcow2_url
+      prompt: "Rocky Linux QCOW2 URL (Enter for Rocky 9 default)"
+      default: "https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2"
+      private: false
+
+  pre_tasks:
+    - name: Verify sparrowdo is installed
+      stat:
+        path: ~/.raku/bin/sparrowdo
+      register: sparrowdo_check
+      failed_when: not sparrowdo_check.stat.exists
+
+    - name: Set build paths
+      set_fact:
+        build_id: "{{ lookup('pipe', 'date +%s') }}"
+
+    - name: Set golden image path
+      set_fact:
+        timestamped_golden_path: "{{ golden_images_dir }}/rocky-golden-{{ build_id }}.qcow2"
+
+    - name: Ensure directories exist
+      file:
+        path: "{{ item }}"
+        state: directory
+        mode: '0755'
+      become: true
+      loop:
+        - "{{ golden_images_dir }}"
+        - /tmp/rocky-test-keys
+
+    - name: Generate SSH keys if needed
+      command: ssh-keygen -t rsa -b 4096 -f /tmp/rocky-test-keys/id_rsa -N "" -C "rocky-test"
+      args:
+        creates: /tmp/rocky-test-keys/id_rsa
+
+    - name: Set SSH key paths
+      set_fact:
+        ssh_private_key_path: /tmp/rocky-test-keys/id_rsa
+        ssh_public_key_path: /tmp/rocky-test-keys/id_rsa.pub
+
+  tasks:
+    - name: Download base image
+      include_role:
+        name: download_image
+      vars:
+        image_path_var: "base_image_path"
+
+    - name: Create golden image
+      include_role:
+        name: golden_image
+      vars:
+        golden_image_path: "{{ timestamped_golden_path }}"
+
+    - name: Bootstrap Sparrowdo
+      include_role:
+        name: bootstrap_sparrowdo
+      vars:
+        golden_image_path: "{{ timestamped_golden_path }}"
+
+    - name: Create symlink to latest golden image
+      file:
+        src: "{{ timestamped_golden_path }}"
+        dest: "{{ golden_image_path }}"
+        state: link
+        force: yes
+      become: true
+
+    - name: Done
+      debug:
+        msg: "Golden image ready: {{ golden_image_path }}"
--- a/ansible/playbooks/run-tests.yml
+++ b/ansible/playbooks/run-tests.yml
@@ -0,0 +1,123 @@
+---
+# Provider-aware test suite runner
+# Loads tests from vars/test-definitions.yml
+# Handles infrastructure provisioning/cleanup based on provider type
+
+- name: Run Test Suite
+  hosts: all
+  gather_facts: true
+
+  vars_files:
+    - ../vars/test-definitions.yml
+
+  vars:
+    # Test filtering (override with -e "test_filter=ssh")
+    test_filter: "{{ default_test_filter }}"
+
+    # Results tracking
+    test_results: []
+    passed_tests: []
+    failed_tests: []
+
+  pre_tasks:
+    - name: Filter tests based on test_filter
+      set_fact:
+        filtered_tests: "{{ tests | selectattr('name', 'match', test_filter) | list }}"
+
+    - name: Validate test list
+      fail:
+        msg: "No tests match filter: {{ test_filter }}"
+      when: filtered_tests | length == 0
+
+    - name: Display test suite information
+      debug:
+        msg:
+          - "=========================================="
+          - "Running Test Suite"
+          - "=========================================="
+          - "Provider: {{ provider }}"
+          - "Total tests: {{ filtered_tests | length }}"
+          - "Test filter: {{ test_filter }}"
+          - "=========================================="
+
+    - name: Display tests to run
+      debug:
+        msg: "  {{ loop_index + 1 }}. {{ item.name }}"
+      loop: "{{ filtered_tests }}"
+      loop_control:
+        index_var: loop_index
+
+  tasks:
+    # Provider-aware test execution loop
+    - name: Execute test suite
+      block:
+        - name: Run each test with provider-specific infrastructure
+          include_tasks: ../tasks/run-test-with-infrastructure.yml
+          loop: "{{ filtered_tests }}"
+          loop_control:
+            loop_var: test_item
+            label: "{{ test_item.name }}"
+      rescue:
+        - name: Handle catastrophic failure
+          debug:
+            msg: "Test suite encountered a critical error"
+
+  post_tasks:
+    - name: Collect test results
+      set_fact:
+        passed_tests: "{{ test_results | selectattr('status', 'equalto', 'passed') | list }}"
+        failed_tests: "{{ test_results | selectattr('status', 'equalto', 'failed') | list }}"
+
+    - name: Display test results summary
+      debug:
+        msg:
+          - "=========================================="
+          - "Test Suite Results"
+          - "=========================================="
+          - "Provider: {{ provider }}"
+          - "Total Tests: {{ test_results | length }}"
+          - "Passed: {{ passed_tests | length }}"
+          - "Failed: {{ failed_tests | length }}"
+          - "=========================================="
+
+    - name: Display passed tests
+      debug:
+        msg: "  ✓ {{ item.name }}"
+      loop: "{{ passed_tests }}"
+      when: passed_tests | length > 0
+
+    - name: Display failed tests
+      debug:
+        msg: "  ✗ {{ item.name }}: {{ item.error | default('Unknown error') }}"
+      loop: "{{ failed_tests }}"
+      when: failed_tests | length > 0
+
+    - name: Save results to file
+      copy:
+        content: |
+          Test Suite Results - {{ ansible_date_time.iso8601 }}
+          Provider: {{ provider }}
+
+          Summary:
+          - Total: {{ test_results | length }}
+          - Passed: {{ passed_tests | length }}
+          - Failed: {{ failed_tests | length }}
+
+          Passed Tests:
+          {% for test in passed_tests %}
+          - {{ test.name }}
+          {% endfor %}
+
+          Failed Tests:
+          {% for test in failed_tests %}
+          - {{ test.name }}: {{ test.error | default('Unknown error') }}
+          {% endfor %}
+        dest: "{{ logs_dir }}/test-results-{{ ansible_date_time.epoch }}.txt"
+      when: test_results | length > 0
+
+    - name: Fail if any tests failed
+      fail:
+        msg: "{{ failed_tests | length }} test(s) failed"
+      when:
+        - failed_tests | length > 0
+        - not continue_on_failure
--- a/ansible/roles/bootstrap_sparrowdo/defaults/main.yml
+++ b/ansible/roles/bootstrap_sparrowdo/defaults/main.yml
@@ -0,0 +1,9 @@
+---
+# Timeouts (seconds)
+vm_boot_timeout: 60
+ssh_port_timeout: 300
+bootstrap_timeout: 900
+
+# VM resources
+vm_memory: 2048
+vm_vcpus: 2
--- a/ansible/roles/bootstrap_sparrowdo/tasks/main.yml
+++ b/ansible/roles/bootstrap_sparrowdo/tasks/main.yml
@@ -0,0 +1,100 @@
+---
+- name: Verify golden image exists
+  stat:
+    path: "{{ golden_image_path }}"
+  register: golden_stat
+  failed_when: not golden_stat.stat.exists
+
+- name: Set bootstrap VM name
+  set_fact:
+    bootstrap_vm: "bootstrap-{{ ansible_date_time.epoch }}"
+
+- name: Bootstrap Sparrowdo
+  block:
+    - name: Create overlay disk
+      command: >
+        qemu-img create -f qcow2
+        -b {{ golden_image_path }} -F qcow2
+        /var/lib/libvirt/images/{{ bootstrap_vm }}.qcow2
+      become: true
+
+    - name: Start bootstrap VM
+      command: >
+        virt-install
+        --name {{ bootstrap_vm }}
+        --memory {{ vm_memory }}
+        --vcpus {{ vm_vcpus }}
+        --disk path=/var/lib/libvirt/images/{{ bootstrap_vm }}.qcow2,format=qcow2
+        --import
+        --os-variant rocky9-unknown
+        --network network=default
+        --noautoconsole
+        --wait 0
+      become: true
+
+    - name: Wait for VM IP
+      shell: >
+        virsh -c qemu:///system domifaddr {{ bootstrap_vm }} --source lease 2>/dev/null |
+        awk '/ipv4/ {print $4}' | cut -d/ -f1 | head -1
+      become: true
+      register: vm_ip
+      until: vm_ip.stdout != "" and vm_ip.stdout != "0.0.0.0"
+      retries: "{{ vm_boot_timeout }}"
+      delay: 2
+      changed_when: false
+
+    - name: Wait for SSH
+      wait_for:
+        host: "{{ vm_ip.stdout }}"
+        port: 22
+        timeout: "{{ ssh_port_timeout }}"
+
+    - name: Run Sparrowdo bootstrap
+      command:
+        argv:
+          - "~/.raku/bin/sparrowdo"
+          - "--host={{ vm_ip.stdout }}"
+          - "--ssh_user={{ ssh_user }}"
+          - "--ssh_private_key={{ ssh_private_key_path }}"
+          - "--bootstrap"
+      timeout: "{{ bootstrap_timeout }}"
+      register: bootstrap_result
+      retries: 3
+      delay: 5
+      until: bootstrap_result.rc == 0
+
+    - name: Shutdown VM
+      command: >
+        ssh -i {{ ssh_private_key_path }}
+        -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ ssh_user }}@{{ vm_ip.stdout }} 'sudo shutdown -h now'
+      ignore_errors: true
+
+    - name: Wait for shutdown
+      shell: virsh -c qemu:///system list --name | grep -q "{{ bootstrap_vm }}"
+      become: true
+      register: vm_running
+      until: vm_running.rc != 0
+      retries: 30
+      delay: 2
+      failed_when: false
+
+  always:
+    - name: Force stop VM if running
+      command: "virsh -c qemu:///system destroy {{ bootstrap_vm }}"
+      become: true
+      ignore_errors: true
+      changed_when: false
+
+    - name: Undefine VM
+      command: "virsh -c qemu:///system undefine {{ bootstrap_vm }}"
+      become: true
+      ignore_errors: true
+      changed_when: false
+
+    - name: Remove overlay disk
+      file:
+        path: "/var/lib/libvirt/images/{{ bootstrap_vm }}.qcow2"
+        state: absent
+      become: true
+      ignore_errors: true
--- a/ansible/roles/cleanup_vm/defaults/main.yml
+++ b/ansible/roles/cleanup_vm/defaults/main.yml
@@ -0,0 +1,17 @@
+---
+# Default variables for cleanup_vm role
+
+# VM name to cleanup (required)
+vm_name: ""
+
+# Force destroy even if running
+force_destroy: true
+
+# Remove disk image
+remove_disk: true
+
+# Cleanup multiple VMs matching pattern
+cleanup_pattern: ""
+
+# Cleanup all VMs in list
+cleanup_vm_list: []
--- a/ansible/roles/cleanup_vm/tasks/cleanup_single.yml
+++ b/ansible/roles/cleanup_vm/tasks/cleanup_single.yml
@@ -0,0 +1,23 @@
+---
+# Cleanup a single VM (used in loop)
+
+- name: Destroy VM {{ vm_to_cleanup }}
+  command: virsh -c qemu:///system destroy {{ vm_to_cleanup }}
+  become: true
+  register: destroy_result
+  failed_when: false
+  changed_when: destroy_result.rc == 0
+
+- name: Undefine VM {{ vm_to_cleanup }}
+  command: virsh -c qemu:///system undefine {{ vm_to_cleanup }}
+  become: true
+  register: undefine_result
+  failed_when: false
+  changed_when: undefine_result.rc == 0
+
+- name: Remove disk for VM {{ vm_to_cleanup }}
+  file:
+    path: "/var/lib/libvirt/images/{{ vm_to_cleanup }}.qcow2"
+    state: absent
+  become: true
+  when: remove_disk
--- a/ansible/roles/cleanup_vm/tasks/main.yml
+++ b/ansible/roles/cleanup_vm/tasks/main.yml
@@ -0,0 +1,77 @@
+---
+# Tasks for cleanup_vm role
+
+- name: Cleanup single VM
+  when: vm_name != ""
+  block:
+    - name: Display cleanup info for single VM
+      debug:
+        msg: "Starting cleanup for VM: {{ vm_name }}"
+
+    - name: Destroy VM
+      command: virsh -c qemu:///system destroy {{ vm_name }}
+      become: true
+      register: destroy_result
+      failed_when: false
+      changed_when: destroy_result.rc == 0
+
+    - name: Display destroy result
+      debug:
+        msg: "{{ 'VM was not running' if destroy_result.rc != 0 else 'VM destroyed' }}"
+
+    - name: Undefine VM
+      command: virsh -c qemu:///system undefine {{ vm_name }}
+      become: true
+      register: undefine_result
+      failed_when: false
+      changed_when: undefine_result.rc == 0
+
+    - name: Display undefine result
+      debug:
+        msg: "{{ 'VM definition already removed' if undefine_result.rc != 0 else 'VM undefined' }}"
+
+    - name: Remove disk image
+      file:
+        path: "/var/lib/libvirt/images/{{ vm_name }}.qcow2"
+        state: absent
+      become: true
+      when: remove_disk
+
+    - name: Display cleanup completion
+      debug:
+        msg: "Cleanup complete for {{ vm_name }}"
+
+- name: Cleanup VMs matching pattern
+  when: cleanup_pattern != ""
+  block:
+    - name: Get list of VMs matching pattern
+      shell: virsh -c qemu:///system list --all --name | grep -E "{{ cleanup_pattern }}"
+      become: true
+      register: matching_vms
+      failed_when: false
+      changed_when: false
+
+    - name: Display matching VMs
+      debug:
+        msg: "Found {{ matching_vms.stdout_lines | length }} VMs matching pattern: {{ cleanup_pattern }}"
+      when: matching_vms.stdout_lines | length > 0
+
+    - name: Cleanup each matching VM
+      include_tasks: cleanup_single.yml
+      loop: "{{ matching_vms.stdout_lines }}"
+      loop_control:
+        loop_var: vm_to_cleanup
+      when: matching_vms.stdout_lines | length > 0
+
+- name: Cleanup VMs from list
+  when: cleanup_vm_list | length > 0
+  block:
+    - name: Display VMs to cleanup
+      debug:
+        msg: "Cleaning up {{ cleanup_vm_list | length }} VMs from list"
+
+    - name: Cleanup each VM in list
+      include_tasks: cleanup_single.yml
+      loop: "{{ cleanup_vm_list }}"
+      loop_control:
+        loop_var: vm_to_cleanup
--- a/ansible/roles/download_image/defaults/main.yml
+++ b/ansible/roles/download_image/defaults/main.yml
@@ -0,0 +1,6 @@
+---
+# Default variables for download_image role
+# Most settings come from inventory (group_vars)
+
+# Return variable name for the downloaded image path
+image_path_var: "base_image_path"
--- a/ansible/roles/download_image/tasks/main.yml
+++ b/ansible/roles/download_image/tasks/main.yml
@@ -0,0 +1,24 @@
+---
+- name: Ensure base images directory exists
+  file:
+    path: "{{ base_images_dir }}"
+    state: directory
+    mode: '0755'
+  become: true
+
+- name: Set cached image path
+  set_fact:
+    cached_image_path: "{{ base_images_dir }}/{{ qcow2_url | basename }}"
+
+- name: Download QCOW2 image
+  get_url:
+    url: "{{ qcow2_url }}"
+    dest: "{{ cached_image_path }}"
+    mode: '0644'
+    timeout: "{{ download_timeout }}"
+    force: "{{ force_download }}"
+  become: true
+
+- name: Set image path fact
+  set_fact:
+    "{{ image_path_var }}": "{{ cached_image_path }}"
--- a/ansible/roles/golden_image/defaults/main.yml
+++ b/ansible/roles/golden_image/defaults/main.yml
@@ -0,0 +1,4 @@
+---
+# Paths (passed from playbook)
+base_image_path: ""
+golden_image_path: ""
--- a/ansible/roles/golden_image/tasks/main.yml
+++ b/ansible/roles/golden_image/tasks/main.yml
@@ -0,0 +1,38 @@
+---
+- name: Verify base image exists
+  stat:
+    path: "{{ base_image_path }}"
+  register: base_image_stat
+  failed_when: not base_image_stat.stat.exists
+
+- name: Ensure golden image directory exists
+  file:
+    path: "{{ golden_image_path | dirname }}"
+    state: directory
+    mode: '0755'
+  become: true
+
+- name: Copy base image to golden image
+  copy:
+    src: "{{ base_image_path }}"
+    dest: "{{ golden_image_path }}"
+    remote_src: true
+    mode: '0644'
+  become: true
+
+- name: Customize golden image
+  command: >
+    virt-customize -a {{ golden_image_path }}
+    --install perl,git,wget,tar,openssh-server,vim
+    --run-command 'useradd -m rocky 2>/dev/null || true'
+    --run-command 'echo "rocky:rockypass" | chpasswd'
+    --run-command 'echo "rocky ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/rocky'
+    --run-command 'chmod 0440 /etc/sudoers.d/rocky'
+    --run-command 'systemctl enable sshd'
+    --ssh-inject root:file:{{ ssh_public_key_path }}
+    --ssh-inject rocky:file:{{ ssh_public_key_path }}
+    --root-password password:{{ root_password }}
+    --selinux-relabel
+  environment:
+    LIBGUESTFS_BACKEND: direct
+  become: true
--- a/ansible/roles/provision_vm/defaults/main.yml
+++ b/ansible/roles/provision_vm/defaults/main.yml
@@ -0,0 +1,18 @@
+---
+# Default variables for provision_vm role
+# Most settings come from inventory (group_vars/libvirt.yml)
+
+# VM name (required, passed from playbook/tasks)
+vm_name: ""
+
+# Maximum wait time for IP (seconds)
+max_wait_ip: 30
+
+# OS variant for virt-install
+os_variant: "rocky9-unknown"
+
+# Use transient VM (doesn't survive reboot)
+vm_transient: true
+
+# Return variable name for VM IP
+vm_ip_var: "provisioned_vm_ip"
--- a/ansible/roles/provision_vm/tasks/main.yml
+++ b/ansible/roles/provision_vm/tasks/main.yml
@@ -0,0 +1,82 @@
+---
+# Tasks for provision_vm role
+
+- name: Validate VM name
+  fail:
+    msg: "vm_name is required"
+  when: vm_name == ""
+
+- name: Validate golden image path
+  stat:
+    path: "{{ golden_image_path }}"
+  register: golden_image_stat
+  failed_when: not golden_image_stat.stat.exists
+
+- name: Set VM disk path
+  set_fact:
+    vm_disk_path: "/var/lib/libvirt/images/{{ vm_name }}.qcow2"
+
+- name: Display provisioning info
+  debug:
+    msg: "Creating VM: {{ vm_name }}"
+
+- name: Create linked clone overlay disk
+  command: >
+    qemu-img create -f qcow2
+    -b {{ golden_image_path }}
+    -F qcow2
+    {{ vm_disk_path }}
+  become: true
+  register: disk_created
+  changed_when: true
+
+- name: Build virt-install command
+  set_fact:
+    virt_install_cmd: >
+      virt-install
+      --name {{ vm_name }}
+      --memory {{ vm_memory }}
+      --vcpus {{ vm_vcpus }}
+      --disk path={{ vm_disk_path }},format=qcow2
+      --import
+      --os-variant {{ os_variant }}
+      --network network={{ vm_network }}
+      --noautoconsole
+      --wait 0
+      {% if vm_transient %}--transient{% endif %}
+
+- name: Start VM with virt-install
+  command: "{{ virt_install_cmd }}"
+  become: true
+  register: vm_started
+  changed_when: true
+  failed_when: vm_started.rc != 0
+
+- name: Wait for VM to obtain IP address
+  shell: >
+    virsh -c qemu:///system domifaddr {{ vm_name }} --source lease 2>/dev/null |
+    awk '/ipv4/ {print $4}' | cut -d/ -f1 | head -1
+  become: true
+  register: vm_ip_result
+  until: vm_ip_result.stdout != "" and vm_ip_result.stdout != "0.0.0.0"
+  retries: "{{ max_wait_ip }}"
+  delay: 2
+  changed_when: false
+
+- name: Set VM IP fact
+  set_fact:
+    "{{ vm_ip_var }}": "{{ vm_ip_result.stdout }}"
+
+- name: Display VM IP
+  debug:
+    msg: "IP obtained: {{ vm_ip_result.stdout }}"
+
+- name: Export VM IP and name
+  set_stats:
+    data:
+      "{{ vm_ip_var }}": "{{ vm_ip_result.stdout }}"
+      provisioned_vm_name: "{{ vm_name }}"
+
+- name: Register VM for cleanup
+  set_fact:
+    provisioned_vms: "{{ provisioned_vms | default([]) + [{'name': vm_name, 'ip': vm_ip_result.stdout, 'disk': vm_disk_path}] }}"
--- a/ansible/roles/run_test/defaults/main.yml
+++ b/ansible/roles/run_test/defaults/main.yml
@@ -0,0 +1,15 @@
+---
+# Default variables for run_test role
+# Most settings come from inventory (group_vars)
+
+# Test name (required, passed from playbook/tasks)
+test_name: ""
+
+# Test repository URL (required, passed from playbook/tasks)
+test_repo_url: ""
+
+# Test repository branch (passed from playbook/tasks)
+test_repo_branch: "main"
+
+# Test timeout (passed from playbook/tasks)
+test_timeout: 900
--- a/ansible/roles/run_test/tasks/main.yml
+++ b/ansible/roles/run_test/tasks/main.yml
@@ -0,0 +1,164 @@
+---
+# Tasks for run_test role
+
+- name: Validate test parameters
+  fail:
+    msg: "{{ item.msg }}"
+  when: item.condition
+  loop:
+    - { condition: "{{ test_name == '' }}", msg: "test_name is required" }
+    - { condition: "{{ test_repo_url == '' }}", msg: "test_repo_url is required" }
+
+- name: Generate unique VM name
+  set_fact:
+    test_vm_name: "{{ test_name }}-{{ ansible_date_time.epoch }}"
+
+- name: Create working directory
+  file:
+    path: "{{ work_dir }}/{{ test_vm_name }}"
+    state: directory
+    mode: '0755'
+
+- name: Display test info
+  debug:
+    msg:
+      - "Running test: {{ test_name }}"
+      - "Repository: {{ test_repo_url }}"
+      - "Branch: {{ test_repo_branch }}"
+      - "VM: {{ test_vm_name }}"
+
+# Provision VM
+- name: Provision test VM
+  include_role:
+    name: provision_vm
+  vars:
+    vm_name: "{{ test_vm_name }}"
+    vm_ip_var: "test_vm_ip"
+
+- name: Set VM IP variable
+  set_fact:
+    vm_ip: "{{ test_vm_ip }}"
+
+- name: Display VM information
+  debug:
+    msg: "VM ready at {{ vm_ip }}"
+
+# Wait for SSH
+- name: Wait for SSH to be ready
+  wait_for:
+    host: "{{ vm_ip }}"
+    port: 22
+    timeout: 60
+    state: started
+  delegate_to: localhost
+
+- name: Test SSH connection
+  command: >
+    ssh -i {{ ssh_private_key_path }}
+    -o StrictHostKeyChecking=no
+    -o ConnectTimeout=5
+    -o UserKnownHostsFile=/dev/null
+    {{ ssh_user }}@{{ vm_ip }}
+    'echo SSH ready'
+  register: ssh_test
+  until: ssh_test.rc == 0
+  retries: 30
+  delay: 2
+  changed_when: false
+
+# Clone test repository
+- name: Clone test repository
+  git:
+    repo: "{{ test_repo_url }}"
+    dest: "{{ work_dir }}/{{ test_vm_name }}/test-repo"
+    version: "{{ test_repo_branch }}"
+  register: repo_cloned
+
+- name: Find sparrowfile
+  find:
+    paths: "{{ work_dir }}/{{ test_vm_name }}/test-repo"
+    patterns:
+      - "main.raku"
+      - "sparrowfile"
+    recurse: true
+  register: sparrowfile_search
+
+- name: Validate sparrowfile exists
+  fail:
+    msg: "No sparrowfile or main.raku found in test repository"
+  when: sparrowfile_search.files | length == 0
+
+- name: Set sparrowfile path
+  set_fact:
+    sparrowfile_path: "{{ sparrowfile_search.files[0].path }}"
+
+- name: Display sparrowfile path
+  debug:
+    msg: "Found sparrowfile: {{ sparrowfile_path }}"
+
+# Run Sparrowdo test
+- name: Build sparrowdo command
+  set_fact:
+    sparrowdo_cmd: >
+      sparrowdo
+      --host={{ vm_ip }}
+      --ssh_user={{ ssh_user }}
+      --ssh_private_key={{ ssh_private_key_path }}
+      --ssh_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+      {% if sparrowdo_no_sudo %}--no_sudo{% endif %}
+      --sparrowfile={{ sparrowfile_path }}
+      {% if sparrowdo_verbose %}--verbose{% endif %}
+      {% if sparrowdo_color %}--color{% endif %}
+
+- name: Create logs directory
+  file:
+    path: "{{ logs_dir }}/{{ test_vm_name }}"
+    state: directory
+    mode: '0755'
+  when: save_logs
+
+- name: Run Sparrowdo test
+  shell: "{{ sparrowdo_cmd }} 2>&1 | tee {{ logs_dir }}/{{ test_vm_name }}/test.log"
+  args:
+    executable: /bin/bash
+  register: sparrowdo_result
+  timeout: "{{ sparrowdo_timeout }}"
+  when: save_logs
+
+- name: Run Sparrowdo test (without logging)
+  command: "{{ sparrowdo_cmd }}"
+  register: sparrowdo_result_nolog
+  timeout: "{{ sparrowdo_timeout }}"
+  when: not save_logs
+
+- name: Display test result
+  debug:
+    msg: "Test {{ test_name }} completed successfully"
+
+- name: Cleanup test VM
+  include_role:
+    name: cleanup_vm
+  vars:
+    vm_name: "{{ test_vm_name }}"
+  when: cleanup_after_test
+
+- name: Archive test results
+  set_fact:
+    test_results: "{{ test_results | default([]) + [{'name': test_name, 'status': 'passed', 'vm': test_vm_name, 'log': logs_dir + '/' + test_vm_name + '/test.log'}] }}"
+  when: save_logs
+
+# Error handling
+- name: Handle test failure
+  block:
+    - name: Archive failed test logs
+      set_fact:
+        test_results: "{{ test_results | default([]) + [{'name': test_name, 'status': 'failed', 'vm': test_vm_name, 'log': logs_dir + '/' + test_vm_name + '/test.log'}] }}"
+      when: save_logs
+
+    - name: Cleanup VM on failure
+      include_role:
+        name: cleanup_vm
+      vars:
+        vm_name: "{{ test_vm_name }}"
+      when: cleanup_after_test
+  when: sparrowdo_result is failed or sparrowdo_result_nolog is failed
--- a/ansible/tasks/run-test-with-infrastructure.yml
+++ b/ansible/tasks/run-test-with-infrastructure.yml
@@ -0,0 +1,102 @@
+---
+# Provider-aware test execution with infrastructure management
+# This task file handles the complete lifecycle:
+#   1. Provision infrastructure (VM/instance based on provider)
+#   2. Run test (provider-agnostic)
+#   3. Collect results
+#   4. Cleanup infrastructure
+
+- name: "Test: {{ test_item.name }}"
+  block:
+    # ===========================================
+    # PROVISION INFRASTRUCTURE (Provider-specific)
+    # ===========================================
+
+    # LIBVIRT: Create linked clone from golden image
+    - name: "[libvirt] Provision VM from golden image"
+      include_role:
+        name: provision_vm
+      vars:
+        vm_name: "test-{{ test_item.name }}-{{ ansible_date_time.epoch }}"
+        test_name: "{{ test_item.name }}"
+      when: provider == 'libvirt'
+
+    # AWS: Launch EC2 instance (future implementation)
+    - name: "[aws] Launch EC2 instance"
+      debug:
+        msg: "TODO: Launch EC2 instance for {{ test_item.name }}"
+      when: provider == 'aws'
+    # TODO: Add actual AWS provisioning role when ready
+    # include_role:
+    #   name: provision_aws_instance
+    # vars:
+    #   instance_name: "test-{{ test_item.name }}-{{ ansible_date_time.epoch }}"
+
+    # AZURE: Create VM (future implementation)
+    - name: "[azure] Create Azure VM"
+      debug:
+        msg: "TODO: Create Azure VM for {{ test_item.name }}"
+      when: provider == 'azure'
+    # TODO: Add actual Azure provisioning role when ready
+
+    # ===========================================
+    # RUN TEST (Provider-agnostic)
+    # ===========================================
+
+    - name: "Execute test: {{ test_item.name }}"
+      include_role:
+        name: run_test
+      vars:
+        test_name: "{{ test_item.name }}"
+        test_repo_url: "{{ test_item.repo_url }}"
+        test_repo_branch: "{{ test_item.branch | default(default_test_branch) }}"
+        test_timeout: "{{ test_item.timeout | default(default_test_timeout) }}"
+
+    # Record success
+    - name: Record test success
+      set_fact:
+        test_results: "{{ test_results + [{'name': test_item.name, 'status': 'passed'}] }}"
+
+  rescue:
+    # Record failure
+    - name: Record test failure
+      set_fact:
+        test_results: "{{ test_results + [{'name': test_item.name, 'status': 'failed', 'error': ansible_failed_result.msg | default('Unknown error')}] }}"
+
+    - name: Display test failure
+      debug:
+        msg: "Test {{ test_item.name }} FAILED: {{ ansible_failed_result.msg | default('Unknown error') }}"
+
+  always:
+    # ===========================================
+    # CLEANUP INFRASTRUCTURE (Provider-specific)
+    # ===========================================
+
+    # LIBVIRT: Destroy VM clone
+    - name: "[libvirt] Cleanup VM"
+      include_role:
+        name: cleanup_vm
+      vars:
+        vm_name: "test-{{ test_item.name }}-{{ ansible_date_time.epoch }}"
+      when: provider == 'libvirt'
+      ignore_errors: true
+
+    # AWS: Terminate EC2 instance (future implementation)
+    - name: "[aws] Terminate EC2 instance"
+      debug:
+        msg: "TODO: Terminate EC2 instance for {{ test_item.name }}"
+      when: provider == 'aws'
+      ignore_errors: true
+    # TODO: Add actual AWS cleanup role when ready
+
+    # AZURE: Delete VM (future implementation)
+    - name: "[azure] Delete Azure VM"
+      debug:
+        msg: "TODO: Delete Azure VM for {{ test_item.name }}"
+      when: provider == 'azure'
+      ignore_errors: true
+    # TODO: Add actual Azure cleanup role when ready
+
+    - name: Test completed
+      debug:
+        msg: "Finished test: {{ test_item.name }}"
--- a/ansible/vars/test-definitions.yml
+++ b/ansible/vars/test-definitions.yml
@@ -0,0 +1,23 @@
+---
+default_test_branch: "main"        # Default git branch
+default_test_timeout: 300          # Default timeout in seconds (15 min)
+default_test_filter: ".*"          # Regex filter (matches all tests)
+
+# Test execution behavior
+retry_failed_tests: false
+continue_on_failure: true          # Keep running tests even if one fails
+
+# ===========================================
+# TEST DEFINITIONS
+# ===========================================
+# Required fields:
+#   - name: Unique test identifier
+#   - repo_url: Git repository URL
+#
+# Optional fields (uses defaults above if omitted):
+#   - branch: Git branch (default: main)
+#   - timeout: Test timeout in seconds (default: 300)
+
+tests:
+  - name: "Sparky_Knot"
+    repo_url: "https://git.resf.org/testing/Sparky_Knot.git"
--- a/docs/ANSIBLE-GUIDE.md
+++ b/docs/ANSIBLE-GUIDE.md
@@ -0,0 +1,562 @@
+# Rocky Linux Testing Framework - Ansible Guide
+
+This guide covers the Ansible-based implementation of the Rocky Linux Testing Framework.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Quick Start](#quick-start)
+- [Ansible Structure](#ansible-structure)
+- [Playbooks](#playbooks)
+- [Roles](#roles)
+- [Configuration](#configuration)
+- [Usage Examples](#usage-examples)
+- [Advanced Usage](#advanced-usage)
+- [Troubleshooting](#troubleshooting)
+
+## Overview
+
+The Ansible implementation provides the same functionality as the shell scripts, but with:
+
+- Better error handling and idempotency
+- Structured configuration management
+- Parallel test execution with better control
+- Reusable roles for composability
+
+## Prerequisites
+
+```bash
+# Install required packages
+sudo dnf install -y ansible qemu-kvm libvirt virt-install guestfs-tools rakudo
+
+# Enable libvirtd
+sudo systemctl enable --now libvirtd
+sudo usermod -a -G libvirt $(whoami)
+
+# Generate SSH keys if not present
+ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N ""
+
+# Install Ansible collections (if needed)
+ansible-galaxy collection install community.libvirt
+```
+
+## Quick Start
+
+### 1. Build Golden Image
+
+```bash
+cd ansible
+
+# Build with defaults (Rocky 9)
+ansible-playbook playbooks/build-golden-image.yml
+
+# Build with custom URL
+ansible-playbook playbooks/build-golden-image.yml \
+  -e "qcow2_url=https://download.rockylinux.org/pub/rocky/8/images/x86_64/Rocky-8-GenericCloud-Base.latest.x86_64.qcow2"
+```
+
+### 2. Run Single Test
+
+```bash
+ansible-playbook playbooks/run-single-test.yml \
+  -e "test_name=my-test" \
+  -e "test_repo_url=https://github.com/your-org/test-repo.git"
+```
+
+### 3. Run Test Suite
+
+```bash
+# Edit playbooks/run-test-suite.yml to define your test matrix
+# Then run:
+ansible-playbook playbooks/run-test-suite.yml
+```
+
+## Ansible Structure
+
+```
+ansible/
+├── ansible.cfg                 # Ansible configuration
+├── inventory/
+│   ├── hosts.yml              # Inventory file
+│   └── group_vars/
+│       └── all.yml            # Global variables
+├── playbooks/
+│   ├── build-golden-image.yml # Build golden image workflow
+│   ├── run-single-test.yml    # Run single test
+│   └── run-test-suite.yml     # Run multiple tests
+└── roles/
+    ├── download_image/        # Download and cache QCOW2
+    ├── golden_image/          # Build golden image
+    ├── bootstrap_sparrowdo/   # Bootstrap Sparrowdo
+    ├── provision_vm/          # Provision test VM
+    ├── run_test/              # Run Sparrowdo test
+    └── cleanup_vm/            # Cleanup VMs
+```
+
+## Playbooks
+
+### build-golden-image.yml
+
+Complete workflow to create a golden image ready for testing.
+
+**Variables:**
+- `qcow2_url`: URL of base QCOW2 image
+- `images_dir`: Directory for images (default: /var/lib/libvirt/images)
+- `golden_image_path`: Output path for golden image
+- `force_download`: Force re-download of base image
+
+**Example:**
+```bash
+ansible-playbook playbooks/build-golden-image.yml \
+  -e "qcow2_url=https://..." \
+  -e "force_download=true"
+```
+
+### run-single-test.yml
+
+Run a single Sparrowdo test against the golden image.
+
+**Required Variables:**
+- `test_name`: Name of the test
+- `test_repo_url`: Git repository URL containing the test
+
+**Optional Variables:**
+- `test_repo_branch`: Branch to use (default: main)
+- `golden_image_path`: Path to golden image
+
+**Example:**
+```bash
+ansible-playbook playbooks/run-single-test.yml \
+  -e "test_name=webserver-test" \
+  -e "test_repo_url=https://github.com/org/webserver-test.git" \
+  -e "test_repo_branch=develop"
+```
+
+### run-test-suite.yml
+
+Run multiple tests in parallel from a test matrix.
+
+**Variables:**
+- `test_matrix`: List of tests to run
+- `max_parallel`: Maximum parallel tests
+- `test_filter`: Regex to filter tests
+
+**Example:**
+```yaml
+# Edit the playbook to define your tests:
+test_matrix:
+  - name: "ssh-test"
+    url: "https://github.com/org/ssh-test.git"
+    branch: "main"
+    description: "SSH service validation"
+  - name: "network-test"
+    url: "https://github.com/org/network-test.git"
+    branch: "main"
+    description: "Network configuration test"
+```
+
+Then run:
+```bash
+ansible-playbook playbooks/run-test-suite.yml
+
+# Or filter tests:
+ansible-playbook playbooks/run-test-suite.yml -e "test_filter=ssh.*"
+```
+
+## Roles
+
+### download_image
+
+Downloads and caches QCOW2 images.
+
+**Variables:**
+- `qcow2_url`: URL to download
+- `images_dir`: Cache directory
+- `force_download`: Force re-download
+- `image_path_var`: Variable name to store result
+
+**Example:**
+```yaml
+- include_role:
+    name: download_image
+  vars:
+    qcow2_url: "https://..."
+    image_path_var: "my_image_path"
+```
+
+### golden_image
+
+Creates a golden image using virt-customize.
+
+**Variables:**
+- `base_image_path`: Source QCOW2 image (required)
+- `golden_image_path`: Output path
+- `ssh_public_key_path`: SSH key to inject
+- `custom_prep_script`: Custom preparation script
+- `use_default_prep`: Use default prep script
+
+**Example:**
+```yaml
+- include_role:
+    name: golden_image
+  vars:
+    base_image_path: "/path/to/base.qcow2"
+    golden_image_path: "/path/to/golden.qcow2"
+```
+
+### bootstrap_sparrowdo
+
+Bootstraps Sparrowdo on a golden image.
+
+**Variables:**
+- `golden_image_path`: Golden image to bootstrap
+- `ssh_private_key_path`: SSH key for connection
+- `ssh_user`: User to connect as
+- `vm_boot_timeout`: Boot timeout in seconds
+- `bootstrap_timeout`: Bootstrap timeout
+
+**Example:**
+```yaml
+- include_role:
+    name: bootstrap_sparrowdo
+  vars:
+    golden_image_path: "/path/to/golden.qcow2"
+```
+
+### provision_vm
+
+Provisions a VM as a linked clone.
+
+**Variables:**
+- `vm_name`: VM name (required)
+- `golden_image_path`: Base image
+- `vm_memory`: Memory in MB
+- `vm_vcpus`: Number of vCPUs
+- `max_wait_ip`: Timeout for IP assignment
+- `vm_ip_var`: Variable name for returned IP
+
+**Example:**
+```yaml
+- include_role:
+    name: provision_vm
+  vars:
+    vm_name: "test-vm-1"
+    vm_ip_var: "test_vm_ip"
+
+- debug:
+    msg: "VM IP: {{ test_vm_ip }}"
+```
+
+### run_test
+
+Runs a complete test: provision -> test -> cleanup.
+
+**Variables:**
+- `test_name`: Test name (required)
+- `test_repo_url`: Git repository (required)
+- `test_repo_branch`: Branch to use
+- `golden_image_path`: Golden image
+- `cleanup_after_test`: Cleanup VM after test
+- `save_logs`: Save test logs
+
+**Example:**
+```yaml
+- include_role:
+    name: run_test
+  vars:
+    test_name: "my-test"
+    test_repo_url: "https://github.com/org/test.git"
+```
+
+### cleanup_vm
+
+Cleans up VMs and disk images.
+
+**Variables:**
+- `vm_name`: Single VM to cleanup
+- `cleanup_pattern`: Regex pattern for multiple VMs
+- `cleanup_vm_list`: List of VMs to cleanup
+- `force_destroy`: Force destroy running VMs
+- `remove_disk`: Remove disk images
+
+**Example:**
+```yaml
+# Cleanup single VM
+- include_role:
+    name: cleanup_vm
+  vars:
+    vm_name: "test-vm-1"
+
+# Cleanup by pattern
+- include_role:
+    name: cleanup_vm
+  vars:
+    cleanup_pattern: "test-.*"
+
+# Cleanup list
+- include_role:
+    name: cleanup_vm
+  vars:
+    cleanup_vm_list:
+      - "vm1"
+      - "vm2"
+```
+
+## Configuration
+
+### Inventory Variables
+
+Edit `ansible/inventory/group_vars/all.yml`:
+
+```yaml
+# VM defaults
+vm_memory: 4096        # Increase for heavy tests
+vm_vcpus: 4
+
+# Parallel execution
+max_parallel: 5        # Run 5 tests concurrently
+
+# Sparrowdo options
+sparrowdo_timeout: 1200  # Increase timeout
+sparrowdo_verbose: false # Less output
+```
+
+### Per-Test Configuration
+
+Override variables when running playbooks:
+
+```bash
+ansible-playbook playbooks/run-single-test.yml \
+  -e "test_name=my-test" \
+  -e "test_repo_url=https://..." \
+  -e "vm_memory=4096" \
+  -e "vm_vcpus=4" \
+  -e "sparrowdo_timeout=1800"
+```
+
+## Usage Examples
+
+### Example 1: Build Golden Image for Rocky 8
+
+```bash
+ansible-playbook playbooks/build-golden-image.yml \
+  -e "qcow2_url=https://download.rockylinux.org/pub/rocky/8/images/x86_64/Rocky-8-GenericCloud-Base.latest.x86_64.qcow2" \
+  -e "golden_image_path=/var/lib/libvirt/images/golden-rocky8.qcow2"
+```
+
+### Example 2: Run Test with Custom Resources
+
+```bash
+ansible-playbook playbooks/run-single-test.yml \
+  -e "test_name=heavy-test" \
+  -e "test_repo_url=https://github.com/org/test.git" \
+  -e "vm_memory=8192" \
+  -e "vm_vcpus=8"
+```
+
+### Example 3: Custom Test Suite
+
+Create `my-test-suite.yml`:
+
+```yaml
+---
+- name: My Custom Test Suite
+  hosts: localhost
+  connection: local
+  gather_facts: true
+  
+  vars:
+    golden_image_path: "/var/lib/libvirt/images/golden.qcow2"
+    
+    my_tests:
+      - name: "database-test"
+        url: "https://github.com/org/db-test.git"
+      - name: "web-test"
+        url: "https://github.com/org/web-test.git"
+  
+  tasks:
+    - name: Run each test
+      include_role:
+        name: run_test
+      vars:
+        test_name: "{{ item.name }}"
+        test_repo_url: "{{ item.url }}"
+      loop: "{{ my_tests }}"
+```
+
+Run it:
+```bash
+ansible-playbook my-test-suite.yml
+```
+
+### Example 4: Custom Golden Image Preparation
+
+Create custom prep script `custom-prep.sh`:
+
+```bash
+#!/bin/bash
+dnf install -y postgresql nginx redis
+systemctl enable postgresql nginx redis
+```
+
+Then:
+
+```bash
+ansible-playbook playbooks/build-golden-image.yml \
+  -e "custom_prep_script=/path/to/custom-prep.sh"
+```
+
+## Advanced Usage
+
+### Using Tags
+
+Add tags to control execution:
+
+```bash
+# Only download image
+ansible-playbook playbooks/build-golden-image.yml --tags download
+
+# Skip bootstrap
+ansible-playbook playbooks/build-golden-image.yml --skip-tags bootstrap
+```
+
+### Ansible Vault for Secrets
+
+Store sensitive data in vault:
+
+```bash
+ansible-vault create ansible/inventory/group_vars/secrets.yml
+```
+
+Add passwords:
+```yaml
+root_password: "my-secure-password"
+rocky_user_password: "another-secure-password"
+```
+
+Use it:
+```bash
+ansible-playbook playbooks/build-golden-image.yml --ask-vault-pass
+```
+
+### Cleanup Orphaned VMs
+
+```bash
+ansible-playbook -m include_role -a name=cleanup_vm \
+  -e "cleanup_pattern=test-.*" \
+  localhost,
+```
+
+## Troubleshooting
+
+### Check Ansible Syntax
+
+```bash
+ansible-playbook playbooks/build-golden-image.yml --syntax-check
+```
+
+### Dry Run (Check Mode)
+
+```bash
+ansible-playbook playbooks/build-golden-image.yml --check
+```
+
+### Verbose Output
+
+```bash
+ansible-playbook playbooks/run-single-test.yml -vvv \
+  -e "test_name=my-test" \
+  -e "test_repo_url=https://..."
+```
+
+### List Tasks
+
+```bash
+ansible-playbook playbooks/build-golden-image.yml --list-tasks
+```
+
+### Debugging Failed Tests
+
+```bash
+# Keep VM after failure
+ansible-playbook playbooks/run-single-test.yml \
+  -e "test_name=my-test" \
+  -e "test_repo_url=https://..." \
+  -e "cleanup_after_test=false"
+
+# Then SSH to investigate
+ssh -i ~/.ssh/id_rsa rocky@<vm-ip>
+```
+
+### Common Issues
+
+**Issue: "Golden image not found"**
+```bash
+# Build golden image first
+ansible-playbook playbooks/build-golden-image.yml
+```
+
+**Issue: "VM failed to get IP"**
+```bash
+# Check libvirt network
+sudo virsh net-list --all
+sudo virsh net-start default
+
+# Increase timeout
+ansible-playbook ... -e "max_wait_ip=60"
+```
+
+**Issue: "Bootstrap timeout"**
+```bash
+# Increase timeout
+ansible-playbook playbooks/build-golden-image.yml \
+  -e "bootstrap_timeout=1800"
+```
+
+**Issue: "Parallel tests failing"**
+```bash
+# Reduce parallelism
+ansible-playbook playbooks/run-test-suite.yml \
+  -e "max_parallel=1"
+```
+
+## Performance Tips
+
+1. **Cache base images**: First download is slow, subsequent runs are fast
+2. **Reuse golden images**: Build once, test many times
+3. **Tune parallel execution**: Balance between speed and resource usage
+4. **Use local mirrors**: Speed up package installation in prep scripts
+5. **Disable verbose logging**: For faster execution in production
+
+## Migration from Shell Scripts
+
+| Shell Script | Ansible Equivalent |
+|--------------|-------------------|
+| `download-image.sh` | Role: `download_image` |
+| `setup_base.sh` | Role: `golden_image` |
+| `bootstrap_golden.sh` | Role: `bootstrap_sparrowdo` |
+| `provision_vm.sh` | Role: `provision_vm` |
+| `run-test.sh` | Role: `run_test` |
+| `cleanup_vm.sh` | Role: `cleanup_vm` |
+| `cleanup-all.sh` | Role: `cleanup_vm` with pattern |
+| Manual workflow | Playbook: `build-golden-image.yml` |
+| Test execution | Playbooks: `run-single-test.yml`, `run-test-suite.yml` |
+
+## Contributing
+
+When adding new functionality:
+
+1. Create a new role in `ansible/roles/`
+2. Add defaults in `defaults/main.yml`
+3. Document variables in role README
+4. Create example playbook in `playbooks/`
+5. Update this documentation
+
+## License
+
+[Your License Here]
+
+## Authors
+
+Rocky Linux Testing Team
--- a/docs/BOOTSTRAP-APPROACH.md
+++ b/docs/BOOTSTRAP-APPROACH.md
@@ -108,30 +108,21 @@ sparrowdo --host $VM_IP --ssh_user rocky --no_sudo --sparrowfile test.raku
 ./scripts/cleanup_vm.sh test-vm-1
 ```

-## Jenkins Pipeline Flow
+## Workflow

-The Jenkinsfile automatically handles bootstrap:
+Bootstrap is handled automatically in the build process:

-```groovy
-stage('Prepare Golden Image') {
-    // Creates golden image with Raku/zef
-    setup_base.sh → golden.qcow2 (with Raku)
-}
+1. **Prepare Golden Image**
+   - setup_base.sh → golden.qcow2 (with Raku)

-stage('Bootstrap Golden Image') {
-    // Bootstraps Sparrowdo ONCE
-    bootstrap_golden.sh → golden.qcow2 (with Sparrowdo)
-}
+2. **Bootstrap Golden Image**
+   - bootstrap_golden.sh → golden.qcow2 (with Sparrowdo)

-stage('Run Tests') {
-    parallel {
-        test1: provision → run test → cleanup
-        test2: provision → run test → cleanup
-        test3: provision → run test → cleanup
-        // No bootstrap in any test!
-    }
-}
-```
+3. **Run Tests in Parallel**
+   - provision → run test → cleanup
+   - provision → run test → cleanup
+   - provision → run test → cleanup
+   - (No bootstrap in any test!)

 ## Time Savings Example

@@ -272,7 +263,7 @@ The bootstrap script outputs:

 If any step fails, the golden image is NOT bootstrapped. Check logs and retry.

-## Integration with CI/CD
+## Integration with Automation

 ### Nightly Golden Image Rebuild
 ```bash
@@ -287,10 +278,11 @@ If any step fails, the golden image is NOT bootstrapped. Check logs and retry.
 # Provisions temp VM, runs one test, destroys
 ```

-### Jenkins Scheduled Build
-```groovy
-// Rebuild golden images weekly
-cron('H 2 * * 0')  // Sunday 2 AM
+### Scheduled Builds
+Use your automation tool (cron, systemd timers, etc.) to rebuild golden images periodically:
+```bash
+# Weekly golden image rebuild
+0 2 * * 0 /path/to/repo/ansible/playbooks/build-golden-image.yml
 ```

 ## Conclusion
--- a/docs/CHANGES.md
+++ b/docs/CHANGES.md
@@ -93,7 +93,7 @@ sparrowdo \
 ### 3. Virsh Connection Issues ✅
 - **Problem**: Non-root users couldn't access libvirt without explicit URI
 - **Solution**: Added `-c qemu:///system` to all virsh commands
- **Impact**: Scripts work for jenkins user and other non-root users
+- **Impact**: Scripts work for non-root users

 ### 4. Image Caching ✅
 - **Problem**: Framework re-downloaded QCOW2 images on every build
@@ -109,17 +109,14 @@ sparrowdo \
 ## File Changes

 ### Modified Files
- `Jenkinsfile` - Added bootstrap step, rocky user, main.raku support
 - `scripts/setup_base.sh` - Inject SSH keys for rocky user
 - `scripts/provision_vm.sh` - Fixed set -e issue, improved error handling
 - `scripts/cleanup_vm.sh` - Added explicit qemu:///system connection
 - `docs/default-prep.sh` - Create rocky user, remove D-Bus commands
- `docs/manual-test-run.sh` - Add bootstrap step, use rocky user
 - `README.md` - Updated prerequisites (guestfs-tools)

 ### New Files
 - `docs/virt-customize-guide.md` - Comprehensive guide on offline image customization
- `docs/manual-steps.md` - Step-by-step manual testing guide
 - `docs/CHANGES.md` - This file

 ## Testing Checklist
@@ -162,20 +159,11 @@ sparrowdo --host $VM_IP --ssh_user rocky --bootstrap --color
 ### For Existing Tests
 If you have existing Sparrowdo tests that assumed root user:

-1. **Update TEST_MATRIX** in Jenkins to use rocky user
+1. **Ensure tests use rocky user** - Update SSH commands to use rocky@host
 2. **Ensure tests use --no_sudo** flag
 3. **Add bootstrap step** before test execution
 4. **Verify sudoers access** if tests need elevated privileges

-### Jenkins Pipeline Changes
-The Jenkinsfile automatically handles:
- Creating rocky user in golden image
- Injecting SSH keys for rocky user
- Running bootstrap before tests
- Using `--no_sudo` flag with rocky user
-
-No manual intervention needed for Jenkins builds.
-
 ## Performance Improvements

 ### Image Caching
--- a/docs/default-prep.sh
+++ b/docs/default-prep.sh
@@ -1,61 +0,0 @@
-#!/bin/bash
-set -e
-
-echo "Preparing standard golden image..."
-
-# NOTE: This script runs inside virt-customize (offline mode)
-# Cannot use systemctl, firewall-cmd, or other D-Bus dependent commands
-# Use systemctl enable only (works offline), or direct file manipulation
-
-# Update system (optional - can be slow)
-# Uncomment if you want latest packages:
-# dnf update -y
-
-# Install common testing dependencies including Raku/Sparrowdo
-dnf install -y \
-    perl \
-    git \
-    wget \
-    tar \
-    openssh-server \
-    vim \
-    rakudo \
-    rakudo-zef
-
-# Enable services (these work in offline mode)
-# systemctl enable works by creating symlinks, no D-Bus needed
-systemctl enable sshd
-
-# Create rocky user (standard non-root user for Rocky Linux)
-useradd -m rocky 2>/dev/null || true
-echo "rocky:rockypass" | chpasswd
-
-# Add rocky user to sudoers
-echo "rocky ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/rocky
-chmod 0440 /etc/sudoers.d/rocky
-
-# Install Sparrowdo for rocky user
-# Note: This needs to run as rocky user, but we're in offline mode
-# So we prepare the environment, and actual Sparrowdo install happens on first boot
-mkdir -p /home/rocky/.sparrowdo-bootstrap
-cat > /home/rocky/.sparrowdo-bootstrap/install.sh << 'BOOTSTRAP_EOF'
-#!/bin/bash
-# This script will be run on first boot by rocky user
-if [ ! -f /home/rocky/.sparrowdo-installed ]; then
-    zef install --/test Sparrowdo
-    touch /home/rocky/.sparrowdo-installed
-fi
-BOOTSTRAP_EOF
-chmod +x /home/rocky/.sparrowdo-bootstrap/install.sh
-chown -R rocky:rocky /home/rocky/.sparrowdo-bootstrap
-
-# Create testuser for backward compatibility
-useradd -m testuser 2>/dev/null || true
-echo "testuser:testpass" | chpasswd
-
-# Add testuser to sudoers
-echo "testuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/testuser
-chmod 0440 /etc/sudoers.d/testuser
-
-echo "Golden image preparation complete!"
-echo "NOTE: Sparrowdo will be installed on first SSH connection by rocky user"
--- a/docs/manual-steps.md
+++ b/docs/manual-steps.md
@@ -1,431 +0,0 @@
-# Manual Test Run - Step by Step
-
-This guide provides the exact commands Jenkins runs, so you can execute them manually for testing.
-
-## Quick Start - Automated Script
-
-I've created an interactive script that walks through all steps:
-
-```bash
-cd /Users/ssimpson/Documents/git/resf-test-jenkins
-
-# Edit the configuration in the script first:
-vim docs/manual-test-run.sh
-# Update: TEST_REPO_URL, TEST_REPO_BRANCH, TEST_NAME
-
-# Run the interactive script:
-./docs/manual-test-run.sh
-```
-
-The script will pause at each step so you can see what's happening.
-
---
-
-## Manual Step-by-Step Commands
-
-If you prefer to run each command manually, here's the exact sequence:
-
-### Prerequisites Check
-
-```bash
-# Verify required tools
-which qemu-img virsh virt-install virt-customize sparrowdo git curl
-
-# Verify libvirt is running
-sudo systemctl status libvirtd
-
-# Verify SSH keys exist
-ls -la ~/.ssh/id_rsa ~/.ssh/id_rsa.pub
-```
-
---
-
-### Step 1: Set Variables
-
-```bash
-# Set your configuration
-export BUILD_ID="manual-$(date +%s)"
-export IMAGES_DIR="/var/lib/libvirt/images"
-export WORK_DIR="/Users/ssimpson/Documents/git/resf-test-jenkins/manual-test-${BUILD_ID}"
-export SCRIPT_DIR="/Users/ssimpson/Documents/git/resf-test-jenkins"
-
-# QCOW2 Image URL
-export QCOW2_URL="https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2"
-
-# SSH Keys
-export SSH_PRIVATE_KEY="${HOME}/.ssh/id_rsa"
-export SSH_PUBLIC_KEY="${HOME}/.ssh/id_rsa.pub"
-
-# Test Configuration - EDIT THESE
-export TEST_NAME="my-test"
-export TEST_REPO_URL="https://github.com/your-org/your-test-repo.git"
-export TEST_REPO_BRANCH="main"
-
-# Image download behavior (set to "true" to force re-download)
-export REDOWNLOAD_IMAGE="false"
-
-echo "Configuration set for build: ${BUILD_ID}"
-```
-
---
-
-### Step 2: Initialize Environment
-
-```bash
-# Create working directory
-mkdir -p "${WORK_DIR}"
-cd "${WORK_DIR}"
-
-# Ensure images directory exists and is writable
-sudo mkdir -p "${IMAGES_DIR}"
-sudo chown ${USER}:${USER} "${IMAGES_DIR}"
-
-# Verify scripts are executable
-chmod +x "${SCRIPT_DIR}/scripts"/*.sh
-
-echo "Environment initialized"
-```
-
---
-
-### Step 3: Download Base QCOW2 Image
-
-```bash
-# Extract filename from URL for caching
-export IMAGE_FILENAME=$(basename "${QCOW2_URL}")
-export CACHED_IMAGE="${IMAGES_DIR}/${IMAGE_FILENAME}"
-export BASE_IMAGE="${IMAGES_DIR}/base-${BUILD_ID}.qcow2"
-
-echo "Image URL: ${QCOW2_URL}"
-echo "Cached image: ${CACHED_IMAGE}"
-echo "Build image: ${BASE_IMAGE}"
-
-if [ "${REDOWNLOAD_IMAGE}" = "true" ]; then
-    echo "REDOWNLOAD_IMAGE is enabled - forcing fresh download"
-    curl -L --progress-bar -o "${BASE_IMAGE}" "${QCOW2_URL}"
-else
-    if [ -f "${CACHED_IMAGE}" ]; then
-        echo "Found cached image, creating copy for this build..."
-        cp "${CACHED_IMAGE}" "${BASE_IMAGE}"
-    else
-        echo "No cached image found, downloading..."
-        curl -L --progress-bar -o "${CACHED_IMAGE}" "${QCOW2_URL}"
-        echo "Creating copy for this build..."
-        cp "${CACHED_IMAGE}" "${BASE_IMAGE}"
-    fi
-fi
-
-# Verify the image
-echo "Verifying image..."
-qemu-img info "${BASE_IMAGE}" | head -10
-
-echo "Base image ready: ${BASE_IMAGE}"
-```
-
---
-
-### Step 4: Prepare Golden Image
-
-```bash
-# Set golden image path
-export GOLDEN_IMAGE="${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"
-
-# Set prep script (you can customize this)
-export PREP_SCRIPT="${SCRIPT_DIR}/docs/default-prep.sh"
-
-echo "Creating golden image..."
-echo "Base: ${BASE_IMAGE}"
-echo "Golden: ${GOLDEN_IMAGE}"
-echo "Prep Script: ${PREP_SCRIPT}"
-echo "SSH Key: ${SSH_PUBLIC_KEY}"
-
-# Run the setup script
-"${SCRIPT_DIR}/scripts/setup_base.sh" \
-    "${BASE_IMAGE}" \
-    "${PREP_SCRIPT}" \
-    "${GOLDEN_IMAGE}" \
-    "${SSH_PUBLIC_KEY}"
-
-echo "Golden image ready: ${GOLDEN_IMAGE}"
-```
-
---
-
-### Step 5: Provision Test VM
-
-```bash
-# Set VM name
-export VM_NAME="${TEST_NAME}-${BUILD_ID}"
-
-echo "Provisioning VM: ${VM_NAME}"
-
-# Provision the VM and capture its IP
-export VM_IP=$("${SCRIPT_DIR}/scripts/provision_vm.sh" "${VM_NAME}" "${GOLDEN_IMAGE}" 60)
-
-if [ "$VM_IP" = "ERROR" ] || [ -z "$VM_IP" ]; then
-    echo "ERROR: Failed to provision VM"
-    exit 1
-fi
-
-echo "VM provisioned successfully"
-echo "VM Name: ${VM_NAME}"
-echo "IP Address: ${VM_IP}"
-
-# Wait for SSH to be ready
-echo "Waiting for SSH to be ready..."
-for i in {1..30}; do
-    if ssh -i "${SSH_PRIVATE_KEY}" \
-           -o StrictHostKeyChecking=no \
-           -o ConnectTimeout=5 \
-           -o UserKnownHostsFile=/dev/null \
-           root@${VM_IP} 'echo "SSH ready"' 2>/dev/null; then
-        echo "SSH connection established!"
-        break
-    fi
-    echo "Attempt $i/30..."
-    sleep 2
-done
-
-# Test SSH connection manually
-echo "Testing SSH connection..."
-ssh -i "${SSH_PRIVATE_KEY}" \
-    -o StrictHostKeyChecking=no \
-    -o UserKnownHostsFile=/dev/null \
-    root@${VM_IP} 'hostname && cat /etc/rocky-release'
-```
-
---
-
-### Step 6: Clone Test Repository
-
-```bash
-# Create test workspace
-export TEST_WORKSPACE="${WORK_DIR}/test-workspace"
-mkdir -p "${TEST_WORKSPACE}"
-cd "${TEST_WORKSPACE}"
-
-echo "Cloning test repository..."
-echo "URL: ${TEST_REPO_URL}"
-echo "Branch: ${TEST_REPO_BRANCH}"
-
-# Clone the repository
-git clone -b "${TEST_REPO_BRANCH}" "${TEST_REPO_URL}" test-repo
-
-# Find the sparrowfile
-export SPARROWFILE=$(find test-repo -name sparrowfile -type f | head -1)
-
-if [ -z "$SPARROWFILE" ]; then
-    echo "ERROR: No sparrowfile found in repository"
-    exit 1
-fi
-
-echo "Found sparrowfile: ${SPARROWFILE}"
-
-# Show sparrowfile contents
-echo "Sparrowfile contents:"
-cat "${SPARROWFILE}"
-```
-
---
-
-### Step 7: Run Sparrowdo Test
-
-```bash
-# Create logs directory
-mkdir -p "${TEST_WORKSPACE}/logs"
-
-echo "=========================================="
-echo "Running Sparrowdo Test"
-echo "=========================================="
-echo "Target: root@${VM_IP}"
-echo "Sparrowfile: ${SPARROWFILE}"
-echo "Log: ${TEST_WORKSPACE}/logs/test.log"
-echo ""
-
-# Run the test
-timeout 900 sparrowdo \
-    --host="${VM_IP}" \
-    --ssh_user=root \
-    --ssh_private_key="${SSH_PRIVATE_KEY}" \
-    --ssh_args="-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null" \
-    --no_sudo \
-    --sparrowfile="${SPARROWFILE}" 2>&1 | tee "${TEST_WORKSPACE}/logs/test.log"
-
-echo ""
-echo "Test completed!"
-echo "Logs saved to: ${TEST_WORKSPACE}/logs/test.log"
-```
-
---
-
-### Step 8: Cleanup
-
-```bash
-echo "=========================================="
-echo "Cleanup"
-echo "=========================================="
-
-# Destroy the VM
-echo "Destroying VM: ${VM_NAME}"
-"${SCRIPT_DIR}/scripts/cleanup_vm.sh" "${VM_NAME}"
-
-# Optional: Remove temporary images
-echo ""
-echo "Temporary images:"
-echo "  Base: ${BASE_IMAGE}"
-echo "  Golden: ${GOLDEN_IMAGE}"
-echo ""
-read -p "Remove temporary images? (y/n): " -n 1 -r
-echo
-
-if [[ $REPLY =~ ^[Yy]$ ]]; then
-    echo "Removing images..."
-    sudo rm -f "${BASE_IMAGE}"
-    sudo rm -f "${GOLDEN_IMAGE}"
-    echo "Images removed"
-else
-    echo "Images preserved at:"
-    echo "  ${BASE_IMAGE}"
-    echo "  ${GOLDEN_IMAGE}"
-fi
-```
-
---
-
-### Step 9: View Results
-
-```bash
-echo "=========================================="
-echo "Test Run Summary"
-echo "=========================================="
-echo "Build ID: ${BUILD_ID}"
-echo "Test Name: ${TEST_NAME}"
-echo "VM Name: ${VM_NAME}"
-echo "Working Directory: ${WORK_DIR}"
-echo "Test Logs: ${TEST_WORKSPACE}/logs/test.log"
-echo ""
-
-# View the test log
-echo "=== Test Log ==="
-cat "${TEST_WORKSPACE}/logs/test.log"
-```
-
---
-
-## Troubleshooting During Manual Run
-
-### VM Won't Start
-```bash
-# Check libvirt status
-sudo systemctl status libvirtd
-
-# List all VMs
-virsh list --all
-
-# Check VM details
-virsh dominfo "${VM_NAME}"
-
-# View VM console
-virsh console "${VM_NAME}"  # Ctrl+] to exit
-```
-
-### Can't Get IP Address
-```bash
-# Check network
-virsh net-list --all
-virsh net-info default
-
-# View DHCP leases
-virsh net-dhcp-leases default
-
-# Try alternative IP detection
-virsh domifaddr "${VM_NAME}" --source lease
-virsh domifaddr "${VM_NAME}" --source agent
-virsh domifaddr "${VM_NAME}" --source arp
-```
-
-### SSH Connection Fails
-```bash
-# Test SSH manually with verbose output
-ssh -vvv -i "${SSH_PRIVATE_KEY}" \
-    -o StrictHostKeyChecking=no \
-    -o UserKnownHostsFile=/dev/null \
-    root@${VM_IP}
-
-# Check if SSH key was injected
-sudo virt-cat -a "${GOLDEN_IMAGE}" /root/.ssh/authorized_keys
-
-# Check VM network from inside
-virsh console "${VM_NAME}"
-# Then inside VM:
-# ip addr
-# systemctl status sshd
-# firewall-cmd --list-all
-```
-
-### Sparrowdo Fails
-```bash
-# Test connectivity first
-ssh -i "${SSH_PRIVATE_KEY}" \
-    -o StrictHostKeyChecking=no \
-    root@${VM_IP} 'which perl && perl --version'
-
-# Run sparrowdo with debug output
-sparrowdo \
-    --host="${VM_IP}" \
-    --ssh_user=root \
-    --ssh_private_key="${SSH_PRIVATE_KEY}" \
-    --verbose \
-    --sparrowfile="${SPARROWFILE}"
-```
-
-### Clean Up Failed VMs
-```bash
-# List all VMs
-virsh list --all
-
-# Force cleanup
-for vm in $(virsh list --all --name | grep "${BUILD_ID}"); do
-    virsh destroy "$vm" 2>/dev/null || true
-    virsh undefine "$vm" 2>/dev/null || true
-    sudo rm -f "/var/lib/libvirt/images/${vm}.qcow2"
-done
-```
-
---
-
-## Quick Cleanup Script
-
-If something goes wrong and you need to clean up everything:
-
-```bash
-# Save this as cleanup-all.sh
-#!/bin/bash
-
-BUILD_ID="${1:-manual}"
-
-echo "Cleaning up build: ${BUILD_ID}"
-
-# Stop and remove VMs
-for vm in $(virsh list --all --name | grep "${BUILD_ID}"); do
-    echo "Removing VM: $vm"
-    virsh destroy "$vm" 2>/dev/null || true
-    virsh undefine "$vm" 2>/dev/null || true
-    sudo rm -f "/var/lib/libvirt/images/${vm}.qcow2"
-done
-
-# Remove images
-sudo rm -f "/var/lib/libvirt/images/base-${BUILD_ID}.qcow2"
-sudo rm -f "/var/lib/libvirt/images/golden-${BUILD_ID}.qcow2"
-
-# Remove working directory
-rm -rf "/Users/ssimpson/Documents/git/resf-test-jenkins/manual-test-${BUILD_ID}"
-
-echo "Cleanup complete"
-```
-
-Then run:
-```bash
-chmod +x cleanup-all.sh
-./cleanup-all.sh manual-1234567890  # Use your BUILD_ID
-```
--- a/docs/manual-test-run.sh
+++ b/docs/manual-test-run.sh
@@ -1,361 +0,0 @@
-#!/bin/bash
-#
-# Manual Test Run - Simulates Jenkins Pipeline Steps
-# This script walks through all the steps Jenkins would take
-#
-
-set -e
-
-# Colors for output
-GREEN='\033[0;32m'
-BLUE='\033[0;34m'
-YELLOW='\033[1;33m'
-RED='\033[0;31m'
-NC='\033[0m' # No Color
-
-echo_step() {
-    echo -e "${BLUE}========================================${NC}"
-    echo -e "${BLUE}$1${NC}"
-    echo -e "${BLUE}========================================${NC}"
-}
-
-echo_info() {
-    echo -e "${GREEN}[INFO]${NC} $1"
-}
-
-echo_warn() {
-    echo -e "${YELLOW}[WARN]${NC} $1"
-}
-
-echo_error() {
-    echo -e "${RED}[ERROR]${NC} $1"
-}
-
-# =============================================================================
-# CONFIGURATION - Edit these variables
-# =============================================================================
-
-# Build ID (simulating Jenkins BUILD_NUMBER)
-BUILD_ID="manual-$(date +%s)"
-
-# QCOW2 Image URL
-QCOW2_URL="https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2"
-
-# Image storage directory
-IMAGES_DIR="/var/lib/libvirt/images"
-
-# Image download behavior
-# Set to "true" to force re-download even if cached image exists
-REDOWNLOAD_IMAGE="false"
-
-# SSH Keys
-SSH_PRIVATE_KEY="${HOME}/.ssh/id_rsa"
-SSH_PUBLIC_KEY="${HOME}/.ssh/id_rsa.pub"
-
-# Test repository configuration
-# Edit this to point to your actual test repository
-TEST_NAME="example-test"
-TEST_REPO_URL="https://github.com/your-org/your-test-repo.git"
-TEST_REPO_BRANCH="main"
-
-# Working directory (this script's location)
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
-WORK_DIR="${SCRIPT_DIR}/manual-test-${BUILD_ID}"
-
-# Golden image prep script location
-PREP_SCRIPT="${SCRIPT_DIR}/docs/default-prep.sh"
-
-# =============================================================================
-# Step 1: Initialize
-# =============================================================================
-
-echo_step "Step 1: Initialize Environment"
-
-echo_info "Build ID: ${BUILD_ID}"
-echo_info "Working Directory: ${WORK_DIR}"
-echo_info "Images Directory: ${IMAGES_DIR}"
-echo_info "Scripts Directory: ${SCRIPT_DIR}/scripts"
-
-# Create working directory
-mkdir -p "${WORK_DIR}"
-cd "${WORK_DIR}"
-
-# Ensure images directory exists
-sudo mkdir -p "${IMAGES_DIR}"
-sudo chown ${USER}:${USER} "${IMAGES_DIR}"
-
-# Make scripts executable
-chmod +x "${SCRIPT_DIR}/scripts"/*.sh
-
-echo_info "Initialization complete"
-echo ""
-read -p "Press Enter to continue to Step 2..."
-
-# =============================================================================
-# Step 2: Download Base Image
-# =============================================================================
-
-echo_step "Step 2: Download Base QCOW2 Image"
-
-# Extract filename from URL for caching
-IMAGE_FILENAME=$(basename "${QCOW2_URL}")
-CACHED_IMAGE="${IMAGES_DIR}/${IMAGE_FILENAME}"
-BASE_IMAGE="${IMAGES_DIR}/base-${BUILD_ID}.qcow2"
-
-echo_info "Image URL: ${QCOW2_URL}"
-echo_info "Cached image path: ${CACHED_IMAGE}"
-echo_info "Build-specific image: ${BASE_IMAGE}"
-echo_info "Redownload setting: ${REDOWNLOAD_IMAGE}"
-echo ""
-
-if [ "${REDOWNLOAD_IMAGE}" = "true" ]; then
-    echo_warn "REDOWNLOAD_IMAGE is enabled - forcing fresh download"
-    echo_info "Downloading from: ${QCOW2_URL}"
-    curl -L --progress-bar -o "${BASE_IMAGE}" "${QCOW2_URL}"
-else
-    if [ -f "${CACHED_IMAGE}" ]; then
-        echo_info "Found cached image: ${CACHED_IMAGE}"
-        echo_info "Creating copy for build ${BUILD_ID}..."
-        cp "${CACHED_IMAGE}" "${BASE_IMAGE}"
-    else
-        echo_info "No cached image found at: ${CACHED_IMAGE}"
-        echo_info "Downloading from: ${QCOW2_URL}"
-        curl -L --progress-bar -o "${CACHED_IMAGE}" "${QCOW2_URL}"
-        echo_info "Creating copy for build ${BUILD_ID}..."
-        cp "${CACHED_IMAGE}" "${BASE_IMAGE}"
-    fi
-fi
-
-echo ""
-echo_info "Verifying image..."
-qemu-img info "${BASE_IMAGE}" | head -10
-
-echo ""
-read -p "Press Enter to continue to Step 3..."
-
-# =============================================================================
-# Step 3: Prepare Golden Image
-# =============================================================================
-
-echo_step "Step 3: Prepare Golden Image"
-
-GOLDEN_IMAGE="${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"
-
-echo_info "Golden image will be created at: ${GOLDEN_IMAGE}"
-echo_info "Using prep script: ${PREP_SCRIPT}"
-echo_info "Using SSH public key: ${SSH_PUBLIC_KEY}"
-
-# Check if prep script exists
-if [ ! -f "${PREP_SCRIPT}" ]; then
-    echo_error "Prep script not found: ${PREP_SCRIPT}"
-    echo_info "You can create it or use the default one in docs/"
-    exit 1
-fi
-
-# Check if SSH keys exist
-if [ ! -f "${SSH_PUBLIC_KEY}" ]; then
-    echo_error "SSH public key not found: ${SSH_PUBLIC_KEY}"
-    echo_info "Generate one with: ssh-keygen -t rsa -b 4096 -f ${HOME}/.ssh/id_rsa"
-    exit 1
-fi
-
-echo_info "Running setup_base.sh..."
-"${SCRIPT_DIR}/scripts/setup_base.sh" \
-    "${BASE_IMAGE}" \
-    "${PREP_SCRIPT}" \
-    "${GOLDEN_IMAGE}" \
-    "${SSH_PUBLIC_KEY}"
-
-echo_info "Golden image ready: ${GOLDEN_IMAGE}"
-
-echo ""
-read -p "Press Enter to continue to Step 4 (Bootstrap)..."
-
-# =============================================================================
-# Step 4: Bootstrap Golden Image
-# =============================================================================
-
-echo_step "Step 4: Bootstrap Golden Image with Sparrowdo"
-
-echo_info "Bootstrapping installs Sparrowdo and dependencies ONCE in the golden image"
-echo_info "This allows all test VMs to skip bootstrap and run tests immediately"
-echo ""
-
-"${SCRIPT_DIR}/scripts/bootstrap_golden.sh" "${GOLDEN_IMAGE}" "${SSH_PRIVATE_KEY}"
-
-echo ""
-read -p "Press Enter to continue to Step 5 (Provision Test VM)..."
-
-# =============================================================================
-# Step 5: Provision Test VM
-# =============================================================================
-
-echo_step "Step 5: Provision Test VM"
-
-VM_NAME="${TEST_NAME}-${BUILD_ID}"
-
-echo_info "Creating VM: ${VM_NAME}"
-echo_info "Using golden image: ${GOLDEN_IMAGE}"
-
-# Provision VM and capture output
-PROVISION_OUTPUT=$("${SCRIPT_DIR}/scripts/provision_vm.sh" "${VM_NAME}" "${GOLDEN_IMAGE}" 60)
-
-# Extract just the IP address (last line of output)
-VM_IP=$(echo "$PROVISION_OUTPUT" | tail -1)
-
-if [ "$VM_IP" = "ERROR" ] || [ -z "$VM_IP" ]; then
-    echo_error "Failed to provision VM"
-    echo_error "Provision output:"
-    echo "$PROVISION_OUTPUT"
-    exit 1
-fi
-
-echo_info "VM provisioned successfully"
-echo_info "IP Address: ${VM_IP}"
-
-# Wait for SSH to be ready (test with rocky user)
-echo_info "Waiting for SSH to be ready (testing with rocky user)..."
-for i in {1..30}; do
-    if ssh -i "${SSH_PRIVATE_KEY}" \
-           -o StrictHostKeyChecking=no \
-           -o ConnectTimeout=5 \
-           -o UserKnownHostsFile=/dev/null \
-           rocky@${VM_IP} 'echo "SSH ready"' 2>/dev/null; then
-        echo_info "SSH connection established for rocky user"
-        break
-    fi
-    echo_info "Attempt $i/30..."
-    sleep 2
-done
-
-echo ""
-read -p "Press Enter to continue to Step 6 (Clone Test Repository)..."
-
-# =============================================================================
-# Step 6: Clone Test Repository
-# =============================================================================
-
-echo_step "Step 6: Clone Test Repository"
-
-echo_info "Cloning: ${TEST_REPO_URL}"
-echo_info "Branch: ${TEST_REPO_BRANCH}"
-
-# Create test workspace
-TEST_WORKSPACE="${WORK_DIR}/test-workspace"
-mkdir -p "${TEST_WORKSPACE}"
-cd "${TEST_WORKSPACE}"
-
-# Clone the test repository
-if git clone -b "${TEST_REPO_BRANCH}" "${TEST_REPO_URL}" test-repo; then
-    echo_info "Repository cloned successfully"
-else
-    echo_error "Failed to clone repository"
-    echo_warn "Cleanup will still run. Press Ctrl+C to abort or Enter to continue cleanup..."
-    read
-    "${SCRIPT_DIR}/scripts/cleanup_vm.sh" "${VM_NAME}"
-    exit 1
-fi
-
-# Find sparrowfile (look for main.raku or sparrowfile)
-SPARROWFILE=$(find test-repo -name main.raku -type f | head -1)
-if [ -z "$SPARROWFILE" ]; then
-    SPARROWFILE=$(find test-repo -name sparrowfile -type f | head -1)
-fi
-
-if [ -z "$SPARROWFILE" ]; then
-    echo_error "No sparrowfile or main.raku found in repository"
-    echo_warn "Cleanup will still run. Press Ctrl+C to abort or Enter to continue cleanup..."
-    read
-    "${SCRIPT_DIR}/scripts/cleanup_vm.sh" "${VM_NAME}"
-    exit 1
-fi
-
-echo_info "Found sparrowfile: ${SPARROWFILE}"
-
-echo ""
-read -p "Press Enter to continue to Step 7 (Run Test)..."
-
-# =============================================================================
-# Step 7: Run Sparrowdo Test
-# =============================================================================
-
-echo_step "Step 7: Run Sparrowdo Test"
-
-# Create logs directory
-mkdir -p logs
-
-echo_info "Running Sparrowdo test against VM..."
-echo_info "Target: rocky@${VM_IP}"
-echo_info "Sparrowfile: ${SPARROWFILE}"
-
-# Run sparrowdo test
-if timeout 900 sparrowdo \
-    --host="${VM_IP}" \
-    --ssh_user=rocky \
-    --ssh_private_key="${SSH_PRIVATE_KEY}" \
-    --ssh_args="-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null" \
-    --no_sudo \
-    --sparrowfile="${SPARROWFILE}" \
-    --verbose \
-    --color 2>&1 | tee logs/test.log; then
-    echo_info "Test completed successfully!"
-else
-    echo_error "Test failed or timed out"
-    echo_info "Logs saved to: ${TEST_WORKSPACE}/logs/test.log"
-fi
-
-echo ""
-echo_info "Test logs location: ${TEST_WORKSPACE}/logs/test.log"
-
-echo ""
-read -p "Press Enter to continue to Step 8 (Cleanup)..."
-
-# =============================================================================
-# Step 8: Cleanup
-# =============================================================================
-
-echo_step "Step 8: Cleanup"
-
-echo_info "Cleaning up VM: ${VM_NAME}"
-"${SCRIPT_DIR}/scripts/cleanup_vm.sh" "${VM_NAME}"
-
-echo ""
-echo_warn "Do you want to remove temporary images? (y/n)"
-echo_info "Base image: ${BASE_IMAGE}"
-echo_info "Golden image: ${GOLDEN_IMAGE}"
-read -p "Remove images? (y/n): " -n 1 -r
-echo
-
-if [[ $REPLY =~ ^[Yy]$ ]]; then
-    echo_info "Removing temporary images..."
-    sudo rm -f "${BASE_IMAGE}"
-    sudo rm -f "${GOLDEN_IMAGE}"
-    echo_info "Images removed"
-else
-    echo_info "Images preserved for inspection"
-    echo_info "Base image: ${BASE_IMAGE}"
-    echo_info "Golden image: ${GOLDEN_IMAGE}"
-fi
-
-# =============================================================================
-# Summary
-# =============================================================================
-
-echo ""
-echo_step "Summary"
-
-echo_info "Build ID: ${BUILD_ID}"
-echo_info "Test Name: ${TEST_NAME}"
-echo_info "Working Directory: ${WORK_DIR}"
-echo_info "Test Logs: ${TEST_WORKSPACE}/logs/test.log"
-
-if [ -f "${BASE_IMAGE}" ]; then
-    echo_info "Base Image: ${BASE_IMAGE} (preserved)"
-fi
-
-if [ -f "${GOLDEN_IMAGE}" ]; then
-    echo_info "Golden Image: ${GOLDEN_IMAGE} (preserved)"
-fi
-
-echo ""
-echo_info "Manual test run complete!"
--- a/scripts/bootstrap_golden.sh
+++ b/scripts/bootstrap_golden.sh
@@ -1,107 +0,0 @@
-#!/bin/bash
-# Bootstrap the golden image by booting it, running sparrowdo --bootstrap, then shutting down
-# This should be run ONCE after creating the golden image
-
-GOLDEN_IMAGE="$1"
-SSH_PRIVATE_KEY="${2:-$HOME/.ssh/id_rsa}"
-
-if [ -z "$GOLDEN_IMAGE" ]; then
-    echo "Usage: $0 <golden_image_path> [ssh_private_key]"
-    exit 1
-fi
-
-if [ ! -f "$GOLDEN_IMAGE" ]; then
-    echo "ERROR: Golden image not found: $GOLDEN_IMAGE"
-    exit 1
-fi
-
-BOOTSTRAP_VM="bootstrap-golden-$(date +%s)"
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-echo "=========================================="
-echo "Bootstrapping Golden Image"
-echo "=========================================="
-echo "Golden Image: $GOLDEN_IMAGE"
-echo "Bootstrap VM: $BOOTSTRAP_VM"
-echo "SSH Key: $SSH_PRIVATE_KEY"
-echo ""
-
-# Provision temporary VM from golden image
-echo "[1/4] Provisioning temporary VM..."
-VM_IP=$("${SCRIPT_DIR}/provision_vm.sh" "$BOOTSTRAP_VM" "$GOLDEN_IMAGE" 60)
-
-if [ "$VM_IP" = "ERROR" ] || [ -z "$VM_IP" ]; then
-    echo "ERROR: Failed to provision bootstrap VM"
-    exit 1
-fi
-
-VM_IP=$(echo "$VM_IP" | tail -1)
-echo "VM IP: $VM_IP"
-
-# Wait for SSH
-echo ""
-echo "[2/4] Waiting for SSH to be ready..."
-for i in {1..30}; do
-    if ssh -i "$SSH_PRIVATE_KEY" \
-           -o StrictHostKeyChecking=no \
-           -o ConnectTimeout=5 \
-           -o UserKnownHostsFile=/dev/null \
-           rocky@${VM_IP} 'echo "SSH ready"' 2>/dev/null; then
-        echo "SSH connection established"
-        break
-    fi
-    sleep 2
-done
-
-# Run sparrowdo bootstrap
-echo ""
-echo "[3/4] Running Sparrowdo bootstrap..."
-if sparrowdo \
-    --host="${VM_IP}" \
-    --ssh_user=rocky \
-    --ssh_private_key="${SSH_PRIVATE_KEY}" \
-    --ssh_args="-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null" \
-    --bootstrap \
-    --color; then
-    echo "Bootstrap completed successfully!"
-else
-    echo "WARNING: Bootstrap may have failed"
-fi
-
-# Shutdown the VM cleanly to save changes
-echo ""
-echo "[4/4] Shutting down VM to save changes..."
-ssh -i "$SSH_PRIVATE_KEY" \
-    -o StrictHostKeyChecking=no \
-    -o UserKnownHostsFile=/dev/null \
-    rocky@${VM_IP} 'sudo shutdown -h now' 2>/dev/null || true
-
-# Wait for VM to shut down
-echo "Waiting for VM to shut down..."
-sleep 10
-
-for i in {1..30}; do
-    if ! sudo virsh -c qemu:///system list --name 2>/dev/null | grep -q "$BOOTSTRAP_VM"; then
-        echo "VM has shut down"
-        break
-    fi
-    sleep 2
-done
-
-# Clean up VM definition (disk contains the bootstrapped state)
-echo "Cleaning up VM definition..."
-sudo virsh -c qemu:///system undefine "$BOOTSTRAP_VM" 2>/dev/null || true
-
-echo ""
-echo "=========================================="
-echo "Bootstrap Complete"
-echo "=========================================="
-echo "Golden image has been bootstrapped with Sparrowdo"
-echo "The image now contains:"
-echo "  - Raku/Rakudo runtime"
-echo "  - Sparrowdo and dependencies"
-echo "  - All required testing tools"
-echo ""
-echo "This image can now be used for testing without"
-echo "running bootstrap on each test VM."
-echo "=========================================="
--- a/scripts/build-golden.sh
+++ b/scripts/build-golden.sh
@@ -1,51 +0,0 @@
-#!/bin/bash
-
-BASE_IMAGE="$1"
-GOLDEN_IMAGE="$2"
-SSH_PUB_KEY="${3:-$HOME/.ssh/id_rsa.pub}"
-
-if [ -z "$BASE_IMAGE" ] || [ -z "$GOLDEN_IMAGE" ]; then
-    echo "Usage: $0 <base_image> <golden_image> [ssh_pub_key]"
-    exit 1
-fi
-
-echo "Creating golden image: $GOLDEN_IMAGE"
-cp "$BASE_IMAGE" "$GOLDEN_IMAGE"
-
-export LIBGUESTFS_BACKEND=direct
-
-sudo virt-customize -a "$GOLDEN_IMAGE" \
-    --install rakudo,rakudo-zef,perl,git,openssh-server \
-    --run-command 'useradd -m rocky && echo "rocky:rockypass" | chpasswd' \
-    --run-command 'echo "rocky ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/rocky && chmod 0440 /etc/sudoers.d/rocky' \
-    --ssh-inject root:file:"$SSH_PUB_KEY" \
-    --ssh-inject rocky:file:"$SSH_PUB_KEY" \
-    --root-password password:rockytesting \
-    --run-command 'systemctl enable sshd' \
-    --selinux-relabel
-
-echo "Bootstrapping Sparrowdo..."
-BOOTSTRAP_VM="bootstrap-$$"
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-VM_IP=$("$SCRIPT_DIR/provision_vm.sh" "$BOOTSTRAP_VM" "$GOLDEN_IMAGE" 60 | tail -1)
-
-if [ -z "$VM_IP" ] || [ "$VM_IP" = "ERROR" ]; then
-    echo "ERROR: Failed to provision bootstrap VM"
-    exit 1
-fi
-
-sleep 5
-
-SSH_KEY="${SSH_PUB_KEY%.pub}"
-sparrowdo --host="$VM_IP" --ssh_user=rocky --ssh_private_key="$SSH_KEY" \
-    --ssh_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-    --bootstrap --color
-
-ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-    rocky@"$VM_IP" 'sudo shutdown -h now' 2>/dev/null || true
-
-sleep 10
-"$SCRIPT_DIR/cleanup_vm.sh" "$BOOTSTRAP_VM"
-
-echo "Golden image ready: $GOLDEN_IMAGE"
--- a/scripts/cleanup-all.sh
+++ b/scripts/cleanup-all.sh
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-PATTERN="${1:-.*}"
-
-echo "Cleaning up VMs matching: $PATTERN"
-
-for vm in $(virsh -c qemu:///system list --all --name | grep -E "$PATTERN"); do
-    echo "Destroying $vm"
-    virsh -c qemu:///system destroy "$vm" 2>/dev/null || true
-    virsh -c qemu:///system undefine "$vm" 2>/dev/null || true
-    rm -f "/var/lib/libvirt/images/${vm}.qcow2" 2>/dev/null || true
-done
-
-echo "Cleanup complete"
--- a/scripts/cleanup_vm.sh
+++ b/scripts/cleanup_vm.sh
@@ -1,25 +0,0 @@
-#!/bin/bash
-set -e
-
-VM_NAME="$1"
-
-if [ -z "$VM_NAME" ]; then
-    echo "Usage: $0 <vm_name>"
-    exit 1
-fi
-
-echo "[Cleanup] Starting cleanup for VM: $VM_NAME"
-
-# Force stop VM
-echo "[Cleanup] Destroying VM..."
-sudo virsh -c qemu:///system destroy "$VM_NAME" 2>/dev/null || echo "[Cleanup] VM was not running"
-
-# Remove VM definition
-echo "[Cleanup] Undefining VM..."
-sudo virsh -c qemu:///system undefine "$VM_NAME" 2>/dev/null || echo "[Cleanup] VM definition already removed"
-
-# Remove disk image
-echo "[Cleanup] Removing disk image..."
-sudo rm -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" || echo "[Cleanup] Disk already removed"
-
-echo "[Cleanup] Complete for $VM_NAME"
--- a/scripts/download-image.sh
+++ b/scripts/download-image.sh
@@ -1,22 +0,0 @@
-#!/bin/bash
-
-URL="$1"
-OUTPUT_DIR="${2:-/var/lib/libvirt/images}"
-FORCE="${3:-false}"
-
-if [ -z "$URL" ]; then
-    echo "Usage: $0 <url> [output_dir] [force]"
-    exit 1
-fi
-
-FILENAME=$(basename "$URL")
-CACHED="$OUTPUT_DIR/$FILENAME"
-
-if [ "$FORCE" = "true" ] || [ ! -f "$CACHED" ]; then
-    echo "Downloading $URL to $CACHED"
-    curl -L --progress-bar -o "$CACHED" "$URL"
-else
-    echo "Using cached image: $CACHED"
-fi
-
-echo "$CACHED"
--- a/scripts/provision_vm.sh
+++ b/scripts/provision_vm.sh
@@ -1,72 +0,0 @@
-#!/bin/bash
-# Note: Not using 'set -e' here because we handle errors explicitly
-# and arithmetic operations can cause false failures with set -e
-
-VM_NAME="$1"
-GOLDEN_IMAGE="$2"
-MAX_WAIT="${3:-30}"
-
-if [ -z "$VM_NAME" ] || [ -z "$GOLDEN_IMAGE" ]; then
-    echo "Usage: $0 <vm_name> <golden_image_path> [max_wait_seconds]"
-    exit 1
-fi
-
-echo "[Provision] Creating VM: $VM_NAME"
-
-# Create linked clone (very fast - just a pointer to base)
-VM_DISK="/var/lib/libvirt/images/${VM_NAME}.qcow2"
-
-echo "[Provision] Creating overlay disk: $VM_DISK"
-sudo qemu-img create -f qcow2 -b "$GOLDEN_IMAGE" -F qcow2 "$VM_DISK" 2>/dev/null
-
-# Define and start VM
-echo "[Provision] Starting VM with virt-install..."
-VIRT_INSTALL_OUTPUT=$(sudo virt-install \
-    --name "$VM_NAME" \
-    --memory 2048 \
-    --vcpus 2 \
-    --disk path="$VM_DISK",format=qcow2 \
-    --import \
-    --os-variant rocky9-unknown \
-    --network network=default \
-    --noautoconsole \
-    --wait 0 \
-    --transient \
-    2>&1) || {
-    echo "[Provision] ERROR: virt-install failed"
-    echo "$VIRT_INSTALL_OUTPUT" | grep -v "WARNING"
-    echo "[Provision] Cleaning up disk..."
-    sudo rm -f "$VM_DISK"
-    echo "ERROR"
-    exit 1
-}
-
-# Wait for IP address
-echo "[Provision] Waiting for VM to obtain IP address (max ${MAX_WAIT}s)..."
-COUNTER=0
-while [ $COUNTER -lt $MAX_WAIT ]; do
-    # Try to get IP from DHCP lease (explicitly use system connection)
-    IP=$(sudo virsh -c qemu:///system domifaddr "$VM_NAME" --source lease 2>/dev/null | awk '/ipv4/ {print $4}' | cut -d/ -f1 | head -1)
-
-    if [ -n "$IP" ] && [ "$IP" != "0.0.0.0" ]; then
-        echo "[Provision] IP obtained: $IP"
-        echo "$IP"
-        exit 0
-    fi
-
-    sleep 2
-    ((COUNTER++))
-
-    # Show progress every 5 iterations
-    if [ $((COUNTER % 5)) -eq 0 ]; then
-        echo "[Provision] Still waiting... (${COUNTER}/${MAX_WAIT}s)"
-    fi
-done
-
-echo "[Provision] ERROR: Could not obtain IP for $VM_NAME after $MAX_WAIT seconds"
-echo "[Provision] Destroying failed VM..."
-sudo virsh destroy "$VM_NAME" 2>/dev/null || true
-sudo virsh undefine "$VM_NAME" 2>/dev/null || true
-sudo rm -f "$VM_DISK"
-echo "ERROR"
-exit 1
--- a/scripts/run-test.sh
+++ b/scripts/run-test.sh
@@ -1,60 +0,0 @@
-#!/bin/bash
-
-TEST_NAME="$1"
-TEST_REPO="$2"
-GOLDEN_IMAGE="$3"
-SSH_KEY="${4:-$HOME/.ssh/id_rsa}"
-
-if [ -z "$TEST_NAME" ] || [ -z "$TEST_REPO" ] || [ -z "$GOLDEN_IMAGE" ]; then
-    echo "Usage: $0 <test_name> <test_repo_url> <golden_image> [ssh_key]"
-    exit 1
-fi
-
-VM_NAME="$TEST_NAME-$$"
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-WORK_DIR="/tmp/test-$VM_NAME"
-
-mkdir -p "$WORK_DIR"
-cd "$WORK_DIR"
-
-cleanup() {
-    echo "Cleaning up..."
-    "$SCRIPT_DIR/cleanup_vm.sh" "$VM_NAME" 2>/dev/null || true
-    cd /tmp
-    rm -rf "$WORK_DIR"
-}
-
-trap cleanup EXIT
-
-echo "Provisioning VM..."
-VM_IP=$("$SCRIPT_DIR/provision_vm.sh" "$VM_NAME" "$GOLDEN_IMAGE" 60 | tail -1)
-
-if [ -z "$VM_IP" ] || [ "$VM_IP" = "ERROR" ]; then
-    echo "ERROR: Failed to provision VM"
-    exit 1
-fi
-
-echo "VM ready at $VM_IP"
-
-echo "Cloning test repository..."
-git clone "$TEST_REPO" test-repo
-
-SPARROWFILE=$(find test-repo -name "main.raku" -o -name "sparrowfile" | head -1)
-
-if [ -z "$SPARROWFILE" ]; then
-    echo "ERROR: No sparrowfile found"
-    exit 1
-fi
-
-echo "Running test: $SPARROWFILE"
-sparrowdo \
-    --host="$VM_IP" \
-    --ssh_user=rocky \
-    --ssh_private_key="$SSH_KEY" \
-    --ssh_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-    --no_sudo \
-    --sparrowfile="$SPARROWFILE" \
-    --verbose \
-    --color
-
-echo "Test completed successfully"
--- a/scripts/setup_base.sh
+++ b/scripts/setup_base.sh
@@ -1,66 +0,0 @@
-#!/bin/bash
-set -e
-
-# Signature: ./setup_base.sh <qcow2_path> <prep_script_path> <output_golden_image> <ssh_pub_key>
-QCOW2_PATH="$1"
-PREP_SCRIPT_PATH="$2"
-GOLDEN_IMAGE="$3"
-SSH_PUB_KEY="$4"
-
-if [ -z "$QCOW2_PATH" ] || [ -z "$GOLDEN_IMAGE" ] || [ -z "$SSH_PUB_KEY" ]; then
-    echo "Usage: $0 <qcow2_path> <prep_script_path> <output_golden_image> <ssh_pub_key>"
-    exit 1
-fi
-
-echo "=========================================="
-echo "Setting up golden image"
-echo "=========================================="
-echo "Source QCOW2: $QCOW2_PATH"
-echo "Output Golden: $GOLDEN_IMAGE"
-echo "Prep Script: $PREP_SCRIPT_PATH"
-echo "SSH Key: $SSH_PUB_KEY"
-
-# Ensure libvirt is running
-sudo systemctl is-active --quiet libvirtd || sudo systemctl start libvirtd
-sleep 2
-
-# Copy original to golden
-echo "[Step 1/3] Copying base image to golden image..."
-cp "$QCOW2_PATH" "$GOLDEN_IMAGE"
-
-# Apply custom preparation script if provided
-if [ -f "$PREP_SCRIPT_PATH" ]; then
-    echo "[Step 2/3] Applying custom preparation script..."
-    export LIBGUESTFS_BACKEND=direct
-
-    # Run the prep script inside the image
-    sudo virt-customize -a "$GOLDEN_IMAGE" \
-        --run "$PREP_SCRIPT_PATH" \
-        --ssh-inject root:file:"$SSH_PUB_KEY" \
-        --ssh-inject rocky:file:"$SSH_PUB_KEY" \
-        --root-password password:rockytesting \
-        --selinux-relabel 2>&1 || {
-        echo "ERROR: virt-customize failed"
-        exit 1
-    }
-else
-    echo "[Step 2/3] No custom prep script provided, applying defaults..."
-    export LIBGUESTFS_BACKEND=direct
-
-    sudo virt-customize -a "$GOLDEN_IMAGE" \
-        --ssh-inject root:file:"$SSH_PUB_KEY" \
-        --root-password password:rockytesting \
-        --install perl,git,wget,tar,openssh-server \
-        --run-command 'systemctl enable sshd' \
-        --selinux-relabel 2>&1 || {
-        echo "ERROR: virt-customize failed"
-        exit 1
-    }
-fi
-
-echo "[Step 3/3] Verifying golden image..."
-qemu-img info "$GOLDEN_IMAGE" | head -5
-
-echo "=========================================="
-echo "Golden image ready: $GOLDEN_IMAGE"
-echo "=========================================="