admin/resf-testing-repo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates

Browse Source
This commit is contained in:
Stephen Simpson
2025-11-26 08:15:00 -06:00
parent 3cbd4525a0
commit bb829c9b63
18 changed files with 2440 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
Jenkinsfile vendored
View File

@@ -34,40 +34,46 @@ set -e
echo "Preparing standard golden image..." echo "Preparing standard golden image..."
# Update system # NOTE: This runs inside virt-customize (offline mode)
dnf update -y # Cannot use firewall-cmd, hostnamectl, or systemctl start/restart
# Only systemctl enable works (creates symlinks)
# Install common testing dependencies # Update system (optional - can be slow, commented out by default)
# dnf update -y
# Install common testing dependencies including Raku/Sparrowdo
dnf install -y \\ dnf install -y \\
perl \\ perl \\
git \\ git \\
wget \\ wget \\
tar \\ tar \\
openssh-server \\ openssh-server \\
firewalld \\ vim \\
chrony \\ rakudo \\
vim rakudo-zef
# Configure services # Enable services (works offline)
systemctl enable sshd systemctl enable sshd
systemctl enable firewalld
systemctl enable chronyd
# Configure firewall # Create rocky user (standard non-root user)
firewall-cmd --permanent --add-service=ssh useradd -m rocky 2>/dev/null || true
firewall-cmd --reload echo "rocky:rockypass" | chpasswd
# Set consistent hostname # Add rocky to sudoers (needed for sparrowdo --bootstrap)
hostnamectl set-hostname test-node echo "rocky ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/rocky
chmod 0440 /etc/sudoers.d/rocky
# Create testing user # Create testuser for backward compatibility
useradd -m testuser 2>/dev/null || true useradd -m testuser 2>/dev/null || true
echo "testuser:testpass" | chpasswd echo "testuser:testpass" | chpasswd
echo "testuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
# Add to sudoers
echo "testuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/testuser
chmod 0440 /etc/sudoers.d/testuser
echo "Golden image preparation complete!" echo "Golden image preparation complete!"
''', ''',
description: 'Shell script to run inside the golden image' description: 'Shell script to run inside the golden image (virt-customize offline mode)'
) )
// Test Selection // Test Selection
@@ -84,6 +90,13 @@ echo "Golden image preparation complete!"
description: 'Maximum concurrent VMs' description: 'Maximum concurrent VMs'
) )
// Image Download Control
booleanParam(
name: 'REDOWNLOAD_IMAGE',
defaultValue: false,
description: 'Force re-download of base QCOW2 image even if it exists'
)
// Cleanup Control // Cleanup Control
booleanParam( booleanParam(
name: 'KEEP_GOLDEN_IMAGE', name: 'KEEP_GOLDEN_IMAGE',
@@ -142,18 +155,42 @@ echo "Golden image preparation complete!"
steps { steps {
script { script {
dir(IMAGES_DIR) { dir(IMAGES_DIR) {
sh ''' // Extract filename from URL for caching
if [ ! -f "base-${BUILD_ID}.qcow2" ]; then def imageFilename = params.QCOW2_URL.split('/').last()
def cachedImagePath = "${IMAGES_DIR}/${imageFilename}"
def buildImagePath = "${IMAGES_DIR}/base-${BUILD_ID}.qcow2"
if (params.REDOWNLOAD_IMAGE) {
echo "REDOWNLOAD_IMAGE is enabled - forcing fresh download"
sh """
echo "Downloading QCOW2 image from: ${QCOW2_URL}" echo "Downloading QCOW2 image from: ${QCOW2_URL}"
curl -L --progress-bar -o "base-${BUILD_ID}.qcow2" "${QCOW2_URL}" curl -L --progress-bar -o "${buildImagePath}" "${QCOW2_URL}"
echo "" echo ""
echo "Image downloaded successfully:" echo "Image downloaded successfully:"
qemu-img info "base-${BUILD_ID}.qcow2" | head -5 qemu-img info "${buildImagePath}" | head -5
"""
} else {
sh """
if [ -f "${cachedImagePath}" ]; then
echo "Found cached image: ${cachedImagePath}"
echo "Creating copy for build ${BUILD_ID}..."
cp "${cachedImagePath}" "${buildImagePath}"
qemu-img info "${buildImagePath}" | head -5
else else
echo "Using cached base image" echo "No cached image found at: ${cachedImagePath}"
echo "Downloading QCOW2 image from: ${QCOW2_URL}"
curl -L --progress-bar -o "${cachedImagePath}" "${QCOW2_URL}"
echo ""
echo "Image downloaded successfully:"
qemu-img info "${cachedImagePath}" | head -5
echo "Creating copy for build ${BUILD_ID}..."
cp "${cachedImagePath}" "${buildImagePath}"
fi fi
''' """
env.BASE_IMAGE_PATH = "${IMAGES_DIR}/base-${BUILD_ID}.qcow2" }
env.BASE_IMAGE_PATH = buildImagePath
echo "Base image ready at: ${buildImagePath}"
} }
} }
} }
@@ -192,6 +229,30 @@ echo "Golden image preparation complete!"
} }
} }
stage('Bootstrap Golden Image') {
steps {
script {
echo "=========================================="
echo "Bootstrapping Sparrowdo in Golden Image"
echo "=========================================="
echo "This installs Sparrowdo ONCE in the golden image"
echo "All test VMs will inherit this and skip bootstrap"
echo ""
// Expand SSH private key path
def sshPrivateKey = params.SSH_PRIVATE_KEY_PATH.replace('${HOME}', env.HOME)
sh """
./scripts/bootstrap_golden.sh \\
${GOLDEN_IMAGE_PATH} \\
${sshPrivateKey}
"""
echo "Golden image successfully bootstrapped"
}
}
}
stage('Parse Test Matrix') { stage('Parse Test Matrix') {
steps { steps {
script { script {
@@ -265,29 +326,38 @@ echo "Golden image preparation complete!"
echo "[${testName}] Cloning test repository..." echo "[${testName}] Cloning test repository..."
sh "git clone -b ${testBranch} ${testUrl} test-repo || true" sh "git clone -b ${testBranch} ${testUrl} test-repo || true"
// Verify sparrowfile exists // Verify sparrowfile exists (check for main.raku or sparrowfile)
def sparrowfilePath = sh( def sparrowfilePath = sh(
script: 'find test-repo -name sparrowfile -type f | head -1', script: 'find test-repo -name main.raku -type f | head -1',
returnStdout: true returnStdout: true
).trim() ).trim()
if (!sparrowfilePath) { if (!sparrowfilePath) {
error "No sparrowfile found in test repository" sparrowfilePath = sh(
script: 'find test-repo -name sparrowfile -type f | head -1',
returnStdout: true
).trim()
}
if (!sparrowfilePath) {
error "No sparrowfile or main.raku found in test repository"
} }
echo "[${testName}] Found sparrowfile: ${sparrowfilePath}" echo "[${testName}] Found sparrowfile: ${sparrowfilePath}"
// Run test // Run test (bootstrap was already done on golden image)
echo "[${testName}] Running Sparrowdo..." echo "[${testName}] Running Sparrowdo test..."
sh """ sh """
mkdir -p logs mkdir -p logs
timeout 900 sparrowdo \\ timeout 900 sparrowdo \\
--host=${targetIp} \\ --host=${targetIp} \\
--ssh_user=root \\ --ssh_user=rocky \\
--ssh_private_key=${sshPrivateKey} \\ --ssh_private_key=${sshPrivateKey} \\
--ssh_args='-o StrictHostKeyChecking=no -o ConnectTimeout=10' \\ --ssh_args='-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null' \\
--no_sudo \\ --no_sudo \\
--sparrowfile=${sparrowfilePath} 2>&1 | tee logs/test.log --sparrowfile=${sparrowfilePath} \\
--verbose \\
--color 2>&1 | tee logs/test.log
""" """
echo "[${testName}] Test completed successfully" echo "[${testName}] Test completed successfully"

61
Jenkinsfile.simple Normal file
View File

@@ -0,0 +1,61 @@
pipeline {
agent { label 'fedora-testing' }
parameters {
string(name: 'QCOW2_URL', defaultValue: 'https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2')
text(name: 'TEST_REPOS', defaultValue: 'https://github.com/org/test1\nhttps://github.com/org/test2')
choice(name: 'MAX_PARALLEL', choices: ['1', '2', '3', '5'], description: 'Concurrent tests')
}
environment {
IMAGES_DIR = '/var/lib/libvirt/images'
BUILD_ID = "${env.BUILD_NUMBER}"
}
stages {
stage('Download') {
steps {
sh './scripts/download-image.sh "${QCOW2_URL}" "${IMAGES_DIR}" > base.txt'
script {
env.BASE_IMAGE = readFile('base.txt').trim()
}
}
}
stage('Build Golden') {
steps {
sh './scripts/build-golden.sh "${BASE_IMAGE}" "${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"'
script {
env.GOLDEN_IMAGE = "${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"
}
}
}
stage('Run Tests') {
steps {
script {
def repos = params.TEST_REPOS.split('\n').findAll { it.trim() }
def tasks = [:]
repos.each { repo ->
def testName = repo.tokenize('/').last().replace('.git', '')
tasks[testName] = {
lock(resource: 'vm-slots', quantity: 1) {
sh "./scripts/run-test.sh '${testName}' '${repo}' '${GOLDEN_IMAGE}'"
}
}
}
parallel tasks
}
}
}
}
post {
always {
sh "./scripts/cleanup-all.sh '.*-${BUILD_ID}' || true"
sh "rm -f '${IMAGES_DIR}/golden-${BUILD_ID}.qcow2' || true"
}
}
}

553
README.md
View File

@@ -1,388 +1,325 @@
# Sparrowdo Testing Orchestrator # Rocky Linux Testing Framework
Automated testing framework for Rocky Linux using Sparrowdo, QCOW2 images, and Jenkins. Simple, portable Sparrowdo testing framework for Rocky Linux.
## Overview ## Overview
This framework provides a production-ready continuous integration/testing system that: This framework provides automated testing for Rocky Linux:
- **Automates VM Provisioning**: Spins up isolated Rocky Linux VMs on-demand using QCOW2 images - **Simple Scripts**: 4 standalone bash scripts, easily portable
- **Runs Sparrowdo Tests in Parallel**: Executes your test suite across multiple isolated environments simultaneously - **VM Isolation**: Each test runs in a fresh VM
- **Supports Dynamic Configuration**: Runtime customization of base images, preparation scripts, test selection, and concurrency - **Parallel Execution**: Run multiple tests concurrently
- **Provides Web Interface**: Jenkins UI for triggering builds, viewing results, and downloading logs - **Fast Provisioning**: Linked clones (copy-on-write) for speed
- **Scales Across Machines**: Multiple team members can connect their desktops as Jenkins agents - **Jenkins Ready**: Simple Jenkinsfile orchestrates scripts
## Prerequisites
```bash
sudo dnf install -y qemu-kvm libvirt virt-install guestfs-tools rakudo
sudo systemctl enable --now libvirtd
sudo usermod -a -G libvirt $(whoami)
ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N ""
```
## Quick Start
### Manual Test Run
```bash
# 1. Download base image (cached automatically)
BASE=$(./scripts/download-image.sh \
https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2)
# 2. Build golden image (includes Sparrowdo bootstrap)
./scripts/build-golden.sh "$BASE" /var/lib/libvirt/images/golden.qcow2
# 3. Run test
./scripts/run-test.sh my-test \
https://github.com/your-org/test-repo.git \
/var/lib/libvirt/images/golden.qcow2
# 4. Cleanup
./scripts/cleanup-all.sh
```
### Jenkins
1. Create new Pipeline job in Jenkins
2. Point to this repository
3. Use `Jenkinsfile.simple`
4. Configure parameters:
- **QCOW2_URL**: Rocky Linux base image URL
- **TEST_REPOS**: One test repo URL per line
- **MAX_PARALLEL**: Number of concurrent tests
5. Build with Parameters
## How It Works ## How It Works
``` ```
User clicks "Build" in Jenkins Download QCOW2 → Build Golden Image → Run Tests in Parallel → Cleanup
↓ (includes bootstrap) (isolated VMs)
Download QCOW2 base image
↓
Run custom prep script (install packages, configure services)
↓
Create golden image
↓
Create linked clones for each test (fast - copy-on-write)
↓
Run Sparrowdo tests in parallel (isolated VMs)
↓
Collect logs and archive results
↓
Auto-cleanup VMs and temporary images
``` ```
**Golden Image Creation:**
1. Copy base QCOW2 image
2. Install Raku, Sparrowdo, dependencies via virt-customize
3. Create `rocky` user with sudo and SSH keys
4. Boot VM and run `sparrowdo --bootstrap` (once)
5. Shutdown VM, golden image ready
**Test Execution:**
1. Provision VM as linked clone (1 second)
2. Clone test repository
3. Run `sparrowdo --no_sudo` with test
4. Cleanup VM
## Scripts
### download-image.sh
```bash
./scripts/download-image.sh <url> [output_dir] [force]
```
Downloads and caches QCOW2 images. Returns path to cached image.
### build-golden.sh
```bash
./scripts/build-golden.sh <base_image> <golden_image> [ssh_pub_key]
```
Creates golden image from base QCOW2. Installs Raku/Sparrowdo and bootstraps.
### run-test.sh
```bash
./scripts/run-test.sh <test_name> <test_repo_url> <golden_image> [ssh_key]
```
Provisions VM, clones test repo, runs Sparrowdo test, cleans up.
### provision_vm.sh
```bash
./scripts/provision_vm.sh <vm_name> <golden_image> [timeout]
```
Creates VM as linked clone, starts it, returns IP address.
### cleanup_vm.sh
```bash
./scripts/cleanup_vm.sh <vm_name>
```
Destroys VM and removes disk image.
### cleanup-all.sh
```bash
./scripts/cleanup-all.sh [pattern]
```
Emergency cleanup for orphaned VMs matching pattern.
## Directory Structure ## Directory Structure
``` ```
. .
├── Jenkinsfile # Jenkins Pipeline definition ├── Jenkinsfile.simple # Simple Jenkins pipeline
├── README.md # This file ├── README.md # This file
├── scripts/ ├── scripts/
│ ├── setup_base.sh # Prepares golden image from QCOW2 │ ├── download-image.sh # Download/cache images
│ ├── provision_vm.sh # Creates and starts test VM │ ├── build-golden.sh # Create golden image
│ └── cleanup_vm.sh # Destroys VM and removes disk │ ├── run-test.sh # Run single test
├── tests/ │ ├── provision_vm.sh # Provision VM
│ └── (sample tests here) │ ├── cleanup_vm.sh # Cleanup VM
└── docs/ │ └── cleanup-all.sh # Emergency cleanup
└── (additional documentation) └── docs/ # Additional documentation
``` ```
## Prerequisites ## Test Repository Format
### On Jenkins Agent (Fedora/Rocky Linux Desktop) Your test repository needs `sparrowfile` or `main.raku`:
1. **KVM/QEMU/Libvirt** ```raku
```bash #!/usr/bin/env raku
sudo dnf install -y qemu-kvm libvirt virt-install libguestfs-tools-c use Sparrowdo;
sudo systemctl enable --now libvirtd
sudo usermod -a -G libvirt $(whoami)
```
2. **Sparrowdo** task-run 'check-sshd', %(
```bash plugin => 'systemd-service',
# Install Raku (Perl 6) args => ['sshd', 'running']
sudo dnf install -y rakudo );
# Install zef (Raku module manager) task-run 'verify-rocky-version', %(
git clone https://github.com/ugexe/zef.git plugin => 'shell-command',
cd zef && raku -I. bin/zef install . args => 'cat /etc/rocky-release'
);
# Install Sparrowdo
zef install Sparrowdo
```
3. **SSH Keys**
```bash
# Generate SSH key pair if needed
ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N ""
```
4. **Jenkins Agent**
- Connect your Fedora desktop as a Jenkins agent
- Label it as `fedora-testing`
- Ensure the agent user has sudo access for virsh/libvirt commands
## Quick Start
### 1. Clone Repository
```bash
cd ~/
git clone <your-repo-url> testing-orchestrator
cd testing-orchestrator
``` ```
### 2. Test Scripts Locally (Optional)
```bash
# Make scripts executable
chmod +x scripts/*.sh
# Download a test QCOW2 image
cd /var/lib/libvirt/images
curl -LO https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2
# Create golden image
cd ~/testing-orchestrator
./scripts/setup_base.sh \
/var/lib/libvirt/images/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2 \
"" \
/var/lib/libvirt/images/golden-test.qcow2 \
~/.ssh/id_rsa.pub
# Provision a test VM
./scripts/provision_vm.sh test-vm-1 /var/lib/libvirt/images/golden-test.qcow2
# Clean up
./scripts/cleanup_vm.sh test-vm-1
```
### 3. Configure Jenkins Job
1. In Jenkins, create a new **Pipeline** job
2. Name it "rocky-sparrowdo-tests"
3. Under "Pipeline", select "Pipeline script from SCM"
4. Set SCM to "Git" and provide your repository URL
5. Set Script Path to "Jenkinsfile"
6. Save
### 4. Run Your First Build
1. Click "Build with Parameters"
2. Configure parameters:
- **QCOW2_URL**: Rocky Linux base image URL
- **TEST_MATRIX**: JSON array of your Sparrowdo test repositories
- **GOLDEN_PREP_SCRIPT**: Customize image preparation
- **TEST_FILTER**: Regex to filter which tests run
- **MAX_CONCURRENT**: Number of parallel VMs
3. Click "Build"
## Configuration ## Configuration
### TEST_MATRIX Format ### Golden Image Users
The `TEST_MATRIX` parameter accepts a JSON array of test definitions: - **rocky**: Primary test user (password: `rockypass`, sudo: yes)
- **root**: Root user (password: `rockytesting`)
```json Both have SSH keys injected from `~/.ssh/id_rsa.pub`.
[
{ ### VM Resources
"name": "login-tests",
"url": "https://github.com/your-org/login-tests.git", Default per VM (edit `provision_vm.sh` to change):
"branch": "main", - Memory: 2048 MB
"description": "SSH login and connectivity tests" - CPUs: 2
}, - Disk: Linked clone (only diffs stored)
{
"name": "database-tests", ## Portability
"url": "https://github.com/your-org/database-tests.git",
"branch": "develop", All logic is in standalone shell scripts. Easy to integrate with:
"description": "PostgreSQL configuration tests"
} **GitHub Actions:**
] ```yaml
- name: Run test
run: ./scripts/run-test.sh my-test $REPO $GOLDEN
``` ```
Each test repository should contain a `sparrowfile` at the root or in a subdirectory. **GitLab CI:**
```yaml
test:
script:
- ./scripts/run-test.sh my-test $REPO $GOLDEN
```
### Custom Golden Image Preparation **Windmill:**
```typescript
The `GOLDEN_PREP_SCRIPT` parameter accepts a bash script that runs inside the QCOW2 image during preparation: await bash.run(`./scripts/run-test.sh ${name} ${repo} ${golden}`)
```
**Manually:**
```bash ```bash
#!/bin/bash ./scripts/run-test.sh my-test https://github.com/org/test.git golden.qcow2
set -e
# Update system
dnf update -y
# Install dependencies
dnf install -y perl git wget postgresql-server
# Configure services
systemctl enable postgresql
firewall-cmd --permanent --add-service=postgresql
echo "Custom preparation complete!"
``` ```
### Test Filtering
Use the `TEST_FILTER` parameter to run specific tests:
- `.*` - Run all tests
- `login.*` - Run all tests starting with "login"
- `database-postgres` - Run only the specific test
- `(api|integration).*` - Run API or integration tests
## Features
### Linked Clones for Speed
The framework uses QCOW2 linked clones (copy-on-write), which means:
- Creating a new VM takes seconds, not minutes
- Each test VM only stores differences from the golden image
- Disk space usage is minimal
### Parallel Execution with Isolation
- Each test runs in its own isolated VM
- Tests cannot interfere with each other
- Concurrency is controlled via `MAX_CONCURRENT` parameter
- Jenkins lock mechanism prevents desktop overload
### Automatic Cleanup
- VMs are automatically destroyed after each test
- Temporary images are removed after build
- Orphaned VMs from failed builds are cleaned up
- Optional: Keep golden image for debugging
### Test Result Archival
- Each test's output is captured in `logs/test.log`
- Logs are archived as Jenkins artifacts
- Download logs directly from Jenkins UI
## Troubleshooting ## Troubleshooting
### VM Won't Start ### List VMs
```bash ```bash
# Check libvirt status virsh -c qemu:///system list --all
sudo systemctl status libvirtd
# Verify image integrity
qemu-img info /var/lib/libvirt/images/golden-*.qcow2
# Check disk space
df -h /var/lib/libvirt/images/
``` ```
### VM Doesn't Get IP Address ### Get VM IP
```bash ```bash
# Check default network virsh -c qemu:///system domifaddr <vm-name>
sudo virsh net-list --all
sudo virsh net-start default # If it's not running
# Verify DHCP
sudo virsh net-dhcp-leases default
``` ```
### SSH Connection Fails ### SSH to VM
```bash ```bash
# Test SSH manually (get IP from Jenkins logs) ssh -i ~/.ssh/id_rsa rocky@<vm-ip>
ssh -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@<VM_IP>
# Check SSH key injection
sudo virt-cat -a /var/lib/libvirt/images/golden-*.qcow2 /root/.ssh/authorized_keys
``` ```
### Sparrowdo Test Fails ### View Console
```bash ```bash
# View test logs in Jenkins artifacts virsh -c qemu:///system console <vm-name>
# Or run Sparrowdo manually against VM: # Press Ctrl+] to exit
sparrowdo \
--host=<VM_IP> \
--ssh_user=root \
--ssh_private_key=~/.ssh/id_rsa \
--sparrowfile=/path/to/sparrowfile
``` ```
### Permission Denied Errors ### Check Network
```bash ```bash
# Ensure user is in libvirt group virsh -c qemu:///system net-list --all
sudo usermod -a -G libvirt $(whoami) virsh -c qemu:///system net-start default # If stopped
newgrp libvirt
# Fix image directory permissions
sudo chown -R $(whoami):$(whoami) /var/lib/libvirt/images
``` ```
### Jenkins Agent Offline ### Force Cleanup
```bash ```bash
# Check agent service (if running as systemd service) ./scripts/cleanup-all.sh
sudo systemctl status jenkins-agent rm -f /var/lib/libvirt/images/golden-*.qcow2
# View logs
sudo journalctl -u jenkins-agent -f
# Test connection from Jenkins controller
ssh jenkins@<agent-host> 'echo "Connection successful"'
``` ```
### Bootstrap Fails
```bash
# Check if VM is accessible
ssh -i ~/.ssh/id_rsa rocky@<vm-ip>
# Manually bootstrap
sparrowdo --host <vm-ip> --ssh_user rocky --bootstrap --color
```
## Performance
### Image Caching
- First download: ~5-10 minutes (2GB)
- Subsequent builds: 2 seconds (cached)
### Golden Image Build
- Base preparation: ~2-3 minutes (virt-customize)
- Bootstrap: ~5-10 minutes (sparrowdo --bootstrap)
- Total: ~7-13 minutes (once per build)
### Test Execution
- VM provision: ~30 seconds (boot + IP)
- Test runtime: Varies by test
- VM cleanup: ~2 seconds
### Example: 70 Tests
- Golden image: 10 minutes (once)
- 70 tests @ 3 concurrent: ~15 minutes
- **Total: 25 minutes**
Compare to bootstrapping each VM: 70 × 7 min = 490 minutes (8+ hours)!
## Advanced Usage ## Advanced Usage
### Custom Golden Image
```bash
# Create custom prep script
cat > custom-prep.sh << 'EOF'
#!/bin/bash
dnf install -y postgresql-server nginx
systemctl enable postgresql nginx
EOF
# Use virt-customize directly
sudo virt-customize -a golden.qcow2 \
--run custom-prep.sh \
--selinux-relabel
```
### Testing Beta Images ### Testing Beta Images
Change the `QCOW2_URL` parameter to point to beta images:
```
https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base-9.4-beta.x86_64.qcow2
```
### Custom VM Resources
Edit `scripts/provision_vm.sh` to adjust memory/CPU:
```bash ```bash
sudo virt-install \ BASE=$(./scripts/download-image.sh \
--name "$VM_NAME" \ https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base-9.5-beta.x86_64.qcow2 \
--memory 4096 \ # Increase memory /var/lib/libvirt/images true) # Force re-download
--vcpus 4 \ # Increase CPUs
...
``` ```
### Preserving Golden Images for Debugging ### Running Subset of Tests
1. Set `KEEP_GOLDEN_IMAGE` to `true`
2. After build, the golden image is preserved
3. Boot it manually for inspection:
```bash ```bash
VM_NAME="debug-vm" # In Jenkins, edit TEST_REPOS parameter
sudo virt-install \ https://github.com/org/test1.git
--name "$VM_NAME" \ https://github.com/org/test3.git
--memory 2048 \ # (omit test2)
--vcpus 2 \
--disk /var/lib/libvirt/images/golden-<BUILD_ID>.qcow2 \
--import \
--os-variant rocky9-unknown \
--network network=default
# Connect via console
sudo virsh console $VM_NAME
``` ```
### Running Tests Without Jenkins ### Debugging Test Failures
```bash ```bash
# Manual test execution # Run test manually and keep VM alive
./scripts/setup_base.sh \ VM_NAME="debug-vm-$$"
/path/to/base.qcow2 \ VM_IP=$(./scripts/provision_vm.sh "$VM_NAME" golden.qcow2 60 | tail -1)
/path/to/prep-script.sh \
/path/to/golden.qcow2 \
~/.ssh/id_rsa.pub
IP=$(./scripts/provision_vm.sh my-test-vm /path/to/golden.qcow2) # SSH to VM
ssh -i ~/.ssh/id_rsa rocky@$VM_IP
sparrowdo \ # Manually run test steps
--host=$IP \ cd /tmp
--ssh_user=root \ git clone https://github.com/org/test.git
--ssh_private_key=~/.ssh/id_rsa \ cd test
--sparrowfile=/path/to/test/sparrowfile sparrowdo --host localhost --ssh_user rocky --no_sudo --sparrowfile main.raku
./scripts/cleanup_vm.sh my-test-vm # Cleanup when done
./scripts/cleanup_vm.sh "$VM_NAME"
``` ```
## Contributing ## Contributing
### Adding New Tests This framework is designed to be simple and portable. Keep it that way:
1. Create a new Sparrowdo test repository - Scripts should be standalone bash
2. Add a `sparrowfile` at the root or in a subdirectory - Minimal dependencies
3. Add the test to the `TEST_MATRIX` parameter in Jenkins - Easy to read, no over-commenting
- Portable across CI/CD platforms
### Modifying Scripts
1. Test changes locally first
2. Update documentation if behavior changes
3. Commit and push to repository
4. Jenkins will use updated scripts on next build
## Support
For issues or questions:
- Check logs in Jenkins artifacts
- Review troubleshooting section above
- Verify prerequisites are installed correctly
## License ## License
[Specify your license here] [Your License Here]
## Authors ## Authors

103
README.simple.md Normal file
View File

@@ -0,0 +1,103 @@
# Rocky Linux Testing Framework
Simple, portable testing framework for Rocky Linux using Sparrowdo.
## Quick Start
### Prerequisites
```bash
sudo dnf install -y qemu-kvm libvirt virt-install guestfs-tools rakudo
sudo systemctl enable --now libvirtd
sudo usermod -a -G libvirt $(whoami)
```
### Manual Test
```bash
# Download base image
BASE=$(./scripts/download-image.sh https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2)
# Build golden image (includes Sparrowdo bootstrap)
./scripts/build-golden.sh "$BASE" /var/lib/libvirt/images/golden.qcow2
# Run a test
./scripts/run-test.sh my-test https://github.com/your-org/test-repo.git /var/lib/libvirt/images/golden.qcow2
# Cleanup
./scripts/cleanup-all.sh
```
### Jenkins
1. Create Pipeline job
2. Point to `Jenkinsfile.simple`
3. Set parameters:
- `QCOW2_URL`: Rocky Linux image URL
- `TEST_REPOS`: One repo URL per line
- `MAX_PARALLEL`: Number of concurrent tests
## Scripts
- `download-image.sh` - Download/cache QCOW2 images
- `build-golden.sh` - Create golden image with Sparrowdo
- `run-test.sh` - Run single test in isolated VM
- `provision_vm.sh` - Create and start VM
- `cleanup_vm.sh` - Destroy VM
- `cleanup-all.sh` - Emergency cleanup
## How It Works
1. Download Rocky Linux QCOW2 (cached)
2. Build golden image:
- Install Raku, Sparrowdo, dependencies
- Create `rocky` user with sudo
- Inject SSH keys
- Bootstrap Sparrowdo
3. For each test:
- Provision VM (linked clone, fast)
- Clone test repo
- Run `sparrowdo --no_sudo`
- Cleanup VM
## Test Repository Format
Your test repo needs either `sparrowfile` or `main.raku`:
```raku
#!/usr/bin/env raku
use Sparrowdo;
task-run 'check-service', %(
plugin => 'systemd-service',
args => ['sshd', 'running']
);
```
## Portability
All logic is in shell scripts. Easy to move to:
- GitHub Actions: Call scripts in workflow
- GitLab CI: Call scripts in `.gitlab-ci.yml`
- Windmill: Call scripts as tasks
- Manually: Run scripts directly
## Troubleshooting
```bash
# List VMs
virsh -c qemu:///system list --all
# Get VM IP
virsh -c qemu:///system domifaddr <vm-name>
# SSH to VM
ssh -i ~/.ssh/id_rsa rocky@<ip>
# View VM console
virsh -c qemu:///system console <vm-name>
# Force cleanup
./scripts/cleanup-all.sh
```

304
docs/BOOTSTRAP-APPROACH.md Normal file
View File

@@ -0,0 +1,304 @@
# Bootstrap Approach for Sparrowdo
## Overview
This framework bootstraps Sparrowdo **ONCE** in the golden image, not on every test VM. This provides significant time and resource savings.
## How It Works
### Traditional Approach (Slow) ❌
```
For each test:
1. Provision VM from golden image
2. Bootstrap Sparrowdo (5-10 minutes)
3. Run test
4. Destroy VM
```
**Problem**: If you run 70 tests, you bootstrap 70 times = 6-12 hours wasted!
### Our Approach (Fast) ✅
```
Once per golden image:
1. Create golden image with Raku/zef installed
2. Boot temporary VM from golden image
3. Bootstrap Sparrowdo (5-10 minutes)
4. Shutdown VM (changes saved to golden image)
For each test:
1. Provision VM from bootstrapped golden image
2. Run test immediately (no bootstrap needed!)
3. Destroy VM
```
**Benefit**: Bootstrap once, test 70 times = 5-10 minutes total!
## Implementation Details
### Step 1: Golden Image Preparation (Offline)
The `docs/default-prep.sh` script runs inside `virt-customize` (offline mode):
```bash
# Install Raku and zef package manager
dnf install -y rakudo rakudo-zef
# Create rocky user with sudo access
useradd -m rocky
echo "rocky ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/rocky
```
**Why offline?** This is fast and requires no network/VM boot.
### Step 2: Bootstrap Sparrowdo (Online - Once)
The `scripts/bootstrap_golden.sh` script:
1. **Boots temporary VM** from golden image
2. **Runs sparrowdo --bootstrap** via SSH
3. **Shuts down VM cleanly** (changes persist in golden image)
4. **Cleans up VM definition**
```bash
./scripts/bootstrap_golden.sh /path/to/golden.qcow2 ~/.ssh/id_rsa
```
**What bootstrap installs:**
- Sparrowdo Raku module
- All Sparrowdo dependencies
- Testing utilities
- Configuration files
### Step 3: Test Execution (Fast)
Each test VM is a **linked clone** of the bootstrapped golden image:
```bash
# Provision takes < 1 second (copy-on-write)
VM_IP=$(provision_vm.sh test-vm golden.qcow2)
# Run test immediately (no bootstrap!)
sparrowdo --host $VM_IP --ssh_user rocky --no_sudo --sparrowfile test.raku
```
## Manual Workflow
### First Time Setup
```bash
# 1. Create base golden image
./scripts/setup_base.sh \
/var/lib/libvirt/images/Rocky-9-GenericCloud-Base.qcow2 \
docs/default-prep.sh \
/var/lib/libvirt/images/golden-rocky9.qcow2 \
~/.ssh/id_rsa.pub
# 2. Bootstrap the golden image (one time, 5-10 minutes)
./scripts/bootstrap_golden.sh \
/var/lib/libvirt/images/golden-rocky9.qcow2 \
~/.ssh/id_rsa
```
### Running Tests
```bash
# No bootstrap needed! Just run tests directly
./scripts/provision_vm.sh test-vm-1 /var/lib/libvirt/images/golden-rocky9.qcow2
# ... run sparrowdo tests ...
./scripts/cleanup_vm.sh test-vm-1
```
## Jenkins Pipeline Flow
The Jenkinsfile automatically handles bootstrap:
```groovy
stage('Prepare Golden Image') {
// Creates golden image with Raku/zef
setup_base.sh → golden.qcow2 (with Raku)
}
stage('Bootstrap Golden Image') {
// Bootstraps Sparrowdo ONCE
bootstrap_golden.sh → golden.qcow2 (with Sparrowdo)
}
stage('Run Tests') {
parallel {
test1: provision → run test → cleanup
test2: provision → run test → cleanup
test3: provision → run test → cleanup
// No bootstrap in any test!
}
}
```
## Time Savings Example
### 70 Tests with Bootstrap-Per-Test
```
Bootstrap time: 7 minutes per test
Total bootstrap time: 70 × 7 = 490 minutes (8.2 hours)
Test time: 70 × 2 minutes = 140 minutes (2.3 hours)
TOTAL: 630 minutes (10.5 hours)
```
### 70 Tests with Bootstrap-Once
```
Bootstrap time: 7 minutes once
Total bootstrap time: 1 × 7 = 7 minutes
Test time: 70 × 2 minutes = 140 minutes (2.3 hours)
TOTAL: 147 minutes (2.5 hours)
```
**Savings: 8 hours!** (80% reduction in total time)
## Disk Space Considerations
### Without Linked Clones
```
Base image: 2 GB
Golden image: 2 GB
Test VMs: 70 × 2 GB = 140 GB
TOTAL: 144 GB
```
### With Linked Clones (Our Approach)
```
Base image: 2 GB (cached, reused)
Golden image: 2.5 GB (with Sparrowdo)
Test VMs: 70 × ~100 MB = 7 GB (only diffs stored)
TOTAL: 11.5 GB
```
**Savings: 132 GB!** (92% reduction in disk usage)
## Updating the Golden Image
If you need to update Sparrowdo or dependencies:
```bash
# Option 1: Rebuild from scratch
rm -f /var/lib/libvirt/images/golden-rocky9.qcow2
./scripts/setup_base.sh ... # Create fresh
./scripts/bootstrap_golden.sh ... # Bootstrap fresh
# Option 2: Update existing golden image
# Boot a VM from golden image, update packages, shutdown
VM_IP=$(./scripts/provision_vm.sh update-vm golden-rocky9.qcow2)
ssh rocky@$VM_IP 'zef upgrade Sparrowdo'
ssh rocky@$VM_IP 'sudo shutdown -h now'
# Changes are saved to golden image
```
## Troubleshooting
### Bootstrap Script Hangs
```bash
# Check if VM is running
virsh -c qemu:///system list | grep bootstrap
# Connect to VM console
virsh -c qemu:///system console bootstrap-golden-XXXXX
# Check bootstrap logs on VM
ssh rocky@VM_IP 'cat ~/.zef/*log'
```
### Test Fails with "Sparrowdo not found"
```bash
# Verify Sparrowdo is in golden image
ssh rocky@GOLDEN_VM_IP 'which sparrowdo'
ssh rocky@GOLDEN_VM_IP 'sparrowdo --version'
# If missing, re-run bootstrap
./scripts/bootstrap_golden.sh /path/to/golden.qcow2
```
### Bootstrap Fails on First Try
```bash
# Common issue: Network not ready during bootstrap
# Solution: Increase wait time in bootstrap script
# Or manually retry bootstrap command
sparrowdo --host VM_IP --ssh_user rocky --bootstrap --color
```
## Best Practices
1. **Cache base images** - Reuse downloaded QCOW2 files
2. **Bootstrap once per Rocky version** - Create golden-rocky8.qcow2, golden-rocky9.qcow2, etc.
3. **Version your golden images** - Use timestamps: golden-rocky9-20250125.qcow2
4. **Test golden images** - Always verify bootstrap succeeded before running full test suite
5. **Update periodically** - Rebuild golden images monthly to get security updates
## Security Considerations
### SSH Keys
- Golden image contains injected SSH public key
- Anyone with private key can SSH to any VM from this golden image
- **Recommendation**: Use dedicated testing SSH keys, not personal keys
### Passwords
- Rocky user password: `rockypass` (change in prep script if needed)
- Root password: `rockytesting`
- **Recommendation**: Disable password auth, use keys only
### Sudo Access
- Rocky user has NOPASSWD sudo (required for bootstrap)
- **Recommendation**: Only use these VMs in isolated test networks
## Advanced: Pre-cached Test Dependencies
You can extend the bootstrap to pre-install common test dependencies:
```bash
# In bootstrap_golden.sh, after sparrowdo --bootstrap:
ssh rocky@$VM_IP 'zef install Test::Class'
ssh rocky@$VM_IP 'zef install JSON::Fast'
ssh rocky@$VM_IP 'sudo dnf install -y postgresql-server'
```
This makes tests even faster by eliminating package install time during tests.
## Monitoring Bootstrap Success
The bootstrap script outputs:
```
[1/4] Provisioning temporary VM...
[2/4] Waiting for SSH to be ready...
[3/4] Running Sparrowdo bootstrap...
[4/4] Shutting down VM to save changes...
```
If any step fails, the golden image is NOT bootstrapped. Check logs and retry.
## Integration with CI/CD
### Nightly Golden Image Rebuild
```bash
# Cron job to rebuild golden images nightly
0 2 * * * cd /path/to/repo && ./scripts/rebuild-golden.sh
```
### Pre-commit Hook to Validate Tests
```bash
# .git/hooks/pre-commit
./scripts/validate-tests.sh
# Provisions temp VM, runs one test, destroys
```
### Jenkins Scheduled Build
```groovy
// Rebuild golden images weekly
cron('H 2 * * 0') // Sunday 2 AM
```
## Conclusion
Bootstrapping the golden image once provides:
- **10x faster test execution** (no per-test bootstrap)
- **90% less disk usage** (linked clones vs full copies)
- **Simpler test scripts** (no bootstrap logic needed)
- **Better reliability** (bootstrap failures affect one build, not all tests)
This approach is essential for running large test suites efficiently!

225
docs/CHANGES.md Normal file
View File

@@ -0,0 +1,225 @@
# Recent Changes Summary
## User Configuration Changes
### Non-Root User (rocky)
- **Changed from**: `root` user for testing
- **Changed to**: `rocky` user (standard Rocky Linux user)
- **Reason**: Sparrowdo best practices require non-root user with sudo privileges
### SSH Key Injection
- SSH keys are now injected for both `root` and `rocky` users
- The `rocky` user has NOPASSWD sudo access (required for Sparrowdo bootstrap)
### Sparrowdo Bootstrap Step
- **New requirement**: Must run `sparrowdo --bootstrap` before running tests
- Bootstrap installs necessary Raku/Sparrowdo dependencies on the target VM
- Bootstrap runs with the `rocky` user
## Test Execution Flow
### Old Flow (Root User)
```bash
1. Provision VM
2. Wait for SSH (root user)
3. Run sparrowdo test (root user, --no_sudo)
```
### New Flow (Rocky User with Bootstrap)
```bash
1. Provision VM
2. Wait for SSH (rocky user)
3. Run sparrowdo --bootstrap (rocky user)
4. Run sparrowdo test (rocky user, --no_sudo, --verbose, --color)
```
## Sparrowfile Naming
- Tests can use either `sparrowfile` or `main.raku`
- The framework checks for `main.raku` first, then falls back to `sparrowfile`
## Command Examples
### Manual Bootstrap
```bash
sparrowdo \
--host 192.168.124.54 \
--ssh_user rocky \
--ssh_private_key ~/.ssh/id_rsa \
--bootstrap \
--color
```
### Manual Test Run
```bash
sparrowdo \
--host 192.168.124.54 \
--ssh_user rocky \
--ssh_private_key ~/.ssh/id_rsa \
--no_sudo \
--sparrowfile test-repo/main.raku \
--verbose \
--color
```
## Golden Image Changes
### Users Created
1. **rocky** (primary test user)
- Password: `rockypass`
- Sudo: NOPASSWD:ALL
- SSH key: Injected from `~/.ssh/id_rsa.pub`
2. **testuser** (backward compatibility)
- Password: `testpass`
- Sudo: NOPASSWD:ALL
- SSH key: Not injected
3. **root**
- Password: `rockytesting`
- SSH key: Injected from `~/.ssh/id_rsa.pub`
## Bug Fixes
### 1. virt-customize D-Bus Errors ✅
- **Problem**: Prep scripts used `firewall-cmd`, `hostnamectl` which don't work offline
- **Solution**: Removed D-Bus dependent commands from prep scripts
- **Impact**: Golden image creation now works reliably
### 2. VM Provisioning Script Failures ✅
- **Problem**: `set -e` caused script to exit on arithmetic operations
- **Solution**: Removed `set -e`, added explicit error handling
- **Impact**: VMs provision successfully and report proper errors
### 3. Virsh Connection Issues ✅
- **Problem**: Non-root users couldn't access libvirt without explicit URI
- **Solution**: Added `-c qemu:///system` to all virsh commands
- **Impact**: Scripts work for jenkins user and other non-root users
### 4. Image Caching ✅
- **Problem**: Framework re-downloaded QCOW2 images on every build
- **Solution**: Cache images by filename, copy for each build
- **Impact**: Massive time savings (2GB download → 2 second copy)
- **Control**: `REDOWNLOAD_IMAGE` parameter forces fresh download when needed
### 5. IP Address Capture in Manual Script ✅
- **Problem**: Script captured all output including progress messages
- **Solution**: Extract only the last line (the IP address)
- **Impact**: SSH connection works properly
## File Changes
### Modified Files
- `Jenkinsfile` - Added bootstrap step, rocky user, main.raku support
- `scripts/setup_base.sh` - Inject SSH keys for rocky user
- `scripts/provision_vm.sh` - Fixed set -e issue, improved error handling
- `scripts/cleanup_vm.sh` - Added explicit qemu:///system connection
- `docs/default-prep.sh` - Create rocky user, remove D-Bus commands
- `docs/manual-test-run.sh` - Add bootstrap step, use rocky user
- `README.md` - Updated prerequisites (guestfs-tools)
### New Files
- `docs/virt-customize-guide.md` - Comprehensive guide on offline image customization
- `docs/manual-steps.md` - Step-by-step manual testing guide
- `docs/CHANGES.md` - This file
## Testing Checklist
### Before Running Tests
- [ ] SSH keys exist (`~/.ssh/id_rsa` and `~/.ssh/id_rsa.pub`)
- [ ] libvirt is running (`sudo systemctl status libvirtd`)
- [ ] Default network is active (`virsh -c qemu:///system net-list`)
- [ ] User has sudo access
- [ ] guestfs-tools installed (`which virt-customize`)
- [ ] Sparrowdo installed (`which sparrowdo`)
### Golden Image Verification
```bash
# After creating golden image, verify users exist
sudo virt-customize -a /path/to/golden.qcow2 --run-command 'id rocky'
sudo virt-customize -a /path/to/golden.qcow2 --run-command 'cat /home/rocky/.ssh/authorized_keys'
```
### Manual VM Test
```bash
# Provision a test VM
VM_IP=$(./scripts/provision_vm.sh test-vm-$(date +%s) /path/to/golden.qcow2 60)
# Test SSH as rocky user
ssh -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no rocky@$VM_IP
# Test sudo access
ssh -i ~/.ssh/id_rsa rocky@$VM_IP 'sudo whoami'
# Bootstrap Sparrowdo
sparrowdo --host $VM_IP --ssh_user rocky --bootstrap --color
# Clean up
./scripts/cleanup_vm.sh test-vm-XXXXX
```
## Migration Notes
### For Existing Tests
If you have existing Sparrowdo tests that assumed root user:
1. **Update TEST_MATRIX** in Jenkins to use rocky user
2. **Ensure tests use --no_sudo** flag
3. **Add bootstrap step** before test execution
4. **Verify sudoers access** if tests need elevated privileges
### Jenkins Pipeline Changes
The Jenkinsfile automatically handles:
- Creating rocky user in golden image
- Injecting SSH keys for rocky user
- Running bootstrap before tests
- Using `--no_sudo` flag with rocky user
No manual intervention needed for Jenkins builds.
## Performance Improvements
### Image Caching
- **First build**: Downloads 2GB QCOW2 image (~5-10 minutes)
- **Subsequent builds**: Copies from cache (~2 seconds)
- **Disk usage**: One cached image + one per active build
- **Cleanup**: Cached images persist, build images auto-delete
### VM Provisioning Speed
- **Linked clones**: New VM disk created in < 1 second
- **Boot time**: ~10-20 seconds to get IP address
- **Total provision time**: ~30 seconds from start to SSH ready
## Troubleshooting
### Bootstrap Fails
```bash
# Check if rocky user has sudo
ssh -i ~/.ssh/id_rsa rocky@VM_IP 'sudo whoami'
# Check if perl is installed
ssh -i ~/.ssh/id_rsa rocky@VM_IP 'which perl'
# Re-run bootstrap with verbose output
sparrowdo --host VM_IP --ssh_user rocky --bootstrap --color
```
### Test Fails with Permission Denied
```bash
# Verify SSH key is injected
ssh -i ~/.ssh/id_rsa rocky@VM_IP 'cat ~/.ssh/authorized_keys'
# Check if using correct user
# Should be rocky@VM_IP not root@VM_IP
```
### VM Won't Get IP
```bash
# Check network status
virsh -c qemu:///system net-list --all
# Start default network if needed
sudo virsh net-start default
# Check DHCP leases
virsh -c qemu:///system net-dhcp-leases default
```

61
docs/default-prep.sh Executable file
View File

@@ -0,0 +1,61 @@
#!/bin/bash
set -e
echo "Preparing standard golden image..."
# NOTE: This script runs inside virt-customize (offline mode)
# Cannot use systemctl, firewall-cmd, or other D-Bus dependent commands
# Use systemctl enable only (works offline), or direct file manipulation
# Update system (optional - can be slow)
# Uncomment if you want latest packages:
# dnf update -y
# Install common testing dependencies including Raku/Sparrowdo
dnf install -y \
perl \
git \
wget \
tar \
openssh-server \
vim \
rakudo \
rakudo-zef
# Enable services (these work in offline mode)
# systemctl enable works by creating symlinks, no D-Bus needed
systemctl enable sshd
# Create rocky user (standard non-root user for Rocky Linux)
useradd -m rocky 2>/dev/null || true
echo "rocky:rockypass" | chpasswd
# Add rocky user to sudoers
echo "rocky ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/rocky
chmod 0440 /etc/sudoers.d/rocky
# Install Sparrowdo for rocky user
# Note: This needs to run as rocky user, but we're in offline mode
# So we prepare the environment, and actual Sparrowdo install happens on first boot
mkdir -p /home/rocky/.sparrowdo-bootstrap
cat > /home/rocky/.sparrowdo-bootstrap/install.sh << 'BOOTSTRAP_EOF'
#!/bin/bash
# This script will be run on first boot by rocky user
if [ ! -f /home/rocky/.sparrowdo-installed ]; then
zef install --/test Sparrowdo
touch /home/rocky/.sparrowdo-installed
fi
BOOTSTRAP_EOF
chmod +x /home/rocky/.sparrowdo-bootstrap/install.sh
chown -R rocky:rocky /home/rocky/.sparrowdo-bootstrap
# Create testuser for backward compatibility
useradd -m testuser 2>/dev/null || true
echo "testuser:testpass" | chpasswd
# Add testuser to sudoers
echo "testuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/testuser
chmod 0440 /etc/sudoers.d/testuser
echo "Golden image preparation complete!"
echo "NOTE: Sparrowdo will be installed on first SSH connection by rocky user"

431
docs/manual-steps.md Normal file
View File

@@ -0,0 +1,431 @@
# Manual Test Run - Step by Step
This guide provides the exact commands Jenkins runs, so you can execute them manually for testing.
## Quick Start - Automated Script
I've created an interactive script that walks through all steps:
```bash
cd /Users/ssimpson/Documents/git/resf-test-jenkins
# Edit the configuration in the script first:
vim docs/manual-test-run.sh
# Update: TEST_REPO_URL, TEST_REPO_BRANCH, TEST_NAME
# Run the interactive script:
./docs/manual-test-run.sh
```
The script will pause at each step so you can see what's happening.
---
## Manual Step-by-Step Commands
If you prefer to run each command manually, here's the exact sequence:
### Prerequisites Check
```bash
# Verify required tools
which qemu-img virsh virt-install virt-customize sparrowdo git curl
# Verify libvirt is running
sudo systemctl status libvirtd
# Verify SSH keys exist
ls -la ~/.ssh/id_rsa ~/.ssh/id_rsa.pub
```
---
### Step 1: Set Variables
```bash
# Set your configuration
export BUILD_ID="manual-$(date +%s)"
export IMAGES_DIR="/var/lib/libvirt/images"
export WORK_DIR="/Users/ssimpson/Documents/git/resf-test-jenkins/manual-test-${BUILD_ID}"
export SCRIPT_DIR="/Users/ssimpson/Documents/git/resf-test-jenkins"
# QCOW2 Image URL
export QCOW2_URL="https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2"
# SSH Keys
export SSH_PRIVATE_KEY="${HOME}/.ssh/id_rsa"
export SSH_PUBLIC_KEY="${HOME}/.ssh/id_rsa.pub"
# Test Configuration - EDIT THESE
export TEST_NAME="my-test"
export TEST_REPO_URL="https://github.com/your-org/your-test-repo.git"
export TEST_REPO_BRANCH="main"
# Image download behavior (set to "true" to force re-download)
export REDOWNLOAD_IMAGE="false"
echo "Configuration set for build: ${BUILD_ID}"
```
---
### Step 2: Initialize Environment
```bash
# Create working directory
mkdir -p "${WORK_DIR}"
cd "${WORK_DIR}"
# Ensure images directory exists and is writable
sudo mkdir -p "${IMAGES_DIR}"
sudo chown ${USER}:${USER} "${IMAGES_DIR}"
# Verify scripts are executable
chmod +x "${SCRIPT_DIR}/scripts"/*.sh
echo "Environment initialized"
```
---
### Step 3: Download Base QCOW2 Image
```bash
# Extract filename from URL for caching
export IMAGE_FILENAME=$(basename "${QCOW2_URL}")
export CACHED_IMAGE="${IMAGES_DIR}/${IMAGE_FILENAME}"
export BASE_IMAGE="${IMAGES_DIR}/base-${BUILD_ID}.qcow2"
echo "Image URL: ${QCOW2_URL}"
echo "Cached image: ${CACHED_IMAGE}"
echo "Build image: ${BASE_IMAGE}"
if [ "${REDOWNLOAD_IMAGE}" = "true" ]; then
echo "REDOWNLOAD_IMAGE is enabled - forcing fresh download"
curl -L --progress-bar -o "${BASE_IMAGE}" "${QCOW2_URL}"
else
if [ -f "${CACHED_IMAGE}" ]; then
echo "Found cached image, creating copy for this build..."
cp "${CACHED_IMAGE}" "${BASE_IMAGE}"
else
echo "No cached image found, downloading..."
curl -L --progress-bar -o "${CACHED_IMAGE}" "${QCOW2_URL}"
echo "Creating copy for this build..."
cp "${CACHED_IMAGE}" "${BASE_IMAGE}"
fi
fi
# Verify the image
echo "Verifying image..."
qemu-img info "${BASE_IMAGE}" | head -10
echo "Base image ready: ${BASE_IMAGE}"
```
---
### Step 4: Prepare Golden Image
```bash
# Set golden image path
export GOLDEN_IMAGE="${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"
# Set prep script (you can customize this)
export PREP_SCRIPT="${SCRIPT_DIR}/docs/default-prep.sh"
echo "Creating golden image..."
echo "Base: ${BASE_IMAGE}"
echo "Golden: ${GOLDEN_IMAGE}"
echo "Prep Script: ${PREP_SCRIPT}"
echo "SSH Key: ${SSH_PUBLIC_KEY}"
# Run the setup script
"${SCRIPT_DIR}/scripts/setup_base.sh" \
"${BASE_IMAGE}" \
"${PREP_SCRIPT}" \
"${GOLDEN_IMAGE}" \
"${SSH_PUBLIC_KEY}"
echo "Golden image ready: ${GOLDEN_IMAGE}"
```
---
### Step 5: Provision Test VM
```bash
# Set VM name
export VM_NAME="${TEST_NAME}-${BUILD_ID}"
echo "Provisioning VM: ${VM_NAME}"
# Provision the VM and capture its IP
export VM_IP=$("${SCRIPT_DIR}/scripts/provision_vm.sh" "${VM_NAME}" "${GOLDEN_IMAGE}" 60)
if [ "$VM_IP" = "ERROR" ] || [ -z "$VM_IP" ]; then
echo "ERROR: Failed to provision VM"
exit 1
fi
echo "VM provisioned successfully"
echo "VM Name: ${VM_NAME}"
echo "IP Address: ${VM_IP}"
# Wait for SSH to be ready
echo "Waiting for SSH to be ready..."
for i in {1..30}; do
if ssh -i "${SSH_PRIVATE_KEY}" \
-o StrictHostKeyChecking=no \
-o ConnectTimeout=5 \
-o UserKnownHostsFile=/dev/null \
root@${VM_IP} 'echo "SSH ready"' 2>/dev/null; then
echo "SSH connection established!"
break
fi
echo "Attempt $i/30..."
sleep 2
done
# Test SSH connection manually
echo "Testing SSH connection..."
ssh -i "${SSH_PRIVATE_KEY}" \
-o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null \
root@${VM_IP} 'hostname && cat /etc/rocky-release'
```
---
### Step 6: Clone Test Repository
```bash
# Create test workspace
export TEST_WORKSPACE="${WORK_DIR}/test-workspace"
mkdir -p "${TEST_WORKSPACE}"
cd "${TEST_WORKSPACE}"
echo "Cloning test repository..."
echo "URL: ${TEST_REPO_URL}"
echo "Branch: ${TEST_REPO_BRANCH}"
# Clone the repository
git clone -b "${TEST_REPO_BRANCH}" "${TEST_REPO_URL}" test-repo
# Find the sparrowfile
export SPARROWFILE=$(find test-repo -name sparrowfile -type f | head -1)
if [ -z "$SPARROWFILE" ]; then
echo "ERROR: No sparrowfile found in repository"
exit 1
fi
echo "Found sparrowfile: ${SPARROWFILE}"
# Show sparrowfile contents
echo "Sparrowfile contents:"
cat "${SPARROWFILE}"
```
---
### Step 7: Run Sparrowdo Test
```bash
# Create logs directory
mkdir -p "${TEST_WORKSPACE}/logs"
echo "=========================================="
echo "Running Sparrowdo Test"
echo "=========================================="
echo "Target: root@${VM_IP}"
echo "Sparrowfile: ${SPARROWFILE}"
echo "Log: ${TEST_WORKSPACE}/logs/test.log"
echo ""
# Run the test
timeout 900 sparrowdo \
--host="${VM_IP}" \
--ssh_user=root \
--ssh_private_key="${SSH_PRIVATE_KEY}" \
--ssh_args="-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null" \
--no_sudo \
--sparrowfile="${SPARROWFILE}" 2>&1 | tee "${TEST_WORKSPACE}/logs/test.log"
echo ""
echo "Test completed!"
echo "Logs saved to: ${TEST_WORKSPACE}/logs/test.log"
```
---
### Step 8: Cleanup
```bash
echo "=========================================="
echo "Cleanup"
echo "=========================================="
# Destroy the VM
echo "Destroying VM: ${VM_NAME}"
"${SCRIPT_DIR}/scripts/cleanup_vm.sh" "${VM_NAME}"
# Optional: Remove temporary images
echo ""
echo "Temporary images:"
echo " Base: ${BASE_IMAGE}"
echo " Golden: ${GOLDEN_IMAGE}"
echo ""
read -p "Remove temporary images? (y/n): " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
echo "Removing images..."
sudo rm -f "${BASE_IMAGE}"
sudo rm -f "${GOLDEN_IMAGE}"
echo "Images removed"
else
echo "Images preserved at:"
echo " ${BASE_IMAGE}"
echo " ${GOLDEN_IMAGE}"
fi
```
---
### Step 9: View Results
```bash
echo "=========================================="
echo "Test Run Summary"
echo "=========================================="
echo "Build ID: ${BUILD_ID}"
echo "Test Name: ${TEST_NAME}"
echo "VM Name: ${VM_NAME}"
echo "Working Directory: ${WORK_DIR}"
echo "Test Logs: ${TEST_WORKSPACE}/logs/test.log"
echo ""
# View the test log
echo "=== Test Log ==="
cat "${TEST_WORKSPACE}/logs/test.log"
```
---
## Troubleshooting During Manual Run
### VM Won't Start
```bash
# Check libvirt status
sudo systemctl status libvirtd
# List all VMs
virsh list --all
# Check VM details
virsh dominfo "${VM_NAME}"
# View VM console
virsh console "${VM_NAME}" # Ctrl+] to exit
```
### Can't Get IP Address
```bash
# Check network
virsh net-list --all
virsh net-info default
# View DHCP leases
virsh net-dhcp-leases default
# Try alternative IP detection
virsh domifaddr "${VM_NAME}" --source lease
virsh domifaddr "${VM_NAME}" --source agent
virsh domifaddr "${VM_NAME}" --source arp
```
### SSH Connection Fails
```bash
# Test SSH manually with verbose output
ssh -vvv -i "${SSH_PRIVATE_KEY}" \
-o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null \
root@${VM_IP}
# Check if SSH key was injected
sudo virt-cat -a "${GOLDEN_IMAGE}" /root/.ssh/authorized_keys
# Check VM network from inside
virsh console "${VM_NAME}"
# Then inside VM:
# ip addr
# systemctl status sshd
# firewall-cmd --list-all
```
### Sparrowdo Fails
```bash
# Test connectivity first
ssh -i "${SSH_PRIVATE_KEY}" \
-o StrictHostKeyChecking=no \
root@${VM_IP} 'which perl && perl --version'
# Run sparrowdo with debug output
sparrowdo \
--host="${VM_IP}" \
--ssh_user=root \
--ssh_private_key="${SSH_PRIVATE_KEY}" \
--verbose \
--sparrowfile="${SPARROWFILE}"
```
### Clean Up Failed VMs
```bash
# List all VMs
virsh list --all
# Force cleanup
for vm in $(virsh list --all --name | grep "${BUILD_ID}"); do
virsh destroy "$vm" 2>/dev/null || true
virsh undefine "$vm" 2>/dev/null || true
sudo rm -f "/var/lib/libvirt/images/${vm}.qcow2"
done
```
---
## Quick Cleanup Script
If something goes wrong and you need to clean up everything:
```bash
# Save this as cleanup-all.sh
#!/bin/bash
BUILD_ID="${1:-manual}"
echo "Cleaning up build: ${BUILD_ID}"
# Stop and remove VMs
for vm in $(virsh list --all --name | grep "${BUILD_ID}"); do
echo "Removing VM: $vm"
virsh destroy "$vm" 2>/dev/null || true
virsh undefine "$vm" 2>/dev/null || true
sudo rm -f "/var/lib/libvirt/images/${vm}.qcow2"
done
# Remove images
sudo rm -f "/var/lib/libvirt/images/base-${BUILD_ID}.qcow2"
sudo rm -f "/var/lib/libvirt/images/golden-${BUILD_ID}.qcow2"
# Remove working directory
rm -rf "/Users/ssimpson/Documents/git/resf-test-jenkins/manual-test-${BUILD_ID}"
echo "Cleanup complete"
```
Then run:
```bash
chmod +x cleanup-all.sh
./cleanup-all.sh manual-1234567890 # Use your BUILD_ID
```

361
docs/manual-test-run.sh Executable file
View File

@@ -0,0 +1,361 @@
#!/bin/bash
#
# Manual Test Run - Simulates Jenkins Pipeline Steps
# This script walks through all the steps Jenkins would take
#
set -e
# Colors for output
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color
echo_step() {
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}$1${NC}"
echo -e "${BLUE}========================================${NC}"
}
echo_info() {
echo -e "${GREEN}[INFO]${NC} $1"
}
echo_warn() {
echo -e "${YELLOW}[WARN]${NC} $1"
}
echo_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
# =============================================================================
# CONFIGURATION - Edit these variables
# =============================================================================
# Build ID (simulating Jenkins BUILD_NUMBER)
BUILD_ID="manual-$(date +%s)"
# QCOW2 Image URL
QCOW2_URL="https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2"
# Image storage directory
IMAGES_DIR="/var/lib/libvirt/images"
# Image download behavior
# Set to "true" to force re-download even if cached image exists
REDOWNLOAD_IMAGE="false"
# SSH Keys
SSH_PRIVATE_KEY="${HOME}/.ssh/id_rsa"
SSH_PUBLIC_KEY="${HOME}/.ssh/id_rsa.pub"
# Test repository configuration
# Edit this to point to your actual test repository
TEST_NAME="example-test"
TEST_REPO_URL="https://github.com/your-org/your-test-repo.git"
TEST_REPO_BRANCH="main"
# Working directory (this script's location)
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
WORK_DIR="${SCRIPT_DIR}/manual-test-${BUILD_ID}"
# Golden image prep script location
PREP_SCRIPT="${SCRIPT_DIR}/docs/default-prep.sh"
# =============================================================================
# Step 1: Initialize
# =============================================================================
echo_step "Step 1: Initialize Environment"
echo_info "Build ID: ${BUILD_ID}"
echo_info "Working Directory: ${WORK_DIR}"
echo_info "Images Directory: ${IMAGES_DIR}"
echo_info "Scripts Directory: ${SCRIPT_DIR}/scripts"
# Create working directory
mkdir -p "${WORK_DIR}"
cd "${WORK_DIR}"
# Ensure images directory exists
sudo mkdir -p "${IMAGES_DIR}"
sudo chown ${USER}:${USER} "${IMAGES_DIR}"
# Make scripts executable
chmod +x "${SCRIPT_DIR}/scripts"/*.sh
echo_info "Initialization complete"
echo ""
read -p "Press Enter to continue to Step 2..."
# =============================================================================
# Step 2: Download Base Image
# =============================================================================
echo_step "Step 2: Download Base QCOW2 Image"
# Extract filename from URL for caching
IMAGE_FILENAME=$(basename "${QCOW2_URL}")
CACHED_IMAGE="${IMAGES_DIR}/${IMAGE_FILENAME}"
BASE_IMAGE="${IMAGES_DIR}/base-${BUILD_ID}.qcow2"
echo_info "Image URL: ${QCOW2_URL}"
echo_info "Cached image path: ${CACHED_IMAGE}"
echo_info "Build-specific image: ${BASE_IMAGE}"
echo_info "Redownload setting: ${REDOWNLOAD_IMAGE}"
echo ""
if [ "${REDOWNLOAD_IMAGE}" = "true" ]; then
echo_warn "REDOWNLOAD_IMAGE is enabled - forcing fresh download"
echo_info "Downloading from: ${QCOW2_URL}"
curl -L --progress-bar -o "${BASE_IMAGE}" "${QCOW2_URL}"
else
if [ -f "${CACHED_IMAGE}" ]; then
echo_info "Found cached image: ${CACHED_IMAGE}"
echo_info "Creating copy for build ${BUILD_ID}..."
cp "${CACHED_IMAGE}" "${BASE_IMAGE}"
else
echo_info "No cached image found at: ${CACHED_IMAGE}"
echo_info "Downloading from: ${QCOW2_URL}"
curl -L --progress-bar -o "${CACHED_IMAGE}" "${QCOW2_URL}"
echo_info "Creating copy for build ${BUILD_ID}..."
cp "${CACHED_IMAGE}" "${BASE_IMAGE}"
fi
fi
echo ""
echo_info "Verifying image..."
qemu-img info "${BASE_IMAGE}" | head -10
echo ""
read -p "Press Enter to continue to Step 3..."
# =============================================================================
# Step 3: Prepare Golden Image
# =============================================================================
echo_step "Step 3: Prepare Golden Image"
GOLDEN_IMAGE="${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"
echo_info "Golden image will be created at: ${GOLDEN_IMAGE}"
echo_info "Using prep script: ${PREP_SCRIPT}"
echo_info "Using SSH public key: ${SSH_PUBLIC_KEY}"
# Check if prep script exists
if [ ! -f "${PREP_SCRIPT}" ]; then
echo_error "Prep script not found: ${PREP_SCRIPT}"
echo_info "You can create it or use the default one in docs/"
exit 1
fi
# Check if SSH keys exist
if [ ! -f "${SSH_PUBLIC_KEY}" ]; then
echo_error "SSH public key not found: ${SSH_PUBLIC_KEY}"
echo_info "Generate one with: ssh-keygen -t rsa -b 4096 -f ${HOME}/.ssh/id_rsa"
exit 1
fi
echo_info "Running setup_base.sh..."
"${SCRIPT_DIR}/scripts/setup_base.sh" \
"${BASE_IMAGE}" \
"${PREP_SCRIPT}" \
"${GOLDEN_IMAGE}" \
"${SSH_PUBLIC_KEY}"
echo_info "Golden image ready: ${GOLDEN_IMAGE}"
echo ""
read -p "Press Enter to continue to Step 4 (Bootstrap)..."
# =============================================================================
# Step 4: Bootstrap Golden Image
# =============================================================================
echo_step "Step 4: Bootstrap Golden Image with Sparrowdo"
echo_info "Bootstrapping installs Sparrowdo and dependencies ONCE in the golden image"
echo_info "This allows all test VMs to skip bootstrap and run tests immediately"
echo ""
"${SCRIPT_DIR}/scripts/bootstrap_golden.sh" "${GOLDEN_IMAGE}" "${SSH_PRIVATE_KEY}"
echo ""
read -p "Press Enter to continue to Step 5 (Provision Test VM)..."
# =============================================================================
# Step 5: Provision Test VM
# =============================================================================
echo_step "Step 5: Provision Test VM"
VM_NAME="${TEST_NAME}-${BUILD_ID}"
echo_info "Creating VM: ${VM_NAME}"
echo_info "Using golden image: ${GOLDEN_IMAGE}"
# Provision VM and capture output
PROVISION_OUTPUT=$("${SCRIPT_DIR}/scripts/provision_vm.sh" "${VM_NAME}" "${GOLDEN_IMAGE}" 60)
# Extract just the IP address (last line of output)
VM_IP=$(echo "$PROVISION_OUTPUT" | tail -1)
if [ "$VM_IP" = "ERROR" ] || [ -z "$VM_IP" ]; then
echo_error "Failed to provision VM"
echo_error "Provision output:"
echo "$PROVISION_OUTPUT"
exit 1
fi
echo_info "VM provisioned successfully"
echo_info "IP Address: ${VM_IP}"
# Wait for SSH to be ready (test with rocky user)
echo_info "Waiting for SSH to be ready (testing with rocky user)..."
for i in {1..30}; do
if ssh -i "${SSH_PRIVATE_KEY}" \
-o StrictHostKeyChecking=no \
-o ConnectTimeout=5 \
-o UserKnownHostsFile=/dev/null \
rocky@${VM_IP} 'echo "SSH ready"' 2>/dev/null; then
echo_info "SSH connection established for rocky user"
break
fi
echo_info "Attempt $i/30..."
sleep 2
done
echo ""
read -p "Press Enter to continue to Step 6 (Clone Test Repository)..."
# =============================================================================
# Step 6: Clone Test Repository
# =============================================================================
echo_step "Step 6: Clone Test Repository"
echo_info "Cloning: ${TEST_REPO_URL}"
echo_info "Branch: ${TEST_REPO_BRANCH}"
# Create test workspace
TEST_WORKSPACE="${WORK_DIR}/test-workspace"
mkdir -p "${TEST_WORKSPACE}"
cd "${TEST_WORKSPACE}"
# Clone the test repository
if git clone -b "${TEST_REPO_BRANCH}" "${TEST_REPO_URL}" test-repo; then
echo_info "Repository cloned successfully"
else
echo_error "Failed to clone repository"
echo_warn "Cleanup will still run. Press Ctrl+C to abort or Enter to continue cleanup..."
read
"${SCRIPT_DIR}/scripts/cleanup_vm.sh" "${VM_NAME}"
exit 1
fi
# Find sparrowfile (look for main.raku or sparrowfile)
SPARROWFILE=$(find test-repo -name main.raku -type f | head -1)
if [ -z "$SPARROWFILE" ]; then
SPARROWFILE=$(find test-repo -name sparrowfile -type f | head -1)
fi
if [ -z "$SPARROWFILE" ]; then
echo_error "No sparrowfile or main.raku found in repository"
echo_warn "Cleanup will still run. Press Ctrl+C to abort or Enter to continue cleanup..."
read
"${SCRIPT_DIR}/scripts/cleanup_vm.sh" "${VM_NAME}"
exit 1
fi
echo_info "Found sparrowfile: ${SPARROWFILE}"
echo ""
read -p "Press Enter to continue to Step 7 (Run Test)..."
# =============================================================================
# Step 7: Run Sparrowdo Test
# =============================================================================
echo_step "Step 7: Run Sparrowdo Test"
# Create logs directory
mkdir -p logs
echo_info "Running Sparrowdo test against VM..."
echo_info "Target: rocky@${VM_IP}"
echo_info "Sparrowfile: ${SPARROWFILE}"
# Run sparrowdo test
if timeout 900 sparrowdo \
--host="${VM_IP}" \
--ssh_user=rocky \
--ssh_private_key="${SSH_PRIVATE_KEY}" \
--ssh_args="-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null" \
--no_sudo \
--sparrowfile="${SPARROWFILE}" \
--verbose \
--color 2>&1 | tee logs/test.log; then
echo_info "Test completed successfully!"
else
echo_error "Test failed or timed out"
echo_info "Logs saved to: ${TEST_WORKSPACE}/logs/test.log"
fi
echo ""
echo_info "Test logs location: ${TEST_WORKSPACE}/logs/test.log"
echo ""
read -p "Press Enter to continue to Step 8 (Cleanup)..."
# =============================================================================
# Step 8: Cleanup
# =============================================================================
echo_step "Step 8: Cleanup"
echo_info "Cleaning up VM: ${VM_NAME}"
"${SCRIPT_DIR}/scripts/cleanup_vm.sh" "${VM_NAME}"
echo ""
echo_warn "Do you want to remove temporary images? (y/n)"
echo_info "Base image: ${BASE_IMAGE}"
echo_info "Golden image: ${GOLDEN_IMAGE}"
read -p "Remove images? (y/n): " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
echo_info "Removing temporary images..."
sudo rm -f "${BASE_IMAGE}"
sudo rm -f "${GOLDEN_IMAGE}"
echo_info "Images removed"
else
echo_info "Images preserved for inspection"
echo_info "Base image: ${BASE_IMAGE}"
echo_info "Golden image: ${GOLDEN_IMAGE}"
fi
# =============================================================================
# Summary
# =============================================================================
echo ""
echo_step "Summary"
echo_info "Build ID: ${BUILD_ID}"
echo_info "Test Name: ${TEST_NAME}"
echo_info "Working Directory: ${WORK_DIR}"
echo_info "Test Logs: ${TEST_WORKSPACE}/logs/test.log"
if [ -f "${BASE_IMAGE}" ]; then
echo_info "Base Image: ${BASE_IMAGE} (preserved)"
fi
if [ -f "${GOLDEN_IMAGE}" ]; then
echo_info "Golden Image: ${GOLDEN_IMAGE} (preserved)"
fi
echo ""
echo_info "Manual test run complete!"

275
docs/virt-customize-guide.md Normal file
View File

@@ -0,0 +1,275 @@
# virt-customize Guide for Golden Image Preparation
## Overview
The `virt-customize` tool modifies QCOW2 images **offline** (without booting them). This means certain commands that require a running system won't work.
## What Works in virt-customize
### ✅ Package Management
```bash
# Install packages
dnf install -y perl git wget
# Update packages (works but can be slow)
dnf update -y
# Remove packages
dnf remove -y packagename
```
### ✅ File Operations
```bash
# Create/modify files
echo "content" > /etc/myfile
cat > /etc/config << EOF
config_line=value
EOF
# Copy files
cp /source /destination
# Set permissions
chmod 755 /path/to/file
chown user:group /path/to/file
```
### ✅ User Management
```bash
# Create users
useradd -m username
# Set passwords
echo "user:password" | chpasswd
# Modify sudoers
echo "user ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/user
chmod 0440 /etc/sudoers.d/user
```
### ✅ Enable Services (Offline Mode)
```bash
# systemctl enable creates symlinks - works offline!
systemctl enable sshd
systemctl enable chronyd
systemctl enable httpd
```
## What DOESN'T Work
### ❌ D-Bus Dependent Commands
These commands require D-Bus and will fail with:
```
DBUS_ERROR: Failed to connect to socket /run/dbus/system_bus_socket
```
**Examples:**
```bash
# ❌ Don't use these in prep scripts:
systemctl start sshd # Requires running system
systemctl restart sshd # Requires running system
systemctl status sshd # Requires running system
firewall-cmd --add-service=ssh # Requires firewalld running
hostnamectl set-hostname test # Requires D-Bus
timedatectl set-timezone UTC # Requires D-Bus
localectl set-locale LANG=en_US # Requires D-Bus
```
## Workarounds for Common Tasks
### Setting Hostname
**Instead of:** `hostnamectl set-hostname test-node`
**Use virt-customize command line:**
```bash
sudo virt-customize -a image.qcow2 --hostname test-node
```
**Or in script:**
```bash
echo "test-node" > /etc/hostname
```
### Firewall Configuration
**Instead of:** `firewall-cmd --add-service=ssh`
**Use virt-customize command line:**
```bash
sudo virt-customize -a image.qcow2 \
--run-command 'firewall-offline-cmd --add-service=ssh'
```
**Or modify firewalld config directly:**
```bash
# In prep script - add SSH to default zone
cat > /etc/firewalld/zones/public.xml << 'EOF'
<?xml version="1.0" encoding="utf-8"?>
<zone>
<short>Public</short>
<service name="ssh"/>
<service name="dhcpv6-client"/>
</zone>
EOF
```
### Timezone Configuration
**Instead of:** `timedatectl set-timezone America/New_York`
**Use:**
```bash
ln -sf /usr/share/zoneinfo/America/New_York /etc/localtime
```
### SELinux Relabeling
**Always relabel after file modifications:**
```bash
# In virt-customize command (not in script):
sudo virt-customize -a image.qcow2 \
--run custom-prep.sh \
--selinux-relabel
```
## Example: Good Prep Script
```bash
#!/bin/bash
set -e
echo "Preparing golden image..."
# ✅ Install packages
dnf install -y \
perl \
git \
openssh-server
# ✅ Enable services (creates symlinks only)
systemctl enable sshd
# ✅ Create users
useradd -m testuser
echo "testuser:testpass" | chpasswd
# ✅ Configure sudoers
echo "testuser ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/testuser
chmod 0440 /etc/sudoers.d/testuser
# ✅ Set timezone (manual symlink)
ln -sf /usr/share/zoneinfo/UTC /etc/localtime
# ✅ Configure network (direct file edit)
cat > /etc/sysconfig/network-scripts/ifcfg-eth0 << 'EOF'
DEVICE=eth0
BOOTPROTO=dhcp
ONBOOT=yes
EOF
echo "Preparation complete!"
```
## Example: Bad Prep Script (Will Fail)
```bash
#!/bin/bash
set -e
# ❌ These will all fail:
systemctl start sshd # No running systemd
systemctl restart network # No running systemd
firewall-cmd --add-service=ssh # No D-Bus
hostnamectl set-hostname test-node # No D-Bus
timedatectl set-timezone America/New_York # No D-Bus
```
## Using setup_base.sh Script
The `setup_base.sh` script uses both:
1. Your custom prep script (runs inside image)
2. virt-customize command-line options (runs outside)
**Example call:**
```bash
./scripts/setup_base.sh \
/path/to/base.qcow2 \
/path/to/prep-script.sh \
/path/to/golden.qcow2 \
~/.ssh/id_rsa.pub
```
This automatically:
- Runs your prep script inside the image
- Injects SSH key (via `--ssh-inject`)
- Sets root password (via `--root-password`)
- Relabels SELinux (via `--selinux-relabel`)
## Advanced: Direct virt-customize Commands
For complex setups, skip the prep script and use virt-customize directly:
```bash
sudo virt-customize -a golden.qcow2 \
--install 'perl,git,wget,openssh-server' \
--run-command 'systemctl enable sshd' \
--run-command 'useradd -m testuser' \
--run-command 'echo "testuser:testpass" | chpasswd' \
--write '/etc/sudoers.d/testuser:testuser ALL=(ALL) NOPASSWD:ALL' \
--chmod '0440:/etc/sudoers.d/testuser' \
--ssh-inject root:file:~/.ssh/id_rsa.pub \
--root-password password:rockytesting \
--hostname test-node \
--timezone UTC \
--selinux-relabel
```
## Debugging virt-customize Issues
If your prep script fails, run with debugging:
```bash
export LIBGUESTFS_DEBUG=1
export LIBGUESTFS_TRACE=1
sudo virt-customize -v -x \
-a golden.qcow2 \
--run prep-script.sh \
--selinux-relabel
```
This will show:
- Exact commands being run
- Output from each command
- Error messages with full context
## Best Practices
1. **Keep prep scripts simple** - Install packages and create users, that's it
2. **Use virt-customize options** - For hostname, timezone, SSH keys
3. **Test incrementally** - Add one command at a time to find issues
4. **Avoid system state** - Don't start services or query running processes
5. **Always relabel SELinux** - After modifying files
6. **Comment out dnf update** - It's slow; only use when needed
## Common Errors and Solutions
### Error: "DBUS_ERROR: Failed to connect to socket"
**Problem:** Script uses D-Bus dependent command
**Solution:** Remove firewall-cmd, hostnamectl, timedatectl commands
### Error: "systemctl: command not found"
**Problem:** Minimal image doesn't have systemd
**Solution:** Check base image has systemd, or install it
### Error: "SELinux is preventing..."
**Problem:** Files created without proper SELinux context
**Solution:** Add `--selinux-relabel` to virt-customize command
### Error: "dnf: command not found"
**Problem:** Image uses yum not dnf (older RHEL/CentOS)
**Solution:** Use `yum` instead of `dnf` in scripts
## Resources
- [virt-customize man page](http://libguestfs.org/virt-customize.1.html)
- [libguestfs FAQ](http://libguestfs.org/guestfs-faq.1.html)
- Rocky Linux Cloud Image docs

107
scripts/bootstrap_golden.sh Executable file
View File

@@ -0,0 +1,107 @@
#!/bin/bash
# Bootstrap the golden image by booting it, running sparrowdo --bootstrap, then shutting down
# This should be run ONCE after creating the golden image
GOLDEN_IMAGE="$1"
SSH_PRIVATE_KEY="${2:-$HOME/.ssh/id_rsa}"
if [ -z "$GOLDEN_IMAGE" ]; then
echo "Usage: $0 <golden_image_path> [ssh_private_key]"
exit 1
fi
if [ ! -f "$GOLDEN_IMAGE" ]; then
echo "ERROR: Golden image not found: $GOLDEN_IMAGE"
exit 1
fi
BOOTSTRAP_VM="bootstrap-golden-$(date +%s)"
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
echo "=========================================="
echo "Bootstrapping Golden Image"
echo "=========================================="
echo "Golden Image: $GOLDEN_IMAGE"
echo "Bootstrap VM: $BOOTSTRAP_VM"
echo "SSH Key: $SSH_PRIVATE_KEY"
echo ""
# Provision temporary VM from golden image
echo "[1/4] Provisioning temporary VM..."
VM_IP=$("${SCRIPT_DIR}/provision_vm.sh" "$BOOTSTRAP_VM" "$GOLDEN_IMAGE" 60)
if [ "$VM_IP" = "ERROR" ] || [ -z "$VM_IP" ]; then
echo "ERROR: Failed to provision bootstrap VM"
exit 1
fi
VM_IP=$(echo "$VM_IP" | tail -1)
echo "VM IP: $VM_IP"
# Wait for SSH
echo ""
echo "[2/4] Waiting for SSH to be ready..."
for i in {1..30}; do
if ssh -i "$SSH_PRIVATE_KEY" \
-o StrictHostKeyChecking=no \
-o ConnectTimeout=5 \
-o UserKnownHostsFile=/dev/null \
rocky@${VM_IP} 'echo "SSH ready"' 2>/dev/null; then
echo "SSH connection established"
break
fi
sleep 2
done
# Run sparrowdo bootstrap
echo ""
echo "[3/4] Running Sparrowdo bootstrap..."
if sparrowdo \
--host="${VM_IP}" \
--ssh_user=rocky \
--ssh_private_key="${SSH_PRIVATE_KEY}" \
--ssh_args="-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null" \
--bootstrap \
--color; then
echo "Bootstrap completed successfully!"
else
echo "WARNING: Bootstrap may have failed"
fi
# Shutdown the VM cleanly to save changes
echo ""
echo "[4/4] Shutting down VM to save changes..."
ssh -i "$SSH_PRIVATE_KEY" \
-o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null \
rocky@${VM_IP} 'sudo shutdown -h now' 2>/dev/null || true
# Wait for VM to shut down
echo "Waiting for VM to shut down..."
sleep 10
for i in {1..30}; do
if ! sudo virsh -c qemu:///system list --name 2>/dev/null | grep -q "$BOOTSTRAP_VM"; then
echo "VM has shut down"
break
fi
sleep 2
done
# Clean up VM definition (disk contains the bootstrapped state)
echo "Cleaning up VM definition..."
sudo virsh -c qemu:///system undefine "$BOOTSTRAP_VM" 2>/dev/null || true
echo ""
echo "=========================================="
echo "Bootstrap Complete"
echo "=========================================="
echo "Golden image has been bootstrapped with Sparrowdo"
echo "The image now contains:"
echo " - Raku/Rakudo runtime"
echo " - Sparrowdo and dependencies"
echo " - All required testing tools"
echo ""
echo "This image can now be used for testing without"
echo "running bootstrap on each test VM."
echo "=========================================="

51
scripts/build-golden.sh Executable file
View File

@@ -0,0 +1,51 @@
#!/bin/bash
BASE_IMAGE="$1"
GOLDEN_IMAGE="$2"
SSH_PUB_KEY="${3:-$HOME/.ssh/id_rsa.pub}"
if [ -z "$BASE_IMAGE" ] || [ -z "$GOLDEN_IMAGE" ]; then
echo "Usage: $0 <base_image> <golden_image> [ssh_pub_key]"
exit 1
fi
echo "Creating golden image: $GOLDEN_IMAGE"
cp "$BASE_IMAGE" "$GOLDEN_IMAGE"
export LIBGUESTFS_BACKEND=direct
sudo virt-customize -a "$GOLDEN_IMAGE" \
--install rakudo,rakudo-zef,perl,git,openssh-server \
--run-command 'useradd -m rocky && echo "rocky:rockypass" | chpasswd' \
--run-command 'echo "rocky ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/rocky && chmod 0440 /etc/sudoers.d/rocky' \
--ssh-inject root:file:"$SSH_PUB_KEY" \
--ssh-inject rocky:file:"$SSH_PUB_KEY" \
--root-password password:rockytesting \
--run-command 'systemctl enable sshd' \
--selinux-relabel
echo "Bootstrapping Sparrowdo..."
BOOTSTRAP_VM="bootstrap-$$"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
VM_IP=$("$SCRIPT_DIR/provision_vm.sh" "$BOOTSTRAP_VM" "$GOLDEN_IMAGE" 60 | tail -1)
if [ -z "$VM_IP" ] || [ "$VM_IP" = "ERROR" ]; then
echo "ERROR: Failed to provision bootstrap VM"
exit 1
fi
sleep 5
SSH_KEY="${SSH_PUB_KEY%.pub}"
sparrowdo --host="$VM_IP" --ssh_user=rocky --ssh_private_key="$SSH_KEY" \
--ssh_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
--bootstrap --color
ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
rocky@"$VM_IP" 'sudo shutdown -h now' 2>/dev/null || true
sleep 10
"$SCRIPT_DIR/cleanup_vm.sh" "$BOOTSTRAP_VM"
echo "Golden image ready: $GOLDEN_IMAGE"

14
scripts/cleanup-all.sh Executable file
View File

@@ -0,0 +1,14 @@
#!/bin/bash
PATTERN="${1:-.*}"
echo "Cleaning up VMs matching: $PATTERN"
for vm in $(virsh -c qemu:///system list --all --name | grep -E "$PATTERN"); do
echo "Destroying $vm"
virsh -c qemu:///system destroy "$vm" 2>/dev/null || true
virsh -c qemu:///system undefine "$vm" 2>/dev/null || true
rm -f "/var/lib/libvirt/images/${vm}.qcow2" 2>/dev/null || true
done
echo "Cleanup complete"

4
scripts/cleanup_vm.sh
View File

@@ -12,11 +12,11 @@ echo "[Cleanup] Starting cleanup for VM: $VM_NAME"
# Force stop VM # Force stop VM
echo "[Cleanup] Destroying VM..." echo "[Cleanup] Destroying VM..."
sudo virsh destroy "$VM_NAME" 2>/dev/null || echo "[Cleanup] VM was not running" sudo virsh -c qemu:///system destroy "$VM_NAME" 2>/dev/null || echo "[Cleanup] VM was not running"
# Remove VM definition # Remove VM definition
echo "[Cleanup] Undefining VM..." echo "[Cleanup] Undefining VM..."
sudo virsh undefine "$VM_NAME" 2>/dev/null || echo "[Cleanup] VM definition already removed" sudo virsh -c qemu:///system undefine "$VM_NAME" 2>/dev/null || echo "[Cleanup] VM definition already removed"
# Remove disk image # Remove disk image
echo "[Cleanup] Removing disk image..." echo "[Cleanup] Removing disk image..."

22
scripts/download-image.sh Executable file
View File

@@ -0,0 +1,22 @@
#!/bin/bash
URL="$1"
OUTPUT_DIR="${2:-/var/lib/libvirt/images}"
FORCE="${3:-false}"
if [ -z "$URL" ]; then
echo "Usage: $0 <url> [output_dir] [force]"
exit 1
fi
FILENAME=$(basename "$URL")
CACHED="$OUTPUT_DIR/$FILENAME"
if [ "$FORCE" = "true" ] || [ ! -f "$CACHED" ]; then
echo "Downloading $URL to $CACHED"
curl -L --progress-bar -o "$CACHED" "$URL"
else
echo "Using cached image: $CACHED"
fi
echo "$CACHED"

18
scripts/provision_vm.sh
View File

@@ -1,5 +1,6 @@
#!/bin/bash #!/bin/bash
set -e # Note: Not using 'set -e' here because we handle errors explicitly
# and arithmetic operations can cause false failures with set -e
VM_NAME="$1" VM_NAME="$1"
GOLDEN_IMAGE="$2" GOLDEN_IMAGE="$2"
@@ -20,7 +21,7 @@ sudo qemu-img create -f qcow2 -b "$GOLDEN_IMAGE" -F qcow2 "$VM_DISK" 2>/dev/null
# Define and start VM # Define and start VM
echo "[Provision] Starting VM with virt-install..." echo "[Provision] Starting VM with virt-install..."
sudo virt-install \ VIRT_INSTALL_OUTPUT=$(sudo virt-install \
--name "$VM_NAME" \ --name "$VM_NAME" \
--memory 2048 \ --memory 2048 \
--vcpus 2 \ --vcpus 2 \
@@ -31,14 +32,21 @@ sudo virt-install \
--noautoconsole \ --noautoconsole \
--wait 0 \ --wait 0 \
--transient \ --transient \
2>&1 | grep -v "WARNING" || true 2>&1) || {
echo "[Provision] ERROR: virt-install failed"
echo "$VIRT_INSTALL_OUTPUT" | grep -v "WARNING"
echo "[Provision] Cleaning up disk..."
sudo rm -f "$VM_DISK"
echo "ERROR"
exit 1
}
# Wait for IP address # Wait for IP address
echo "[Provision] Waiting for VM to obtain IP address (max ${MAX_WAIT}s)..." echo "[Provision] Waiting for VM to obtain IP address (max ${MAX_WAIT}s)..."
COUNTER=0 COUNTER=0
while [ $COUNTER -lt $MAX_WAIT ]; do while [ $COUNTER -lt $MAX_WAIT ]; do
# Try to get IP from DHCP lease # Try to get IP from DHCP lease (explicitly use system connection)
IP=$(sudo virsh domifaddr "$VM_NAME" --source lease 2>/dev/null | awk '/ipv4/ {print $4}' | cut -d/ -f1 | head -1) IP=$(sudo virsh -c qemu:///system domifaddr "$VM_NAME" --source lease 2>/dev/null | awk '/ipv4/ {print $4}' | cut -d/ -f1 | head -1)
if [ -n "$IP" ] && [ "$IP" != "0.0.0.0" ]; then if [ -n "$IP" ] && [ "$IP" != "0.0.0.0" ]; then
echo "[Provision] IP obtained: $IP" echo "[Provision] IP obtained: $IP"

60
scripts/run-test.sh Executable file
View File

@@ -0,0 +1,60 @@
#!/bin/bash
TEST_NAME="$1"
TEST_REPO="$2"
GOLDEN_IMAGE="$3"
SSH_KEY="${4:-$HOME/.ssh/id_rsa}"
if [ -z "$TEST_NAME" ] || [ -z "$TEST_REPO" ] || [ -z "$GOLDEN_IMAGE" ]; then
echo "Usage: $0 <test_name> <test_repo_url> <golden_image> [ssh_key]"
exit 1
fi
VM_NAME="$TEST_NAME-$$"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
WORK_DIR="/tmp/test-$VM_NAME"
mkdir -p "$WORK_DIR"
cd "$WORK_DIR"
cleanup() {
echo "Cleaning up..."
"$SCRIPT_DIR/cleanup_vm.sh" "$VM_NAME" 2>/dev/null || true
cd /tmp
rm -rf "$WORK_DIR"
}
trap cleanup EXIT
echo "Provisioning VM..."
VM_IP=$("$SCRIPT_DIR/provision_vm.sh" "$VM_NAME" "$GOLDEN_IMAGE" 60 | tail -1)
if [ -z "$VM_IP" ] || [ "$VM_IP" = "ERROR" ]; then
echo "ERROR: Failed to provision VM"
exit 1
fi
echo "VM ready at $VM_IP"
echo "Cloning test repository..."
git clone "$TEST_REPO" test-repo
SPARROWFILE=$(find test-repo -name "main.raku" -o -name "sparrowfile" | head -1)
if [ -z "$SPARROWFILE" ]; then
echo "ERROR: No sparrowfile found"
exit 1
fi
echo "Running test: $SPARROWFILE"
sparrowdo \
--host="$VM_IP" \
--ssh_user=rocky \
--ssh_private_key="$SSH_KEY" \
--ssh_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
--no_sudo \
--sparrowfile="$SPARROWFILE" \
--verbose \
--color
echo "Test completed successfully"

1
scripts/setup_base.sh
View File

@@ -37,6 +37,7 @@ if [ -f "$PREP_SCRIPT_PATH" ]; then
sudo virt-customize -a "$GOLDEN_IMAGE" \ sudo virt-customize -a "$GOLDEN_IMAGE" \
--run "$PREP_SCRIPT_PATH" \ --run "$PREP_SCRIPT_PATH" \
--ssh-inject root:file:"$SSH_PUB_KEY" \ --ssh-inject root:file:"$SSH_PUB_KEY" \
--ssh-inject rocky:file:"$SSH_PUB_KEY" \
--root-password password:rockytesting \ --root-password password:rockytesting \
--selinux-relabel 2>&1 || { --selinux-relabel 2>&1 || {
echo "ERROR: virt-customize failed" echo "ERROR: virt-customize failed"