Initial commit: Sparrowdo Testing Orchestrator framework
- Add VM provisioning, cleanup, and setup scripts - Add Jenkins pipeline with parameterized builds - Add comprehensive documentation - Support for parallel test execution with QCOW2 linked clones
This commit is contained in:
Stephen Simpson
79
.gitignore
vendored
Normal file
79
.gitignore
vendored
Normal file
@@ -0,0 +1,79 @@
|
||||
# QCOW2 Images
|
||||
*.qcow2
|
||||
*.qcow2.bak
|
||||
|
||||
# Log files
|
||||
*.log
|
||||
*.txt
|
||||
logs/
|
||||
|
||||
# Temporary prep scripts
|
||||
prep-*.sh
|
||||
|
||||
# Workspace directories
|
||||
ws-*/
|
||||
|
||||
# OS generated files
|
||||
.DS_Store
|
||||
.DS_Store?
|
||||
._*
|
||||
.Spotlight-V100
|
||||
.Trashes
|
||||
ehthumbs.db
|
||||
Thumbs.db
|
||||
|
||||
# Editor files
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.vscode/
|
||||
.idea/
|
||||
*.sublime-project
|
||||
*.sublime-workspace
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# Node.js
|
||||
node_modules/
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# Raku/Perl6
|
||||
.precomp/
|
||||
|
||||
# Temporary files
|
||||
tmp/
|
||||
temp/
|
||||
*.tmp
|
||||
*.bak
|
||||
*.cache
|
||||
|
||||
# Jenkins workspace artifacts
|
||||
jenkins-workspace/
|
||||
|
||||
# SSH keys (safety)
|
||||
id_rsa
|
||||
id_rsa.pub
|
||||
*.pem
|
||||
*.key
|
||||
374
Jenkinsfile
vendored
Normal file
374
Jenkinsfile
vendored
Normal file
@@ -0,0 +1,374 @@
|
||||
pipeline {
|
||||
agent {
|
||||
label 'fedora-testing'
|
||||
customWorkspace "/home/jenkins/jenkins-workspace/${BUILD_NUMBER}"
|
||||
}
|
||||
|
||||
parameters {
|
||||
// QCOW2 Image Source
|
||||
string(
|
||||
name: 'QCOW2_URL',
|
||||
defaultValue: 'https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2',
|
||||
description: 'URL to the base QCOW2 image to use for testing'
|
||||
)
|
||||
|
||||
// Test Matrix - JSON array format
|
||||
text(
|
||||
name: 'TEST_MATRIX',
|
||||
defaultValue: '''[
|
||||
{
|
||||
"name": "example-test",
|
||||
"url": "https://github.com/your-org/sparrowdo-tests.git",
|
||||
"branch": "main",
|
||||
"description": "Example Sparrowdo test suite"
|
||||
}
|
||||
]''',
|
||||
description: 'JSON array of test definitions (name, url, branch, description)'
|
||||
)
|
||||
|
||||
// Custom Golden Image Preparation
|
||||
text(
|
||||
name: 'GOLDEN_PREP_SCRIPT',
|
||||
defaultValue: '''#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "Preparing standard golden image..."
|
||||
|
||||
# Update system
|
||||
dnf update -y
|
||||
|
||||
# Install common testing dependencies
|
||||
dnf install -y \\
|
||||
perl \\
|
||||
git \\
|
||||
wget \\
|
||||
tar \\
|
||||
openssh-server \\
|
||||
firewalld \\
|
||||
chrony \\
|
||||
vim
|
||||
|
||||
# Configure services
|
||||
systemctl enable sshd
|
||||
systemctl enable firewalld
|
||||
systemctl enable chronyd
|
||||
|
||||
# Configure firewall
|
||||
firewall-cmd --permanent --add-service=ssh
|
||||
firewall-cmd --reload
|
||||
|
||||
# Set consistent hostname
|
||||
hostnamectl set-hostname test-node
|
||||
|
||||
# Create testing user
|
||||
useradd -m testuser 2>/dev/null || true
|
||||
echo "testuser:testpass" | chpasswd
|
||||
echo "testuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
|
||||
|
||||
echo "Golden image preparation complete!"
|
||||
''',
|
||||
description: 'Shell script to run inside the golden image'
|
||||
)
|
||||
|
||||
// Test Selection
|
||||
string(
|
||||
name: 'TEST_FILTER',
|
||||
defaultValue: '.*',
|
||||
description: 'Regex pattern to filter tests (e.g., "login.*" or ".*")'
|
||||
)
|
||||
|
||||
// Concurrency Control
|
||||
choice(
|
||||
name: 'MAX_CONCURRENT',
|
||||
choices: ['1', '2', '3', '5', '10'],
|
||||
description: 'Maximum concurrent VMs'
|
||||
)
|
||||
|
||||
// Cleanup Control
|
||||
booleanParam(
|
||||
name: 'KEEP_GOLDEN_IMAGE',
|
||||
defaultValue: false,
|
||||
description: 'Keep golden image after testing'
|
||||
)
|
||||
|
||||
// SSH Configuration
|
||||
string(
|
||||
name: 'SSH_PRIVATE_KEY_PATH',
|
||||
defaultValue: '${HOME}/.ssh/id_rsa',
|
||||
description: 'Path to SSH private key for VM access'
|
||||
)
|
||||
|
||||
string(
|
||||
name: 'SSH_PUBLIC_KEY_PATH',
|
||||
defaultValue: '${HOME}/.ssh/id_rsa.pub',
|
||||
description: 'Path to SSH public key to inject into VMs'
|
||||
)
|
||||
}
|
||||
|
||||
environment {
|
||||
BUILD_ID = "${BUILD_NUMBER}"
|
||||
IMAGES_DIR = "/var/lib/libvirt/images"
|
||||
MAX_PARALLEL = "${params.MAX_CONCURRENT}"
|
||||
}
|
||||
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
timeout(time: 2, unit: 'HOURS')
|
||||
buildDiscarder(logRotator(numToKeepStr: '10'))
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Initialize') {
|
||||
steps {
|
||||
script {
|
||||
echo "=========================================="
|
||||
echo "Build ${BUILD_ID}"
|
||||
echo "Node: ${env.NODE_NAME}"
|
||||
echo "Max Concurrent: ${params.MAX_CONCURRENT}"
|
||||
echo "Test Filter: ${params.TEST_FILTER}"
|
||||
echo "=========================================="
|
||||
|
||||
sh '''
|
||||
sudo mkdir -p ${IMAGES_DIR}
|
||||
sudo chown ${USER}:${USER} ${IMAGES_DIR}
|
||||
chmod +x scripts/*.sh
|
||||
echo "Environment initialized"
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Download Base Image') {
|
||||
steps {
|
||||
script {
|
||||
dir(IMAGES_DIR) {
|
||||
sh '''
|
||||
if [ ! -f "base-${BUILD_ID}.qcow2" ]; then
|
||||
echo "Downloading QCOW2 image from: ${QCOW2_URL}"
|
||||
curl -L --progress-bar -o "base-${BUILD_ID}.qcow2" "${QCOW2_URL}"
|
||||
echo ""
|
||||
echo "Image downloaded successfully:"
|
||||
qemu-img info "base-${BUILD_ID}.qcow2" | head -5
|
||||
else
|
||||
echo "Using cached base image"
|
||||
fi
|
||||
'''
|
||||
env.BASE_IMAGE_PATH = "${IMAGES_DIR}/base-${BUILD_ID}.qcow2"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Prepare Golden Image') {
|
||||
steps {
|
||||
script {
|
||||
def goldenImage = "${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"
|
||||
def prepScript = "prep-${BUILD_ID}.sh"
|
||||
|
||||
// Expand SSH key paths
|
||||
def sshPubKey = params.SSH_PUBLIC_KEY_PATH.replace('${HOME}', env.HOME)
|
||||
|
||||
// Write prep script
|
||||
writeFile file: prepScript, text: params.GOLDEN_PREP_SCRIPT
|
||||
sh "chmod +x ${prepScript}"
|
||||
|
||||
env.PREP_SCRIPT_PATH = pwd() + "/" + prepScript
|
||||
env.GOLDEN_IMAGE_PATH = goldenImage
|
||||
env.SSH_PUB_KEY_PATH = sshPubKey
|
||||
|
||||
echo "Creating golden image: ${goldenImage}"
|
||||
echo "Using SSH public key: ${sshPubKey}"
|
||||
|
||||
sh '''
|
||||
./scripts/setup_base.sh \\
|
||||
${BASE_IMAGE_PATH} \\
|
||||
${PREP_SCRIPT_PATH} \\
|
||||
${GOLDEN_IMAGE_PATH} \\
|
||||
${SSH_PUB_KEY_PATH}
|
||||
'''
|
||||
|
||||
sh "rm -f ${prepScript}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Parse Test Matrix') {
|
||||
steps {
|
||||
script {
|
||||
def testMatrix = readJSON text: params.TEST_MATRIX.trim()
|
||||
def filtered = testMatrix.findAll { it.name =~ params.TEST_FILTER }
|
||||
|
||||
if (filtered.isEmpty()) {
|
||||
error "No tests match filter: ${params.TEST_FILTER}"
|
||||
}
|
||||
|
||||
echo "=========================================="
|
||||
echo "Tests to run (${filtered.size()}):"
|
||||
filtered.each { echo " - ${it.name}: ${it.description}" }
|
||||
echo "=========================================="
|
||||
|
||||
env.FILTERED_TESTS = groovy.json.JsonOutput.toJson(filtered)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Run Tests') {
|
||||
steps {
|
||||
script {
|
||||
def tests = readJSON text: env.FILTERED_TESTS
|
||||
def tasks = [:]
|
||||
|
||||
// Expand SSH key path
|
||||
def sshPrivateKey = params.SSH_PRIVATE_KEY_PATH.replace('${HOME}', env.HOME)
|
||||
|
||||
tests.each { test ->
|
||||
def testName = test.name
|
||||
def testUrl = test.url
|
||||
def testBranch = test.branch ?: 'main'
|
||||
|
||||
tasks["${testName}"] = {
|
||||
lock(resource: 'kvm-slots', quantity: 1) {
|
||||
stage("Test: ${testName}") {
|
||||
def vmId = "${testName}-${BUILD_ID}"
|
||||
def targetIp = ""
|
||||
|
||||
try {
|
||||
dir("ws-${testName}") {
|
||||
echo "[${testName}] Provisioning VM..."
|
||||
def ipOutput = sh(
|
||||
script: "${WORKSPACE}/scripts/provision_vm.sh ${vmId} ${GOLDEN_IMAGE_PATH} 60",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
if (ipOutput.contains("ERROR")) {
|
||||
error "Failed to provision VM"
|
||||
}
|
||||
|
||||
// Extract IP from output (last line)
|
||||
targetIp = ipOutput.split('\n').last()
|
||||
echo "[${testName}] VM ready at: ${targetIp}"
|
||||
|
||||
// Wait for SSH to be ready
|
||||
echo "[${testName}] Waiting for SSH..."
|
||||
sh """
|
||||
for i in {1..30}; do
|
||||
if ssh -i ${sshPrivateKey} -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@${targetIp} 'echo SSH ready' 2>/dev/null; then
|
||||
echo "SSH connection established"
|
||||
break
|
||||
fi
|
||||
echo "Attempt \$i/30..."
|
||||
sleep 2
|
||||
done
|
||||
"""
|
||||
|
||||
// Clone test repository
|
||||
echo "[${testName}] Cloning test repository..."
|
||||
sh "git clone -b ${testBranch} ${testUrl} test-repo || true"
|
||||
|
||||
// Verify sparrowfile exists
|
||||
def sparrowfilePath = sh(
|
||||
script: 'find test-repo -name sparrowfile -type f | head -1',
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
if (!sparrowfilePath) {
|
||||
error "No sparrowfile found in test repository"
|
||||
}
|
||||
|
||||
echo "[${testName}] Found sparrowfile: ${sparrowfilePath}"
|
||||
|
||||
// Run test
|
||||
echo "[${testName}] Running Sparrowdo..."
|
||||
sh """
|
||||
mkdir -p logs
|
||||
timeout 900 sparrowdo \\
|
||||
--host=${targetIp} \\
|
||||
--ssh_user=root \\
|
||||
--ssh_private_key=${sshPrivateKey} \\
|
||||
--ssh_args='-o StrictHostKeyChecking=no -o ConnectTimeout=10' \\
|
||||
--no_sudo \\
|
||||
--sparrowfile=${sparrowfilePath} 2>&1 | tee logs/test.log
|
||||
"""
|
||||
|
||||
echo "[${testName}] Test completed successfully"
|
||||
archiveArtifacts artifacts: "logs/**", allowEmptyArchive: true
|
||||
}
|
||||
|
||||
} catch (e) {
|
||||
echo "[${testName}] FAILED: ${e.message}"
|
||||
|
||||
// Archive any logs that were generated
|
||||
dir("ws-${testName}") {
|
||||
archiveArtifacts artifacts: "logs/**", allowEmptyArchive: true
|
||||
}
|
||||
|
||||
currentBuild.result = 'UNSTABLE'
|
||||
|
||||
} finally {
|
||||
echo "[${testName}] Cleaning up VM..."
|
||||
sh "${WORKSPACE}/scripts/cleanup_vm.sh ${vmId} || true"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run all tests in parallel (respecting lock quantity)
|
||||
parallel tasks
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
always {
|
||||
script {
|
||||
echo "=========================================="
|
||||
echo "Post-build cleanup"
|
||||
echo "=========================================="
|
||||
|
||||
if (!params.KEEP_GOLDEN_IMAGE) {
|
||||
sh '''
|
||||
echo "Removing temporary images..."
|
||||
sudo rm -f ${IMAGES_DIR}/base-${BUILD_ID}.qcow2 || true
|
||||
sudo rm -f ${IMAGES_DIR}/golden-${BUILD_ID}.qcow2 || true
|
||||
echo "Temporary images cleaned up"
|
||||
'''
|
||||
} else {
|
||||
echo "Golden image preserved: ${GOLDEN_IMAGE_PATH}"
|
||||
echo "Base image preserved: ${BASE_IMAGE_PATH}"
|
||||
}
|
||||
|
||||
// Clean up any orphaned VMs
|
||||
sh '''
|
||||
echo "Checking for orphaned VMs from build ${BUILD_ID}..."
|
||||
for vm in $(virsh list --all --name 2>/dev/null | grep -E "-${BUILD_ID}"); do
|
||||
echo "Cleaning orphaned VM: $vm"
|
||||
sudo virsh destroy "$vm" 2>/dev/null || true
|
||||
sudo virsh undefine "$vm" 2>/dev/null || true
|
||||
sudo rm -f "/var/lib/libvirt/images/${vm}.qcow2" 2>/dev/null || true
|
||||
done
|
||||
echo "Orphan cleanup complete"
|
||||
'''
|
||||
}
|
||||
}
|
||||
|
||||
success {
|
||||
echo "=========================================="
|
||||
echo "Build ${BUILD_ID} completed successfully!"
|
||||
echo "=========================================="
|
||||
}
|
||||
|
||||
failure {
|
||||
echo "=========================================="
|
||||
echo "Build ${BUILD_ID} failed!"
|
||||
echo "=========================================="
|
||||
}
|
||||
|
||||
unstable {
|
||||
echo "=========================================="
|
||||
echo "Build ${BUILD_ID} completed with test failures"
|
||||
echo "=========================================="
|
||||
}
|
||||
}
|
||||
}
|
||||
389
README.md
Normal file
389
README.md
Normal file
@@ -0,0 +1,389 @@
|
||||
# Sparrowdo Testing Orchestrator
|
||||
|
||||
Automated testing framework for Rocky Linux using Sparrowdo, QCOW2 images, and Jenkins.
|
||||
|
||||
## Overview
|
||||
|
||||
This framework provides a production-ready continuous integration/testing system that:
|
||||
|
||||
- **Automates VM Provisioning**: Spins up isolated Rocky Linux VMs on-demand using QCOW2 images
|
||||
- **Runs Sparrowdo Tests in Parallel**: Executes your test suite across multiple isolated environments simultaneously
|
||||
- **Supports Dynamic Configuration**: Runtime customization of base images, preparation scripts, test selection, and concurrency
|
||||
- **Provides Web Interface**: Jenkins UI for triggering builds, viewing results, and downloading logs
|
||||
- **Scales Across Machines**: Multiple team members can connect their desktops as Jenkins agents
|
||||
|
||||
## How It Works
|
||||
|
||||
```
|
||||
User clicks "Build" in Jenkins
|
||||
↓
|
||||
Download QCOW2 base image
|
||||
↓
|
||||
Run custom prep script (install packages, configure services)
|
||||
↓
|
||||
Create golden image
|
||||
↓
|
||||
Create linked clones for each test (fast - copy-on-write)
|
||||
↓
|
||||
Run Sparrowdo tests in parallel (isolated VMs)
|
||||
↓
|
||||
Collect logs and archive results
|
||||
↓
|
||||
Auto-cleanup VMs and temporary images
|
||||
```
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
.
|
||||
├── Jenkinsfile # Jenkins Pipeline definition
|
||||
├── README.md # This file
|
||||
├── scripts/
|
||||
│ ├── setup_base.sh # Prepares golden image from QCOW2
|
||||
│ ├── provision_vm.sh # Creates and starts test VM
|
||||
│ └── cleanup_vm.sh # Destroys VM and removes disk
|
||||
├── tests/
|
||||
│ └── (sample tests here)
|
||||
└── docs/
|
||||
└── (additional documentation)
|
||||
```
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### On Jenkins Agent (Fedora/Rocky Linux Desktop)
|
||||
|
||||
1. **KVM/QEMU/Libvirt**
|
||||
```bash
|
||||
sudo dnf install -y qemu-kvm libvirt virt-install libguestfs-tools-c
|
||||
sudo systemctl enable --now libvirtd
|
||||
sudo usermod -a -G libvirt $(whoami)
|
||||
```
|
||||
|
||||
2. **Sparrowdo**
|
||||
```bash
|
||||
# Install Raku (Perl 6)
|
||||
sudo dnf install -y rakudo
|
||||
|
||||
# Install zef (Raku module manager)
|
||||
git clone https://github.com/ugexe/zef.git
|
||||
cd zef && raku -I. bin/zef install .
|
||||
|
||||
# Install Sparrowdo
|
||||
zef install Sparrowdo
|
||||
```
|
||||
|
||||
3. **SSH Keys**
|
||||
```bash
|
||||
# Generate SSH key pair if needed
|
||||
ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N ""
|
||||
```
|
||||
|
||||
4. **Jenkins Agent**
|
||||
- Connect your Fedora desktop as a Jenkins agent
|
||||
- Label it as `fedora-testing`
|
||||
- Ensure the agent user has sudo access for virsh/libvirt commands
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Clone Repository
|
||||
|
||||
```bash
|
||||
cd ~/
|
||||
git clone <your-repo-url> testing-orchestrator
|
||||
cd testing-orchestrator
|
||||
```
|
||||
|
||||
### 2. Test Scripts Locally (Optional)
|
||||
|
||||
```bash
|
||||
# Make scripts executable
|
||||
chmod +x scripts/*.sh
|
||||
|
||||
# Download a test QCOW2 image
|
||||
cd /var/lib/libvirt/images
|
||||
curl -LO https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2
|
||||
|
||||
# Create golden image
|
||||
cd ~/testing-orchestrator
|
||||
./scripts/setup_base.sh \
|
||||
/var/lib/libvirt/images/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2 \
|
||||
"" \
|
||||
/var/lib/libvirt/images/golden-test.qcow2 \
|
||||
~/.ssh/id_rsa.pub
|
||||
|
||||
# Provision a test VM
|
||||
./scripts/provision_vm.sh test-vm-1 /var/lib/libvirt/images/golden-test.qcow2
|
||||
|
||||
# Clean up
|
||||
./scripts/cleanup_vm.sh test-vm-1
|
||||
```
|
||||
|
||||
### 3. Configure Jenkins Job
|
||||
|
||||
1. In Jenkins, create a new **Pipeline** job
|
||||
2. Name it "rocky-sparrowdo-tests"
|
||||
3. Under "Pipeline", select "Pipeline script from SCM"
|
||||
4. Set SCM to "Git" and provide your repository URL
|
||||
5. Set Script Path to "Jenkinsfile"
|
||||
6. Save
|
||||
|
||||
### 4. Run Your First Build
|
||||
|
||||
1. Click "Build with Parameters"
|
||||
2. Configure parameters:
|
||||
- **QCOW2_URL**: Rocky Linux base image URL
|
||||
- **TEST_MATRIX**: JSON array of your Sparrowdo test repositories
|
||||
- **GOLDEN_PREP_SCRIPT**: Customize image preparation
|
||||
- **TEST_FILTER**: Regex to filter which tests run
|
||||
- **MAX_CONCURRENT**: Number of parallel VMs
|
||||
3. Click "Build"
|
||||
|
||||
## Configuration
|
||||
|
||||
### TEST_MATRIX Format
|
||||
|
||||
The `TEST_MATRIX` parameter accepts a JSON array of test definitions:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"name": "login-tests",
|
||||
"url": "https://github.com/your-org/login-tests.git",
|
||||
"branch": "main",
|
||||
"description": "SSH login and connectivity tests"
|
||||
},
|
||||
{
|
||||
"name": "database-tests",
|
||||
"url": "https://github.com/your-org/database-tests.git",
|
||||
"branch": "develop",
|
||||
"description": "PostgreSQL configuration tests"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Each test repository should contain a `sparrowfile` at the root or in a subdirectory.
|
||||
|
||||
### Custom Golden Image Preparation
|
||||
|
||||
The `GOLDEN_PREP_SCRIPT` parameter accepts a bash script that runs inside the QCOW2 image during preparation:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Update system
|
||||
dnf update -y
|
||||
|
||||
# Install dependencies
|
||||
dnf install -y perl git wget postgresql-server
|
||||
|
||||
# Configure services
|
||||
systemctl enable postgresql
|
||||
firewall-cmd --permanent --add-service=postgresql
|
||||
|
||||
echo "Custom preparation complete!"
|
||||
```
|
||||
|
||||
### Test Filtering
|
||||
|
||||
Use the `TEST_FILTER` parameter to run specific tests:
|
||||
|
||||
- `.*` - Run all tests
|
||||
- `login.*` - Run all tests starting with "login"
|
||||
- `database-postgres` - Run only the specific test
|
||||
- `(api|integration).*` - Run API or integration tests
|
||||
|
||||
## Features
|
||||
|
||||
### Linked Clones for Speed
|
||||
|
||||
The framework uses QCOW2 linked clones (copy-on-write), which means:
|
||||
- Creating a new VM takes seconds, not minutes
|
||||
- Each test VM only stores differences from the golden image
|
||||
- Disk space usage is minimal
|
||||
|
||||
### Parallel Execution with Isolation
|
||||
|
||||
- Each test runs in its own isolated VM
|
||||
- Tests cannot interfere with each other
|
||||
- Concurrency is controlled via `MAX_CONCURRENT` parameter
|
||||
- Jenkins lock mechanism prevents desktop overload
|
||||
|
||||
### Automatic Cleanup
|
||||
|
||||
- VMs are automatically destroyed after each test
|
||||
- Temporary images are removed after build
|
||||
- Orphaned VMs from failed builds are cleaned up
|
||||
- Optional: Keep golden image for debugging
|
||||
|
||||
### Test Result Archival
|
||||
|
||||
- Each test's output is captured in `logs/test.log`
|
||||
- Logs are archived as Jenkins artifacts
|
||||
- Download logs directly from Jenkins UI
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### VM Won't Start
|
||||
|
||||
```bash
|
||||
# Check libvirt status
|
||||
sudo systemctl status libvirtd
|
||||
|
||||
# Verify image integrity
|
||||
qemu-img info /var/lib/libvirt/images/golden-*.qcow2
|
||||
|
||||
# Check disk space
|
||||
df -h /var/lib/libvirt/images/
|
||||
```
|
||||
|
||||
### VM Doesn't Get IP Address
|
||||
|
||||
```bash
|
||||
# Check default network
|
||||
sudo virsh net-list --all
|
||||
sudo virsh net-start default # If it's not running
|
||||
|
||||
# Verify DHCP
|
||||
sudo virsh net-dhcp-leases default
|
||||
```
|
||||
|
||||
### SSH Connection Fails
|
||||
|
||||
```bash
|
||||
# Test SSH manually (get IP from Jenkins logs)
|
||||
ssh -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@<VM_IP>
|
||||
|
||||
# Check SSH key injection
|
||||
sudo virt-cat -a /var/lib/libvirt/images/golden-*.qcow2 /root/.ssh/authorized_keys
|
||||
```
|
||||
|
||||
### Sparrowdo Test Fails
|
||||
|
||||
```bash
|
||||
# View test logs in Jenkins artifacts
|
||||
# Or run Sparrowdo manually against VM:
|
||||
sparrowdo \
|
||||
--host=<VM_IP> \
|
||||
--ssh_user=root \
|
||||
--ssh_private_key=~/.ssh/id_rsa \
|
||||
--sparrowfile=/path/to/sparrowfile
|
||||
```
|
||||
|
||||
### Permission Denied Errors
|
||||
|
||||
```bash
|
||||
# Ensure user is in libvirt group
|
||||
sudo usermod -a -G libvirt $(whoami)
|
||||
newgrp libvirt
|
||||
|
||||
# Fix image directory permissions
|
||||
sudo chown -R $(whoami):$(whoami) /var/lib/libvirt/images
|
||||
```
|
||||
|
||||
### Jenkins Agent Offline
|
||||
|
||||
```bash
|
||||
# Check agent service (if running as systemd service)
|
||||
sudo systemctl status jenkins-agent
|
||||
|
||||
# View logs
|
||||
sudo journalctl -u jenkins-agent -f
|
||||
|
||||
# Test connection from Jenkins controller
|
||||
ssh jenkins@<agent-host> 'echo "Connection successful"'
|
||||
```
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Testing Beta Images
|
||||
|
||||
Change the `QCOW2_URL` parameter to point to beta images:
|
||||
|
||||
```
|
||||
https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base-9.4-beta.x86_64.qcow2
|
||||
```
|
||||
|
||||
### Custom VM Resources
|
||||
|
||||
Edit `scripts/provision_vm.sh` to adjust memory/CPU:
|
||||
|
||||
```bash
|
||||
sudo virt-install \
|
||||
--name "$VM_NAME" \
|
||||
--memory 4096 \ # Increase memory
|
||||
--vcpus 4 \ # Increase CPUs
|
||||
...
|
||||
```
|
||||
|
||||
### Preserving Golden Images for Debugging
|
||||
|
||||
1. Set `KEEP_GOLDEN_IMAGE` to `true`
|
||||
2. After build, the golden image is preserved
|
||||
3. Boot it manually for inspection:
|
||||
|
||||
```bash
|
||||
VM_NAME="debug-vm"
|
||||
sudo virt-install \
|
||||
--name "$VM_NAME" \
|
||||
--memory 2048 \
|
||||
--vcpus 2 \
|
||||
--disk /var/lib/libvirt/images/golden-<BUILD_ID>.qcow2 \
|
||||
--import \
|
||||
--os-variant rocky9-unknown \
|
||||
--network network=default
|
||||
|
||||
# Connect via console
|
||||
sudo virsh console $VM_NAME
|
||||
```
|
||||
|
||||
### Running Tests Without Jenkins
|
||||
|
||||
```bash
|
||||
# Manual test execution
|
||||
./scripts/setup_base.sh \
|
||||
/path/to/base.qcow2 \
|
||||
/path/to/prep-script.sh \
|
||||
/path/to/golden.qcow2 \
|
||||
~/.ssh/id_rsa.pub
|
||||
|
||||
IP=$(./scripts/provision_vm.sh my-test-vm /path/to/golden.qcow2)
|
||||
|
||||
sparrowdo \
|
||||
--host=$IP \
|
||||
--ssh_user=root \
|
||||
--ssh_private_key=~/.ssh/id_rsa \
|
||||
--sparrowfile=/path/to/test/sparrowfile
|
||||
|
||||
./scripts/cleanup_vm.sh my-test-vm
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
### Adding New Tests
|
||||
|
||||
1. Create a new Sparrowdo test repository
|
||||
2. Add a `sparrowfile` at the root or in a subdirectory
|
||||
3. Add the test to the `TEST_MATRIX` parameter in Jenkins
|
||||
|
||||
### Modifying Scripts
|
||||
|
||||
1. Test changes locally first
|
||||
2. Update documentation if behavior changes
|
||||
3. Commit and push to repository
|
||||
4. Jenkins will use updated scripts on next build
|
||||
|
||||
## Support
|
||||
|
||||
For issues or questions:
|
||||
- Check logs in Jenkins artifacts
|
||||
- Review troubleshooting section above
|
||||
- Verify prerequisites are installed correctly
|
||||
|
||||
## License
|
||||
|
||||
[Specify your license here]
|
||||
|
||||
## Authors
|
||||
|
||||
Rocky Linux Testing Team
|
||||
25
scripts/cleanup_vm.sh
Executable file
25
scripts/cleanup_vm.sh
Executable file
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
VM_NAME="$1"
|
||||
|
||||
if [ -z "$VM_NAME" ]; then
|
||||
echo "Usage: $0 <vm_name>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[Cleanup] Starting cleanup for VM: $VM_NAME"
|
||||
|
||||
# Force stop VM
|
||||
echo "[Cleanup] Destroying VM..."
|
||||
sudo virsh destroy "$VM_NAME" 2>/dev/null || echo "[Cleanup] VM was not running"
|
||||
|
||||
# Remove VM definition
|
||||
echo "[Cleanup] Undefining VM..."
|
||||
sudo virsh undefine "$VM_NAME" 2>/dev/null || echo "[Cleanup] VM definition already removed"
|
||||
|
||||
# Remove disk image
|
||||
echo "[Cleanup] Removing disk image..."
|
||||
sudo rm -f "/var/lib/libvirt/images/${VM_NAME}.qcow2" || echo "[Cleanup] Disk already removed"
|
||||
|
||||
echo "[Cleanup] Complete for $VM_NAME"
|
||||
64
scripts/provision_vm.sh
Executable file
64
scripts/provision_vm.sh
Executable file
@@ -0,0 +1,64 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
VM_NAME="$1"
|
||||
GOLDEN_IMAGE="$2"
|
||||
MAX_WAIT="${3:-30}"
|
||||
|
||||
if [ -z "$VM_NAME" ] || [ -z "$GOLDEN_IMAGE" ]; then
|
||||
echo "Usage: $0 <vm_name> <golden_image_path> [max_wait_seconds]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[Provision] Creating VM: $VM_NAME"
|
||||
|
||||
# Create linked clone (very fast - just a pointer to base)
|
||||
VM_DISK="/var/lib/libvirt/images/${VM_NAME}.qcow2"
|
||||
|
||||
echo "[Provision] Creating overlay disk: $VM_DISK"
|
||||
sudo qemu-img create -f qcow2 -b "$GOLDEN_IMAGE" -F qcow2 "$VM_DISK" 2>/dev/null
|
||||
|
||||
# Define and start VM
|
||||
echo "[Provision] Starting VM with virt-install..."
|
||||
sudo virt-install \
|
||||
--name "$VM_NAME" \
|
||||
--memory 2048 \
|
||||
--vcpus 2 \
|
||||
--disk path="$VM_DISK",format=qcow2 \
|
||||
--import \
|
||||
--os-variant rocky9-unknown \
|
||||
--network network=default \
|
||||
--noautoconsole \
|
||||
--wait 0 \
|
||||
--transient \
|
||||
2>&1 | grep -v "WARNING" || true
|
||||
|
||||
# Wait for IP address
|
||||
echo "[Provision] Waiting for VM to obtain IP address (max ${MAX_WAIT}s)..."
|
||||
COUNTER=0
|
||||
while [ $COUNTER -lt $MAX_WAIT ]; do
|
||||
# Try to get IP from DHCP lease
|
||||
IP=$(sudo virsh domifaddr "$VM_NAME" --source lease 2>/dev/null | awk '/ipv4/ {print $4}' | cut -d/ -f1 | head -1)
|
||||
|
||||
if [ -n "$IP" ] && [ "$IP" != "0.0.0.0" ]; then
|
||||
echo "[Provision] IP obtained: $IP"
|
||||
echo "$IP"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
((COUNTER++))
|
||||
|
||||
# Show progress every 5 iterations
|
||||
if [ $((COUNTER % 5)) -eq 0 ]; then
|
||||
echo "[Provision] Still waiting... (${COUNTER}/${MAX_WAIT}s)"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "[Provision] ERROR: Could not obtain IP for $VM_NAME after $MAX_WAIT seconds"
|
||||
echo "[Provision] Destroying failed VM..."
|
||||
sudo virsh destroy "$VM_NAME" 2>/dev/null || true
|
||||
sudo virsh undefine "$VM_NAME" 2>/dev/null || true
|
||||
sudo rm -f "$VM_DISK"
|
||||
echo "ERROR"
|
||||
exit 1
|
||||
65
scripts/setup_base.sh
Executable file
65
scripts/setup_base.sh
Executable file
@@ -0,0 +1,65 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Signature: ./setup_base.sh <qcow2_path> <prep_script_path> <output_golden_image> <ssh_pub_key>
|
||||
QCOW2_PATH="$1"
|
||||
PREP_SCRIPT_PATH="$2"
|
||||
GOLDEN_IMAGE="$3"
|
||||
SSH_PUB_KEY="$4"
|
||||
|
||||
if [ -z "$QCOW2_PATH" ] || [ -z "$GOLDEN_IMAGE" ] || [ -z "$SSH_PUB_KEY" ]; then
|
||||
echo "Usage: $0 <qcow2_path> <prep_script_path> <output_golden_image> <ssh_pub_key>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=========================================="
|
||||
echo "Setting up golden image"
|
||||
echo "=========================================="
|
||||
echo "Source QCOW2: $QCOW2_PATH"
|
||||
echo "Output Golden: $GOLDEN_IMAGE"
|
||||
echo "Prep Script: $PREP_SCRIPT_PATH"
|
||||
echo "SSH Key: $SSH_PUB_KEY"
|
||||
|
||||
# Ensure libvirt is running
|
||||
sudo systemctl is-active --quiet libvirtd || sudo systemctl start libvirtd
|
||||
sleep 2
|
||||
|
||||
# Copy original to golden
|
||||
echo "[Step 1/3] Copying base image to golden image..."
|
||||
cp "$QCOW2_PATH" "$GOLDEN_IMAGE"
|
||||
|
||||
# Apply custom preparation script if provided
|
||||
if [ -f "$PREP_SCRIPT_PATH" ]; then
|
||||
echo "[Step 2/3] Applying custom preparation script..."
|
||||
export LIBGUESTFS_BACKEND=direct
|
||||
|
||||
# Run the prep script inside the image
|
||||
sudo virt-customize -a "$GOLDEN_IMAGE" \
|
||||
--run "$PREP_SCRIPT_PATH" \
|
||||
--ssh-inject root:file:"$SSH_PUB_KEY" \
|
||||
--root-password password:rockytesting \
|
||||
--selinux-relabel 2>&1 || {
|
||||
echo "ERROR: virt-customize failed"
|
||||
exit 1
|
||||
}
|
||||
else
|
||||
echo "[Step 2/3] No custom prep script provided, applying defaults..."
|
||||
export LIBGUESTFS_BACKEND=direct
|
||||
|
||||
sudo virt-customize -a "$GOLDEN_IMAGE" \
|
||||
--ssh-inject root:file:"$SSH_PUB_KEY" \
|
||||
--root-password password:rockytesting \
|
||||
--install perl,git,wget,tar,openssh-server \
|
||||
--run-command 'systemctl enable sshd' \
|
||||
--selinux-relabel 2>&1 || {
|
||||
echo "ERROR: virt-customize failed"
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
echo "[Step 3/3] Verifying golden image..."
|
||||
qemu-img info "$GOLDEN_IMAGE" | head -5
|
||||
|
||||
echo "=========================================="
|
||||
echo "Golden image ready: $GOLDEN_IMAGE"
|
||||
echo "=========================================="
|
||||
Reference in New Issue
Block a user