Initial commit: Sparrowdo Testing Orchestrator framework
- Add VM provisioning, cleanup, and setup scripts - Add Jenkins pipeline with parameterized builds - Add comprehensive documentation - Support for parallel test execution with QCOW2 linked clones
This commit is contained in:
Stephen Simpson
374
Jenkinsfile
vendored
Normal file
374
Jenkinsfile
vendored
Normal file
@@ -0,0 +1,374 @@
|
||||
pipeline {
|
||||
agent {
|
||||
label 'fedora-testing'
|
||||
customWorkspace "/home/jenkins/jenkins-workspace/${BUILD_NUMBER}"
|
||||
}
|
||||
|
||||
parameters {
|
||||
// QCOW2 Image Source
|
||||
string(
|
||||
name: 'QCOW2_URL',
|
||||
defaultValue: 'https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2',
|
||||
description: 'URL to the base QCOW2 image to use for testing'
|
||||
)
|
||||
|
||||
// Test Matrix - JSON array format
|
||||
text(
|
||||
name: 'TEST_MATRIX',
|
||||
defaultValue: '''[
|
||||
{
|
||||
"name": "example-test",
|
||||
"url": "https://github.com/your-org/sparrowdo-tests.git",
|
||||
"branch": "main",
|
||||
"description": "Example Sparrowdo test suite"
|
||||
}
|
||||
]''',
|
||||
description: 'JSON array of test definitions (name, url, branch, description)'
|
||||
)
|
||||
|
||||
// Custom Golden Image Preparation
|
||||
text(
|
||||
name: 'GOLDEN_PREP_SCRIPT',
|
||||
defaultValue: '''#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "Preparing standard golden image..."
|
||||
|
||||
# Update system
|
||||
dnf update -y
|
||||
|
||||
# Install common testing dependencies
|
||||
dnf install -y \\
|
||||
perl \\
|
||||
git \\
|
||||
wget \\
|
||||
tar \\
|
||||
openssh-server \\
|
||||
firewalld \\
|
||||
chrony \\
|
||||
vim
|
||||
|
||||
# Configure services
|
||||
systemctl enable sshd
|
||||
systemctl enable firewalld
|
||||
systemctl enable chronyd
|
||||
|
||||
# Configure firewall
|
||||
firewall-cmd --permanent --add-service=ssh
|
||||
firewall-cmd --reload
|
||||
|
||||
# Set consistent hostname
|
||||
hostnamectl set-hostname test-node
|
||||
|
||||
# Create testing user
|
||||
useradd -m testuser 2>/dev/null || true
|
||||
echo "testuser:testpass" | chpasswd
|
||||
echo "testuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
|
||||
|
||||
echo "Golden image preparation complete!"
|
||||
''',
|
||||
description: 'Shell script to run inside the golden image'
|
||||
)
|
||||
|
||||
// Test Selection
|
||||
string(
|
||||
name: 'TEST_FILTER',
|
||||
defaultValue: '.*',
|
||||
description: 'Regex pattern to filter tests (e.g., "login.*" or ".*")'
|
||||
)
|
||||
|
||||
// Concurrency Control
|
||||
choice(
|
||||
name: 'MAX_CONCURRENT',
|
||||
choices: ['1', '2', '3', '5', '10'],
|
||||
description: 'Maximum concurrent VMs'
|
||||
)
|
||||
|
||||
// Cleanup Control
|
||||
booleanParam(
|
||||
name: 'KEEP_GOLDEN_IMAGE',
|
||||
defaultValue: false,
|
||||
description: 'Keep golden image after testing'
|
||||
)
|
||||
|
||||
// SSH Configuration
|
||||
string(
|
||||
name: 'SSH_PRIVATE_KEY_PATH',
|
||||
defaultValue: '${HOME}/.ssh/id_rsa',
|
||||
description: 'Path to SSH private key for VM access'
|
||||
)
|
||||
|
||||
string(
|
||||
name: 'SSH_PUBLIC_KEY_PATH',
|
||||
defaultValue: '${HOME}/.ssh/id_rsa.pub',
|
||||
description: 'Path to SSH public key to inject into VMs'
|
||||
)
|
||||
}
|
||||
|
||||
environment {
|
||||
BUILD_ID = "${BUILD_NUMBER}"
|
||||
IMAGES_DIR = "/var/lib/libvirt/images"
|
||||
MAX_PARALLEL = "${params.MAX_CONCURRENT}"
|
||||
}
|
||||
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
timeout(time: 2, unit: 'HOURS')
|
||||
buildDiscarder(logRotator(numToKeepStr: '10'))
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Initialize') {
|
||||
steps {
|
||||
script {
|
||||
echo "=========================================="
|
||||
echo "Build ${BUILD_ID}"
|
||||
echo "Node: ${env.NODE_NAME}"
|
||||
echo "Max Concurrent: ${params.MAX_CONCURRENT}"
|
||||
echo "Test Filter: ${params.TEST_FILTER}"
|
||||
echo "=========================================="
|
||||
|
||||
sh '''
|
||||
sudo mkdir -p ${IMAGES_DIR}
|
||||
sudo chown ${USER}:${USER} ${IMAGES_DIR}
|
||||
chmod +x scripts/*.sh
|
||||
echo "Environment initialized"
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Download Base Image') {
|
||||
steps {
|
||||
script {
|
||||
dir(IMAGES_DIR) {
|
||||
sh '''
|
||||
if [ ! -f "base-${BUILD_ID}.qcow2" ]; then
|
||||
echo "Downloading QCOW2 image from: ${QCOW2_URL}"
|
||||
curl -L --progress-bar -o "base-${BUILD_ID}.qcow2" "${QCOW2_URL}"
|
||||
echo ""
|
||||
echo "Image downloaded successfully:"
|
||||
qemu-img info "base-${BUILD_ID}.qcow2" | head -5
|
||||
else
|
||||
echo "Using cached base image"
|
||||
fi
|
||||
'''
|
||||
env.BASE_IMAGE_PATH = "${IMAGES_DIR}/base-${BUILD_ID}.qcow2"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Prepare Golden Image') {
|
||||
steps {
|
||||
script {
|
||||
def goldenImage = "${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"
|
||||
def prepScript = "prep-${BUILD_ID}.sh"
|
||||
|
||||
// Expand SSH key paths
|
||||
def sshPubKey = params.SSH_PUBLIC_KEY_PATH.replace('${HOME}', env.HOME)
|
||||
|
||||
// Write prep script
|
||||
writeFile file: prepScript, text: params.GOLDEN_PREP_SCRIPT
|
||||
sh "chmod +x ${prepScript}"
|
||||
|
||||
env.PREP_SCRIPT_PATH = pwd() + "/" + prepScript
|
||||
env.GOLDEN_IMAGE_PATH = goldenImage
|
||||
env.SSH_PUB_KEY_PATH = sshPubKey
|
||||
|
||||
echo "Creating golden image: ${goldenImage}"
|
||||
echo "Using SSH public key: ${sshPubKey}"
|
||||
|
||||
sh '''
|
||||
./scripts/setup_base.sh \\
|
||||
${BASE_IMAGE_PATH} \\
|
||||
${PREP_SCRIPT_PATH} \\
|
||||
${GOLDEN_IMAGE_PATH} \\
|
||||
${SSH_PUB_KEY_PATH}
|
||||
'''
|
||||
|
||||
sh "rm -f ${prepScript}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Parse Test Matrix') {
|
||||
steps {
|
||||
script {
|
||||
def testMatrix = readJSON text: params.TEST_MATRIX.trim()
|
||||
def filtered = testMatrix.findAll { it.name =~ params.TEST_FILTER }
|
||||
|
||||
if (filtered.isEmpty()) {
|
||||
error "No tests match filter: ${params.TEST_FILTER}"
|
||||
}
|
||||
|
||||
echo "=========================================="
|
||||
echo "Tests to run (${filtered.size()}):"
|
||||
filtered.each { echo " - ${it.name}: ${it.description}" }
|
||||
echo "=========================================="
|
||||
|
||||
env.FILTERED_TESTS = groovy.json.JsonOutput.toJson(filtered)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Run Tests') {
|
||||
steps {
|
||||
script {
|
||||
def tests = readJSON text: env.FILTERED_TESTS
|
||||
def tasks = [:]
|
||||
|
||||
// Expand SSH key path
|
||||
def sshPrivateKey = params.SSH_PRIVATE_KEY_PATH.replace('${HOME}', env.HOME)
|
||||
|
||||
tests.each { test ->
|
||||
def testName = test.name
|
||||
def testUrl = test.url
|
||||
def testBranch = test.branch ?: 'main'
|
||||
|
||||
tasks["${testName}"] = {
|
||||
lock(resource: 'kvm-slots', quantity: 1) {
|
||||
stage("Test: ${testName}") {
|
||||
def vmId = "${testName}-${BUILD_ID}"
|
||||
def targetIp = ""
|
||||
|
||||
try {
|
||||
dir("ws-${testName}") {
|
||||
echo "[${testName}] Provisioning VM..."
|
||||
def ipOutput = sh(
|
||||
script: "${WORKSPACE}/scripts/provision_vm.sh ${vmId} ${GOLDEN_IMAGE_PATH} 60",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
if (ipOutput.contains("ERROR")) {
|
||||
error "Failed to provision VM"
|
||||
}
|
||||
|
||||
// Extract IP from output (last line)
|
||||
targetIp = ipOutput.split('\n').last()
|
||||
echo "[${testName}] VM ready at: ${targetIp}"
|
||||
|
||||
// Wait for SSH to be ready
|
||||
echo "[${testName}] Waiting for SSH..."
|
||||
sh """
|
||||
for i in {1..30}; do
|
||||
if ssh -i ${sshPrivateKey} -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@${targetIp} 'echo SSH ready' 2>/dev/null; then
|
||||
echo "SSH connection established"
|
||||
break
|
||||
fi
|
||||
echo "Attempt \$i/30..."
|
||||
sleep 2
|
||||
done
|
||||
"""
|
||||
|
||||
// Clone test repository
|
||||
echo "[${testName}] Cloning test repository..."
|
||||
sh "git clone -b ${testBranch} ${testUrl} test-repo || true"
|
||||
|
||||
// Verify sparrowfile exists
|
||||
def sparrowfilePath = sh(
|
||||
script: 'find test-repo -name sparrowfile -type f | head -1',
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
if (!sparrowfilePath) {
|
||||
error "No sparrowfile found in test repository"
|
||||
}
|
||||
|
||||
echo "[${testName}] Found sparrowfile: ${sparrowfilePath}"
|
||||
|
||||
// Run test
|
||||
echo "[${testName}] Running Sparrowdo..."
|
||||
sh """
|
||||
mkdir -p logs
|
||||
timeout 900 sparrowdo \\
|
||||
--host=${targetIp} \\
|
||||
--ssh_user=root \\
|
||||
--ssh_private_key=${sshPrivateKey} \\
|
||||
--ssh_args='-o StrictHostKeyChecking=no -o ConnectTimeout=10' \\
|
||||
--no_sudo \\
|
||||
--sparrowfile=${sparrowfilePath} 2>&1 | tee logs/test.log
|
||||
"""
|
||||
|
||||
echo "[${testName}] Test completed successfully"
|
||||
archiveArtifacts artifacts: "logs/**", allowEmptyArchive: true
|
||||
}
|
||||
|
||||
} catch (e) {
|
||||
echo "[${testName}] FAILED: ${e.message}"
|
||||
|
||||
// Archive any logs that were generated
|
||||
dir("ws-${testName}") {
|
||||
archiveArtifacts artifacts: "logs/**", allowEmptyArchive: true
|
||||
}
|
||||
|
||||
currentBuild.result = 'UNSTABLE'
|
||||
|
||||
} finally {
|
||||
echo "[${testName}] Cleaning up VM..."
|
||||
sh "${WORKSPACE}/scripts/cleanup_vm.sh ${vmId} || true"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run all tests in parallel (respecting lock quantity)
|
||||
parallel tasks
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
always {
|
||||
script {
|
||||
echo "=========================================="
|
||||
echo "Post-build cleanup"
|
||||
echo "=========================================="
|
||||
|
||||
if (!params.KEEP_GOLDEN_IMAGE) {
|
||||
sh '''
|
||||
echo "Removing temporary images..."
|
||||
sudo rm -f ${IMAGES_DIR}/base-${BUILD_ID}.qcow2 || true
|
||||
sudo rm -f ${IMAGES_DIR}/golden-${BUILD_ID}.qcow2 || true
|
||||
echo "Temporary images cleaned up"
|
||||
'''
|
||||
} else {
|
||||
echo "Golden image preserved: ${GOLDEN_IMAGE_PATH}"
|
||||
echo "Base image preserved: ${BASE_IMAGE_PATH}"
|
||||
}
|
||||
|
||||
// Clean up any orphaned VMs
|
||||
sh '''
|
||||
echo "Checking for orphaned VMs from build ${BUILD_ID}..."
|
||||
for vm in $(virsh list --all --name 2>/dev/null | grep -E "-${BUILD_ID}"); do
|
||||
echo "Cleaning orphaned VM: $vm"
|
||||
sudo virsh destroy "$vm" 2>/dev/null || true
|
||||
sudo virsh undefine "$vm" 2>/dev/null || true
|
||||
sudo rm -f "/var/lib/libvirt/images/${vm}.qcow2" 2>/dev/null || true
|
||||
done
|
||||
echo "Orphan cleanup complete"
|
||||
'''
|
||||
}
|
||||
}
|
||||
|
||||
success {
|
||||
echo "=========================================="
|
||||
echo "Build ${BUILD_ID} completed successfully!"
|
||||
echo "=========================================="
|
||||
}
|
||||
|
||||
failure {
|
||||
echo "=========================================="
|
||||
echo "Build ${BUILD_ID} failed!"
|
||||
echo "=========================================="
|
||||
}
|
||||
|
||||
unstable {
|
||||
echo "=========================================="
|
||||
echo "Build ${BUILD_ID} completed with test failures"
|
||||
echo "=========================================="
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user