445 lines
18 KiB
Groovy
445 lines
18 KiB
Groovy
pipeline {
|
|
agent {
|
|
label 'fedora-testing'
|
|
customWorkspace "/home/jenkins/jenkins-workspace/${BUILD_NUMBER}"
|
|
}
|
|
|
|
parameters {
|
|
// QCOW2 Image Source
|
|
string(
|
|
name: 'QCOW2_URL',
|
|
defaultValue: 'https://download.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2',
|
|
description: 'URL to the base QCOW2 image to use for testing'
|
|
)
|
|
|
|
// Test Matrix - JSON array format
|
|
text(
|
|
name: 'TEST_MATRIX',
|
|
defaultValue: '''[
|
|
{
|
|
"name": "example-test",
|
|
"url": "https://github.com/your-org/sparrowdo-tests.git",
|
|
"branch": "main",
|
|
"description": "Example Sparrowdo test suite"
|
|
}
|
|
]''',
|
|
description: 'JSON array of test definitions (name, url, branch, description)'
|
|
)
|
|
|
|
// Custom Golden Image Preparation
|
|
text(
|
|
name: 'GOLDEN_PREP_SCRIPT',
|
|
defaultValue: '''#!/bin/bash
|
|
set -e
|
|
|
|
echo "Preparing standard golden image..."
|
|
|
|
# NOTE: This runs inside virt-customize (offline mode)
|
|
# Cannot use firewall-cmd, hostnamectl, or systemctl start/restart
|
|
# Only systemctl enable works (creates symlinks)
|
|
|
|
# Update system (optional - can be slow, commented out by default)
|
|
# dnf update -y
|
|
|
|
# Install common testing dependencies including Raku/Sparrowdo
|
|
dnf install -y \\
|
|
perl \\
|
|
git \\
|
|
wget \\
|
|
tar \\
|
|
openssh-server \\
|
|
vim \\
|
|
rakudo \\
|
|
rakudo-zef
|
|
|
|
# Enable services (works offline)
|
|
systemctl enable sshd
|
|
|
|
# Create rocky user (standard non-root user)
|
|
useradd -m rocky 2>/dev/null || true
|
|
echo "rocky:rockypass" | chpasswd
|
|
|
|
# Add rocky to sudoers (needed for sparrowdo --bootstrap)
|
|
echo "rocky ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/rocky
|
|
chmod 0440 /etc/sudoers.d/rocky
|
|
|
|
# Create testuser for backward compatibility
|
|
useradd -m testuser 2>/dev/null || true
|
|
echo "testuser:testpass" | chpasswd
|
|
|
|
# Add to sudoers
|
|
echo "testuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/testuser
|
|
chmod 0440 /etc/sudoers.d/testuser
|
|
|
|
echo "Golden image preparation complete!"
|
|
''',
|
|
description: 'Shell script to run inside the golden image (virt-customize offline mode)'
|
|
)
|
|
|
|
// Test Selection
|
|
string(
|
|
name: 'TEST_FILTER',
|
|
defaultValue: '.*',
|
|
description: 'Regex pattern to filter tests (e.g., "login.*" or ".*")'
|
|
)
|
|
|
|
// Concurrency Control
|
|
choice(
|
|
name: 'MAX_CONCURRENT',
|
|
choices: ['1', '2', '3', '5', '10'],
|
|
description: 'Maximum concurrent VMs'
|
|
)
|
|
|
|
// Image Download Control
|
|
booleanParam(
|
|
name: 'REDOWNLOAD_IMAGE',
|
|
defaultValue: false,
|
|
description: 'Force re-download of base QCOW2 image even if it exists'
|
|
)
|
|
|
|
// Cleanup Control
|
|
booleanParam(
|
|
name: 'KEEP_GOLDEN_IMAGE',
|
|
defaultValue: false,
|
|
description: 'Keep golden image after testing'
|
|
)
|
|
|
|
// SSH Configuration
|
|
string(
|
|
name: 'SSH_PRIVATE_KEY_PATH',
|
|
defaultValue: '${HOME}/.ssh/id_rsa',
|
|
description: 'Path to SSH private key for VM access'
|
|
)
|
|
|
|
string(
|
|
name: 'SSH_PUBLIC_KEY_PATH',
|
|
defaultValue: '${HOME}/.ssh/id_rsa.pub',
|
|
description: 'Path to SSH public key to inject into VMs'
|
|
)
|
|
}
|
|
|
|
environment {
|
|
BUILD_ID = "${BUILD_NUMBER}"
|
|
IMAGES_DIR = "/var/lib/libvirt/images"
|
|
MAX_PARALLEL = "${params.MAX_CONCURRENT}"
|
|
}
|
|
|
|
options {
|
|
disableConcurrentBuilds()
|
|
timeout(time: 2, unit: 'HOURS')
|
|
buildDiscarder(logRotator(numToKeepStr: '10'))
|
|
}
|
|
|
|
stages {
|
|
stage('Initialize') {
|
|
steps {
|
|
script {
|
|
echo "=========================================="
|
|
echo "Build ${BUILD_ID}"
|
|
echo "Node: ${env.NODE_NAME}"
|
|
echo "Max Concurrent: ${params.MAX_CONCURRENT}"
|
|
echo "Test Filter: ${params.TEST_FILTER}"
|
|
echo "=========================================="
|
|
|
|
sh '''
|
|
sudo mkdir -p ${IMAGES_DIR}
|
|
sudo chown ${USER}:${USER} ${IMAGES_DIR}
|
|
chmod +x scripts/*.sh
|
|
echo "Environment initialized"
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Download Base Image') {
|
|
steps {
|
|
script {
|
|
dir(IMAGES_DIR) {
|
|
// Extract filename from URL for caching
|
|
def imageFilename = params.QCOW2_URL.split('/').last()
|
|
def cachedImagePath = "${IMAGES_DIR}/${imageFilename}"
|
|
def buildImagePath = "${IMAGES_DIR}/base-${BUILD_ID}.qcow2"
|
|
|
|
if (params.REDOWNLOAD_IMAGE) {
|
|
echo "REDOWNLOAD_IMAGE is enabled - forcing fresh download"
|
|
sh """
|
|
echo "Downloading QCOW2 image from: ${QCOW2_URL}"
|
|
curl -L --progress-bar -o "${buildImagePath}" "${QCOW2_URL}"
|
|
echo ""
|
|
echo "Image downloaded successfully:"
|
|
qemu-img info "${buildImagePath}" | head -5
|
|
"""
|
|
} else {
|
|
sh """
|
|
if [ -f "${cachedImagePath}" ]; then
|
|
echo "Found cached image: ${cachedImagePath}"
|
|
echo "Creating copy for build ${BUILD_ID}..."
|
|
cp "${cachedImagePath}" "${buildImagePath}"
|
|
qemu-img info "${buildImagePath}" | head -5
|
|
else
|
|
echo "No cached image found at: ${cachedImagePath}"
|
|
echo "Downloading QCOW2 image from: ${QCOW2_URL}"
|
|
curl -L --progress-bar -o "${cachedImagePath}" "${QCOW2_URL}"
|
|
echo ""
|
|
echo "Image downloaded successfully:"
|
|
qemu-img info "${cachedImagePath}" | head -5
|
|
echo "Creating copy for build ${BUILD_ID}..."
|
|
cp "${cachedImagePath}" "${buildImagePath}"
|
|
fi
|
|
"""
|
|
}
|
|
|
|
env.BASE_IMAGE_PATH = buildImagePath
|
|
echo "Base image ready at: ${buildImagePath}"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Prepare Golden Image') {
|
|
steps {
|
|
script {
|
|
def goldenImage = "${IMAGES_DIR}/golden-${BUILD_ID}.qcow2"
|
|
def prepScript = "prep-${BUILD_ID}.sh"
|
|
|
|
// Expand SSH key paths
|
|
def sshPubKey = params.SSH_PUBLIC_KEY_PATH.replace('${HOME}', env.HOME)
|
|
|
|
// Write prep script
|
|
writeFile file: prepScript, text: params.GOLDEN_PREP_SCRIPT
|
|
sh "chmod +x ${prepScript}"
|
|
|
|
env.PREP_SCRIPT_PATH = pwd() + "/" + prepScript
|
|
env.GOLDEN_IMAGE_PATH = goldenImage
|
|
env.SSH_PUB_KEY_PATH = sshPubKey
|
|
|
|
echo "Creating golden image: ${goldenImage}"
|
|
echo "Using SSH public key: ${sshPubKey}"
|
|
|
|
sh '''
|
|
./scripts/setup_base.sh \\
|
|
${BASE_IMAGE_PATH} \\
|
|
${PREP_SCRIPT_PATH} \\
|
|
${GOLDEN_IMAGE_PATH} \\
|
|
${SSH_PUB_KEY_PATH}
|
|
'''
|
|
|
|
sh "rm -f ${prepScript}"
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Bootstrap Golden Image') {
|
|
steps {
|
|
script {
|
|
echo "=========================================="
|
|
echo "Bootstrapping Sparrowdo in Golden Image"
|
|
echo "=========================================="
|
|
echo "This installs Sparrowdo ONCE in the golden image"
|
|
echo "All test VMs will inherit this and skip bootstrap"
|
|
echo ""
|
|
|
|
// Expand SSH private key path
|
|
def sshPrivateKey = params.SSH_PRIVATE_KEY_PATH.replace('${HOME}', env.HOME)
|
|
|
|
sh """
|
|
./scripts/bootstrap_golden.sh \\
|
|
${GOLDEN_IMAGE_PATH} \\
|
|
${sshPrivateKey}
|
|
"""
|
|
|
|
echo "Golden image successfully bootstrapped"
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Parse Test Matrix') {
|
|
steps {
|
|
script {
|
|
def testMatrix = readJSON text: params.TEST_MATRIX.trim()
|
|
def filtered = testMatrix.findAll { it.name =~ params.TEST_FILTER }
|
|
|
|
if (filtered.isEmpty()) {
|
|
error "No tests match filter: ${params.TEST_FILTER}"
|
|
}
|
|
|
|
echo "=========================================="
|
|
echo "Tests to run (${filtered.size()}):"
|
|
filtered.each { echo " - ${it.name}: ${it.description}" }
|
|
echo "=========================================="
|
|
|
|
env.FILTERED_TESTS = groovy.json.JsonOutput.toJson(filtered)
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Run Tests') {
|
|
steps {
|
|
script {
|
|
def tests = readJSON text: env.FILTERED_TESTS
|
|
def tasks = [:]
|
|
|
|
// Expand SSH key path
|
|
def sshPrivateKey = params.SSH_PRIVATE_KEY_PATH.replace('${HOME}', env.HOME)
|
|
|
|
tests.each { test ->
|
|
def testName = test.name
|
|
def testUrl = test.url
|
|
def testBranch = test.branch ?: 'main'
|
|
|
|
tasks["${testName}"] = {
|
|
lock(resource: 'kvm-slots', quantity: 1) {
|
|
stage("Test: ${testName}") {
|
|
def vmId = "${testName}-${BUILD_ID}"
|
|
def targetIp = ""
|
|
|
|
try {
|
|
dir("ws-${testName}") {
|
|
echo "[${testName}] Provisioning VM..."
|
|
def ipOutput = sh(
|
|
script: "${WORKSPACE}/scripts/provision_vm.sh ${vmId} ${GOLDEN_IMAGE_PATH} 60",
|
|
returnStdout: true
|
|
).trim()
|
|
|
|
if (ipOutput.contains("ERROR")) {
|
|
error "Failed to provision VM"
|
|
}
|
|
|
|
// Extract IP from output (last line)
|
|
targetIp = ipOutput.split('\n').last()
|
|
echo "[${testName}] VM ready at: ${targetIp}"
|
|
|
|
// Wait for SSH to be ready
|
|
echo "[${testName}] Waiting for SSH..."
|
|
sh """
|
|
for i in {1..30}; do
|
|
if ssh -i ${sshPrivateKey} -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@${targetIp} 'echo SSH ready' 2>/dev/null; then
|
|
echo "SSH connection established"
|
|
break
|
|
fi
|
|
echo "Attempt \$i/30..."
|
|
sleep 2
|
|
done
|
|
"""
|
|
|
|
// Clone test repository
|
|
echo "[${testName}] Cloning test repository..."
|
|
sh "git clone -b ${testBranch} ${testUrl} test-repo || true"
|
|
|
|
// Verify sparrowfile exists (check for main.raku or sparrowfile)
|
|
def sparrowfilePath = sh(
|
|
script: 'find test-repo -name main.raku -type f | head -1',
|
|
returnStdout: true
|
|
).trim()
|
|
|
|
if (!sparrowfilePath) {
|
|
sparrowfilePath = sh(
|
|
script: 'find test-repo -name sparrowfile -type f | head -1',
|
|
returnStdout: true
|
|
).trim()
|
|
}
|
|
|
|
if (!sparrowfilePath) {
|
|
error "No sparrowfile or main.raku found in test repository"
|
|
}
|
|
|
|
echo "[${testName}] Found sparrowfile: ${sparrowfilePath}"
|
|
|
|
// Run test (bootstrap was already done on golden image)
|
|
echo "[${testName}] Running Sparrowdo test..."
|
|
sh """
|
|
mkdir -p logs
|
|
timeout 900 sparrowdo \\
|
|
--host=${targetIp} \\
|
|
--ssh_user=rocky \\
|
|
--ssh_private_key=${sshPrivateKey} \\
|
|
--ssh_args='-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null' \\
|
|
--no_sudo \\
|
|
--sparrowfile=${sparrowfilePath} \\
|
|
--verbose \\
|
|
--color 2>&1 | tee logs/test.log
|
|
"""
|
|
|
|
echo "[${testName}] Test completed successfully"
|
|
archiveArtifacts artifacts: "logs/**", allowEmptyArchive: true
|
|
}
|
|
|
|
} catch (e) {
|
|
echo "[${testName}] FAILED: ${e.message}"
|
|
|
|
// Archive any logs that were generated
|
|
dir("ws-${testName}") {
|
|
archiveArtifacts artifacts: "logs/**", allowEmptyArchive: true
|
|
}
|
|
|
|
currentBuild.result = 'UNSTABLE'
|
|
|
|
} finally {
|
|
echo "[${testName}] Cleaning up VM..."
|
|
sh "${WORKSPACE}/scripts/cleanup_vm.sh ${vmId} || true"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Run all tests in parallel (respecting lock quantity)
|
|
parallel tasks
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
post {
|
|
always {
|
|
script {
|
|
echo "=========================================="
|
|
echo "Post-build cleanup"
|
|
echo "=========================================="
|
|
|
|
if (!params.KEEP_GOLDEN_IMAGE) {
|
|
sh '''
|
|
echo "Removing temporary images..."
|
|
sudo rm -f ${IMAGES_DIR}/base-${BUILD_ID}.qcow2 || true
|
|
sudo rm -f ${IMAGES_DIR}/golden-${BUILD_ID}.qcow2 || true
|
|
echo "Temporary images cleaned up"
|
|
'''
|
|
} else {
|
|
echo "Golden image preserved: ${GOLDEN_IMAGE_PATH}"
|
|
echo "Base image preserved: ${BASE_IMAGE_PATH}"
|
|
}
|
|
|
|
// Clean up any orphaned VMs
|
|
sh '''
|
|
echo "Checking for orphaned VMs from build ${BUILD_ID}..."
|
|
for vm in $(virsh list --all --name 2>/dev/null | grep -E "-${BUILD_ID}"); do
|
|
echo "Cleaning orphaned VM: $vm"
|
|
sudo virsh destroy "$vm" 2>/dev/null || true
|
|
sudo virsh undefine "$vm" 2>/dev/null || true
|
|
sudo rm -f "/var/lib/libvirt/images/${vm}.qcow2" 2>/dev/null || true
|
|
done
|
|
echo "Orphan cleanup complete"
|
|
'''
|
|
}
|
|
}
|
|
|
|
success {
|
|
echo "=========================================="
|
|
echo "Build ${BUILD_ID} completed successfully!"
|
|
echo "=========================================="
|
|
}
|
|
|
|
failure {
|
|
echo "=========================================="
|
|
echo "Build ${BUILD_ID} failed!"
|
|
echo "=========================================="
|
|
}
|
|
|
|
unstable {
|
|
echo "=========================================="
|
|
echo "Build ${BUILD_ID} completed with test failures"
|
|
echo "=========================================="
|
|
}
|
|
}
|
|
}
|