#!groovy // CONFIGURATION // // The following "String Paramaters" can control the way this job functions. // There are usually sensible defaults if they are not set (YYMV) // gitOrg The GitHub Account ie: 'tripwire' // Repo The GitHub Repo of source to test, ie: cap-bridge // RepoBranch The GitHub Repo branch, ie: scm/develop or master // RepoCommit The GitHub Repo commit, ie: a2c72118a21ba9d093f0a4ba12a4e52c8e056e51 // AxonBranch The axon-vagrant branch, ie: scm/develop or master // AxonCommit The axon-vagrant commit, ie: a2c72118a21ba9d093f0a4ba12a4e52c8e056e51 // Platforms Optional. List one or more platforms to spin up. ie: cents-6-x64. Default is complete 'linux' list // TestSuite The testsuite to run, ie: cucumber // JobTimeout Run job with timeout in HOURS // SaveImageOnTestFailure If 'true', the virtual machine will be kept running if a test failure occurs // SaveImageOnProvisioningFailure If 'true', the virtual machine will be kept running when provisioning fails @Library('jenkins-pipeline-common@master') _ def gitOrg = "${params.gitOrg ?: 'tripwire'}" def axonBranch = "${params.AxonBranch ?: 'master'}" saveImageOnProvisioningFailure = "${params.SaveImageOnProvisioningFailure ?: 'false'}" os = reduceOS(params.Platforms ?: 'linux') //without the 'def' keyword 'os' is globally accesible echo "JENKINSJOB starting: ${System.currentTimeMillis()}" pipeline { agent { label "chef && vagrant" } options { timeout(time: 24, unit: 'HOURS') } environment { VSPHERE_PASSWORD = credentials('srvc_jenkinsauto') VSPHERE_USER = 'TSS\\srvc_jenkinsauto' VSPHERE_GROUP = "${os == 'linux' ? 'Axon_Linux' : 'Axon_Windows'}" } stages { stage('Prepare the space') { steps { script { echo "HAVEEXECUTOR: ${System.currentTimeMillis()}" // Wipe the workspace so we are building completely clean deleteDir() gitClone('tripwire', 'axon-vagrant', axonBranch, params.AxonCommit) gitCloneSubdir('CAP', 'cap-jenkins-scripts', 'master') gitCloneSubdir(gitOrg, params.Repo, params.RepoBranch, params.RepoCommit) def lockFile = "${os == 'linux' ? 'posix' : 'windows'}" createLockfiles(params.RepoBranch, params.Repo, lockFile) //https://stackoverflow.com/questions/26214497/is-there-a-way-to-use-private-git-repository-with-berkshelf sshagent (credentials: ['d3731a79-cb13-4994-abae-c189db4bdcb2']) { sh "berks vendor" } dir("./data") { sh "ln -s ../${params.Repo} ./${params.Repo} && ls ./${params.Repo}" } } } } stage ('Provision, Test, Report, and Destroy') { steps { script { def platforms = reducePlatforms(getPlatforms(os), params.Platforms, params.ExcludedPlatforms, params.InstallEG, params.InstallDKMS) // 20 threads/VMs are used by default def threads = 20 if (params.Threads != null) { threads = params.Threads as int echo "Setting threads param to ${threads} (PARALLELJOBS)" } else { echo "No Threads build parameter defined. Setting to default value ${threads} (PARALLELJOBS)" } for (int i = 0; i < platforms.size(); i += threads) { def runList = getTestReportDestroyRunList(i, threads, platforms) try { parallel runList } catch(err) { // something failed while testing. We can continue, but set the status to failed. failTheBuild("Failed on the testing, reporting, or destroying stage") } } echo "JENKINSJOB ending: ${System.currentTimeMillis()}" } } } } post { always { archiveLogFiles() dir ("./${params.Repo}") { deleteDir() } } success { echo 'Success' } failure { echo 'Failed!!!!!!!' } unstable { echo 'Unstable' } aborted { // if user aborts, destroy all VMs sh 'vagrant destroy -f' } } } def test(platform) { // Check if we can connect to the platform. If not, abort if (!checkVMConnectivity(platform)) { return } if (os == "linux") { testLinux(platform) } else { testWindows(platform) } } def testLinux(platform) { // By default, EG is not installed if (params.InstallEG != null && params.InstallEG.equalsIgnoreCase("true")) { sh "vagrant ssh '${platform}' -c \"lsmod | grep twnotify\"" } try { sh "vagrant ssh '${platform}' -c \"cat /vagrant_data/${params.Repo}/accept/Rakefile > /dev/null\"" } catch(err) { echo "Plugin repositories were not rsynced. Force re-sync" sh "vagrant rsync '${platform}'" } sh "vagrant ssh '${platform}' -c \"cd /vagrant_data/${params.Repo}/accept && sudo bundle install --full-index --retry=20 \"" // The '|| true' forces the command to return code 0. If we don't do this, // Jenkins 2 will interpret a completed cucumber run that includes test // failures as the job itself failing. We don't want that - we want to get // results out and have the J2 UI tell us exactly which tests failed. params.TestSuite.split('\\s*,\\s*').each { def suiteToRun = it.trim() if (params.JobTimeout != null) { echo "Executing Rake with Timeout of ${params.JobTimeout}" timeout(time: params.JobTimeout.toInteger(), unit: 'HOURS') { sh "vagrant ssh '${platform}' -c \"cd /vagrant_data/${params.Repo}/accept && sudo bundle exec rake ${suiteToRun}\" || true" } } else { sh "vagrant ssh '${platform}' -c \"cd /vagrant_data/${params.Repo}/accept && sudo bundle exec rake ${suiteToRun}\" || true" } } } def testWindows(platform) { try { // Write the git branch to a file in the repo because we are not copying the .git folder over to the VM. sh "echo ${params.RepoBranch.trim()} >> data/${params.Repo}/git_branch" retryRemoteSSHCommand(platform, "mkdir -p /cygdrive/c/vagrant_data/${params.Repo}", 3, 30) transferFilesToVM(platform, "/cygdrive/c/vagrant_data/${params.Repo}", "data/${params.Repo}/*") } catch(err) { // If winrm-upload reported a failure move on. Jenkins seems to do that even if the operation succeeded echo "Received some error while uploading files: ${err}. Moving on" } // The following line increases the amount of RAM given to each instance of WinRM as run by Vagrant. // The default value (800mb) was too low, causing the JVM to fail when launching ParTE. // (Having the value too low will cause Jenkins 2 to hang and time out when launching ParTE.) sh "vagrant winrm '${platform}' -s cmd -c 'winrm set winrm/config/winrs @{MaxMemoryPerShellMB=\"3000\"}'" loginUsers(platform) fixTWhelloPermissions(platform) fixFIMtoolsPermissions(platform) fixOVALtoolsPermissions(platform) fixUpgradeToolsPermissions(platform) withCredentials([sshUserPrivateKey(credentialsId: "d3731a79-cb13-4994-abae-c189db4bdcb2", keyFileVariable: 'GH_id_rsa')]) { transferFilesToVM(platform, "/cygdrive/c/Users/vagrant/.ssh/GH_id_rsa", GH_id_rsa) } sh "vagrant winrm '${platform}' -s cmd -c \"cd C:/vagrant_data/${params.Repo}/accept && dir\"" sh """ vagrant winrm '${platform}' -e -s powershell -c \" . ~/Documents/WindowsPowershell/ssh-agent-utils.ps1; ssh-add.exe ~/.ssh/GH_id_rsa \\\$env:GIT_SSH_COMMAND='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' cd C:/vagrant_data/${params.Repo}/accept; bundle install --full-index --retry=20\" """ // The 'ver>nul' guarantees a zero return code. Rake may return nonzero if tests // fail, which Jenkins 2 interprets as the script as a whole failing - it is not. // Leave it to the later test report steps to determine that. // // For an explanation of the usage of 'ver>nul', see: // https://stackoverflow.com/a/27822727/189297 params.TestSuite.split('\\s*,\\s*').each { def suiteToRun = it.trim() if (params.JobTimeout != null) { echo "Executing Rake with Timeout of ${params.JobTimeout}" timeout(time: params.JobTimeout.toInteger(), unit: 'HOURS') { sh "vagrant winrm '${platform}' -e -s powershell -c \"cd C:/vagrant_data/${params.Repo}/accept ; bundle exec rake ${suiteToRun}\" || true" } } else { sh "vagrant winrm '${platform}' -e -s powershell -c \"cd C:/vagrant_data/${params.Repo}/accept ; bundle exec rake ${suiteToRun}\" || true" } } } def loginUsers(platform){ // The following block runs only for cap-rsop related test suites. As such, // they are skipped if the "Repo" parameter is not "cap-rsop". // // The tests require that a user that exists on the machine has logged in // at least once before the test will pass. Of the three possible users, // "rsop", "rsop2", "rsop_exists", only the "rsop" and "rsop2" users need to // be logged in. The *.bat files set 'autologin' properties and restart the // machine to accomplish the logins. if (params.Repo == "cap-rsop") { echo 'The following 3 commands should show that users: rsop, rsop2, and rsop_exists have never logged in before.' sh "vagrant winrm '${platform}' -s cmd -c \"net user rsop | findstr /B /C:Last\"" sh "vagrant winrm '${platform}' -s cmd -c \"net user rsop2 | findstr /B /C:Last\"" sh "vagrant winrm '${platform}' -s cmd -c \"net user rsop_exists | findstr /B /C:Last\"" echo 'Setting autologin to login to rsop, then rebooting.' // Use winrm to start powershell with elevated privileges and then use cmd to start the batch file sh "vagrant winrm '${platform}' -e -s powershell -c \"cmd /C C:/vagrant_data/${params.Repo}/tools/vagrant_to_rsop.bat\"" // NOTE (jchai): The shutdown command has been hanging in J2. I think the command is working but the J2 slave // is not receiving the status code from the shutdown command, so let's try to run the command assuming it // works sh "vagrant winrm '${platform}' -s cmd -c \"shutdown -t 0 -r -f\" || true" sleep 240 echo 'Setting autologin to login to rsop2, then rebooting.' sh "vagrant winrm '${platform}' -e -s powershell -c \"cmd /C C:/vagrant_data/${params.Repo}/tools/rsop_to_rsop2.bat\"" sh "vagrant winrm '${platform}' -s cmd -c \"shutdown -t 0 -r -f\" || true" sleep 240 echo 'Setting autologin to login to vagrant, then rebooting.' sh "vagrant winrm '${platform}' -e -s powershell -c \"cmd /C C:/vagrant_data/${params.Repo}/tools/rsop2_to_vagrant.bat\"" sh "vagrant winrm '${platform}' -s cmd -c \"shutdown -t 0 -r -f\" || true" sleep 240 echo 'The following 3 commands should show that users: rsop, rsop2 have logged in, but rsop_exists has never logged in before.' sh "vagrant winrm '${platform}' -s cmd -c \"net user rsop | findstr /B /C:Last\"" sh "vagrant winrm '${platform}' -s cmd -c \"net user rsop2 | findstr /B /C:Last\"" sh "vagrant winrm '${platform}' -s cmd -c \"net user rsop_exists | findstr /B /C:Last\"" } } def fixTWhelloPermissions(platform){ // The following block is required for cap-exec related tests. As such, // they are skipped if the Repo parameter is not "cap-exec". // // The tests require that the file /cap-exec/accept/tools/twhello.exe have // execute permissions, and downloading the executable via git clone does not // set this permission, causing a test to fail in the accept test. if (params.Repo == "cap-exec") { sh "vagrant winrm '${platform}' -s cmd -c 'icacls c:/vagrant_data/cap-exec/accept/tools/twhello.exe /reset'" } } def fixFIMtoolsPermissions(platform){ // The following block is required for cap-fim related tests. As such, // they are skipped if the Repo parameter is not "cap-fim". // // The tests require that the file cap-fim/accept/helpers/reg_tools/changer-win32.exe have // execute permissions, and downloading the executable via git clone does not // set this permission, causing a test to fail in the accept test. if (params.Repo == "cap-fim") { sh "vagrant winrm '${platform}' -s cmd -c 'icacls c:/vagrant_data/cap-fim/accept/helpers/reg_tools/changer-win32.exe /reset'" sh "vagrant winrm '${platform}' -s cmd -c 'icacls c:/vagrant_data/cap-fim/accept/helpers/reg_tools/regln-x86.exe /reset'" } } def fixOVALtoolsPermissions(platform){ // The following block is required for cap-oval related tests. As such, // they are skipped if the Repo parameter is not "cap-oval". if (params.Repo == "cap-oval") { sh "vagrant winrm '${platform}' -s cmd -c 'icacls C:/vagrant_data/cap-oval/accept/helpers/tools/PsGetsid.exe /reset'" sh "vagrant winrm '${platform}' -s cmd -c 'icacls C:/vagrant_data/cap-oval/accept/helpers/tools/PsGetsid64.exe /reset'" sh "vagrant winrm '${platform}' -s cmd -c 'icacls C:/vagrant_data/cap-oval/accept/helpers/tools/accesschk.exe /reset'" sh "vagrant winrm '${platform}' -s cmd -c 'icacls C:/vagrant_data/cap-oval/accept/helpers/tools/accesschk64.exe /reset'" sh "vagrant winrm '${platform}' -s cmd -c 'icacls C:/vagrant_data/cap-oval/accept/helpers/tools/junction.exe /reset'" sh "vagrant winrm '${platform}' -s cmd -c 'icacls C:/vagrant_data/cap-oval/accept/helpers/tools/junction64.exe /reset'" sh "vagrant winrm '${platform}' -s cmd -c 'icacls C:/vagrant_data/cap-oval/accept/helpers/tools/mfacs.exe /reset'" } } def fixUpgradeToolsPermissions(platform){ // The following block is required for cap-upgrade related tests. As such, // they are skipped if the Repo parameter is not "cap-upgrade". if (params.Repo == "cap-upgrade") { sh "vagrant winrm '${platform}' -s cmd -c 'icacls C:/vagrant_data/cap-upgrade/accept/helpers/tools/sleep60.exe /reset'" sh "vagrant winrm '${platform}' -s cmd -c 'icacls C:/vagrant_data/cap-upgrade/accept/helpers/tools/sleep60_x64.exe /reset'" } } // TODO (jchai): Replace/Delete this method. This does not seem to be functional anymore def failTheBuild(String message) { def messageColor = "\u001B[32m" def messageColorReset = "\u001B[0m" currentBuild.result = "FAILURE" echo messageColor + message + messageColorReset } def getTestReportDestroyRunList(start, threads, platforms) { def runList = [:] for (int i = start; i < platforms.size() && i < start + threads; i++) { runList[platforms[i]] = executeTestReportDestroy(platforms[i]) } return runList } def executeTestReportDestroy(platform) { return { def status = false echo "VAGRANTPROV starting: ${System.currentTimeMillis()}" status = vagrant(platform, 5) // try 5 times if (status) { echo "Provisioning of ${platform} was successful. Running tests." if (os == "linux") { reportLinuxInfo(platform) } else { sleep 60 } echo "RAKERSPEC starting: ${System.currentTimeMillis()}" test(platform) echo "RAKERSPEC ending: ${System.currentTimeMillis()}" report(platform) destroy(platform) } else { echo "Provisioning of ${platform} FAILED. Aborting tests." } } } // It will try to perform setup and provisioning maxFails times, if it fails more than that, then it fails the setup process def vagrant(platform, maxFails){ def status = false for (int tries = 1; tries <= maxFails; tries++) { echo "Setup and provisioning attempt no. ${tries}" status = vagrantUpdate(platform) if (status) { status = vagrantUp(platform) } else { sh "vagrant box remove '${platform}'"; continue } if (status) { status = vagrantProvision(platform) // win7 and win2k8 platforms require to reboot, this sleep is to // avoid issues related to vm not restarting before the flow continues if (status && (platform.contains('win7') || platform.contains('win2k8'))) { sleep 65 } } else { vagrantDestroy(platform); continue } if (status) { status = checkVMConnectivity(platform) } else { if (tries == maxFails) { //only effective during the last try if (saveImageOnProvisioningFailure.toLowerCase() == "true") { echo "We had a failure while provisioning and the 'SaveImageOnProvisioningFailure' tag is set. Not killing ${platform}" break } else { vagrantDestroy(platform); continue } } else { vagrantDestroy(platform); continue } } if (status) { return status } else { echo "Setup and provisioning attempt no. ${tries} failed" vagrantDestroy(platform) } } // attempt 3 times to provision echo "Provisioning failed" reportErrorJunit(platform, "vagrant up ${platform} failed ${maxFails} times.", "VagrantProvisioning") sh "ls -1 PlatformFailedHighlightBlueOceanNode" // dummy command that returns a status code != 0 } def vagrantUpdate(platform) { def statusCode = sh(script: "vagrant box update '${platform}'", returnStatus: true) if (statusCode != 0) { echo "Failed to update the vagrant box" } return (statusCode == 0) } def vagrantDestroy(platform) { try { sh "vagrant destroy -f '${platform}'" } catch(err) { sh "vagrant destroy -f '${platform}' || true" } } def vagrantSubCommand(platform, command) { def subcommand = "" if (command == "up") { subcommand = "up '${platform}' --no-provision --provider vsphere" } else if (command == "provision") { subcommand = "provision '${platform}'" } else { return false } def statusCode = -1 try { timeout(time: 30, unit: 'MINUTES') { withCredentials([string(credentialsId: 'SRVC_Artif_Jenkins_R', variable: 'SERVICE_PASSWORD')]) { if (os == "linux") { statusCode = sh( // params.InstallEG is a string parameter. Any value other than 'true' is considered false script: "SERVICE_PASSWORD=\"$SERVICE_PASSWORD\" INSTALLEG=\"${params.InstallEG}\" INSTALLDKMS=\"${params.InstallDKMS}\" SELINUX=\"${params.SELinux}\" BRANCH=\"${params.RepoBranch.trim()}\" vagrant ${subcommand}", returnStatus: true ) } else { statusCode = sh( script: "SERVICE_PASSWORD=\"$SERVICE_PASSWORD\" INSTALLEG=\"${params.InstallEG}\" BRANCH=\"${params.RepoBranch.trim()}\" vagrant ${subcommand}", returnStatus: true ) } } } } catch(err) { echo "Virtual Machine provisioning exceeded 30 minutes" } return (statusCode == 0) } // Create virtual machine def vagrantUp(platform) { def status = vagrantSubCommand(platform, "up") if (!status) { echo "Failed to create the Virtual Machine" } return status } // Provision the Virtual Machine, timeout for provisioning attempt is set to 30 minutes def vagrantProvision(platform) { def status = vagrantSubCommand(platform, "provision") if (!status) { echo "Failed to provision the Virtual Machine" } return status } def report(platform) { // Check if we can connect to the platform. If not, abort if (!checkVMConnectivity(platform)) { reportErrorJunit(platform, "Attempting to report, connectivity lost to ${platform}", "RakeReportStep") return } def vagrantData = "${os == 'linux' ? '/vagrant_data' : '/cygdrive/c/vagrant_data'}" if (params.AxonAgentTest != null && params.AxonAgentTest.equalsIgnoreCase("true")) { transferFilesFromVM(platform, "${vagrantData}/${params.Repo}/accept/reports/*", "data/junit_${platform}") } else { def testFiles = findRemoteFiles(platform, "${vagrantData}/${params.Repo}/", "TEST*.xml") def specFiles = findRemoteFiles(platform, "${vagrantData}/${params.Repo}/", "*spec*.xml") def resultFiles = testFiles + ' ' + specFiles if (!resultFiles.trim().isEmpty()) { retryRemoteSSHCommand(platform, "mkdir -p ${vagrantData}/reports", 5, 30) retryRemoteSSHCommand(platform, "cd ${vagrantData}/${params.Repo}/accept && cp -f ${resultFiles} ${vagrantData}/reports/", 3, 120) transferFilesFromVM(platform, "${vagrantData}/reports/*", "data/junit_${platform}") } else { echo "No test result files were found on ${platform}" } } // The twfim cucumber tests generate one .xml file per suite sh "find data/junit_${platform} -name *.xml -exec sed -i -e 's/classname=\"/classname=\"${platform}\\./g' {} \\;" echo "currentBuild.result before junit: ${currentBuild.result}" junit keepLongStdio: true, testResults: "data/junit_${platform}/*.xml" echo "currentBuild.result after junit: ${currentBuild.result}" gatherLogs(platform) } // Determine if tests failed on the node and gather logs def gatherLogs(platform) { // if set, KeepBuildArtifactsOnSuccess will ensure the generated system logs are saved even in the case of a successful build if (params.AxonAgentTest == null && (checkPlatformFailed(platform) || (params.KeepBuildArtifactsOnSuccess != null && params.KeepBuildArtifactsOnSuccess.equalsIgnoreCase("true")))) { def vagrantData = "${os == 'linux' ? '/vagrant_data' : '/cygdrive/c/vagrant_data'}" // This will only handle the logs from the plugin tests, copy the log files generated from the tests def logFiles = findRemoteFiles(platform, "${vagrantData}/${params.Repo}/accept/rakework", "*.log*") if (!logFiles.trim().isEmpty()) { retryRemoteSSHCommand(platform, "mkdir -p ${vagrantData}/logs", 3, 30) retryRemoteSSHCommand(platform, "cd ${vagrantData}/${params.Repo}/accept/rakework && cp -f ${logFiles} ${vagrantData}/logs/", 3, 180) if (os == "linux") { retryRemoteSSHCommand(platform, "cd ${vagrantData}/logs && sudo chmod +r *", 3, 120) } // compress the log files before transferring them back retryRemoteSSHCommand(platform, "cd ${vagrantData}/logs && tar -czf logs.tar.gz *log*", 3, 120) transferFilesFromVM(platform, "${vagrantData}/logs/*.tar.gz", "logs/${platform}_logs") } else { echo "No log files available on ${platform}" } } } def destroy(platform) { // Check if we can connect to the platform. If not, abort if (!checkVMConnectivity(platform)) { // NOTE (jchai): Should it automatically try to destroy the VM? vagrantDestroy(platform) sh "ls -1 PlatformFailedHighlightBlueOceanNode" // dummy command that returns a status code != 0 return } // Check if the node failed tests. If it didn't, destroy the VM if (checkPlatformFailed(platform)){ if ('true'.equalsIgnoreCase(params.SaveImageOnTestFailure)) { echo "We had a failure. Not killing ${platform}" // Query the test system for its IP Address. Assume that the ip address starts with 10. if (os == "linux") { def test_system_ip_addr = sh(returnStdout: true, script: "vagrant ssh '${platform}' -c \"ip address | grep 'inet 10' | sed -e 's/^.*inet //' -e 's/\\/.*\$//'\"").trim() } else { def test_system_ip_addr = sh(returnStdout: true, script: "vagrant ssh '${platform}' -c \"ipconfig | grep ' IPv4 Address. . . . . . . . . . . : 10' | sed -e 's/^.*IPv4 Address.*: //' -e 's/\\/.*\$//'\"").trim() } echo "IP Address: ${test_system_ip_addr}" sh "ls -1 PlatformFailedHighlightBlueOceanNode" // dummy command that returns a status code != 0 } else { echo "We had failures. Per configuration, killing ${platform}" vagrantDestroy(platform) sh "ls -1 PlatformFailedHighlightBlueOceanNode" // dummy command that returns a status code != 0 } } else { echo "No failures were found. Killing ${platform}" vagrantDestroy(platform) } }
We use cookies to provide and improve our services. By using our site, you consent to our Cookies Policy. Accept Learn more