def call(body) {
def pipelineParams = [:]
body.resolveStrategy = Closure.DELEGATE_FIRST
body.delegate = pipelineParams
body()
pipeline {
agent {
label 'master'
}
triggers {
upstream(upstreamProjects: "../Reference/${pipelineParams.refFamily}",
threshold: hudson.model.Result.SUCCESS)
}
stages {
stage('Clean') {
steps {
sh 'rm -rf out'
}
}
stage('Transform') {
agent {
docker {
image 'cloudfluff/databaker'
reuseNode true
}
}
steps {
script {
ansiColor('xterm') {
sh "jupyter-nbconvert --output-dir=out --ExecutePreprocessor.timeout=None --execute 'main.ipynb'"
}
}
}
}
stage('Validate CSV') {
agent {
docker {
image 'cloudfluff/csvlint'
reuseNode true
}
}
steps {
script {
ansiColor('xterm') {
sh "csvlint -s schema.json"
}
}
}
}
stage('Upload Tidy Data') {
steps {
script {
jobDraft.replace()
uploadTidy(['out/observations.csv'],
"https://ons-opendata.github.io/${pipelineParams.refFamily}/columns.csv")
}
}
}
stage('Test draft dataset') {
agent {
docker {
image 'cloudfluff/gdp-sparql-tests'
reuseNode true
}
}
steps {
script {
pmd = pmdConfig("pmd")
String draftId = pmd.drafter.findDraftset(env.JOB_NAME).id
String endpoint = pmd.drafter.getDraftsetEndpoint(draftId)
String dspath = util.slugise(env.JOB_NAME)
String dsgraph = "${pmd.config.base_uri}/graph/${dspath}"
withCredentials([usernamePassword(credentialsId: pmd.config.credentials, usernameVariable: 'USER', passwordVariable: 'PASS')]) {
sh "sparql-test-runner -t /usr/local/tests -s ${endpoint}?union-with-live=true -a '${USER}:${PASS}' -p \"dsgraph=<${dsgraph}>\""
}
}
}
}
stage('Publish') {
steps {
script {
jobDraft.publish()
}
}
}
}
post {
always {
script {
archiveArtifacts artifacts: 'out/*', excludes: 'out/*.html'
junit 'reports/**/*.xml'
publishHTML([
allowMissing: true, alwaysLinkToLastBuild: true, keepAll: true,
reportDir: 'out', reportFiles: 'main.html',
reportName: 'Transform'])
if (pipelineParams.containsKey('trelloCard')) {
updateCard pipelineParams.trelloCard
}
}
}
success {
build job: '../GDP-tests', wait: false
}
}
}
}