pipeline { agent { label 'master' } triggers{ cron('H 2 * * 0') } stages { stage('Clean') { steps { sh 'rm -rf out' sh 'mkdir out' } } stage('Family scope report') { agent { docker { image 'gsscogs/databaker' reuseNode true alwaysPull true } } steps { sh 'jupyter-nbconvert --output-dir=out --execute family-scope/family-scope.ipynb' sh 'jupyter-nbconvert --output-dir=out --execute family-scope/geo-scope.ipynb' sh 'cp family-scope/*.html out/' } } stage('Dataset size report') { agent { docker { image 'gsscogs/databaker' reuseNode true alwaysPull true } } steps { sh 'cd dataset-stats && jupyter-nbconvert --to python --stdout size.ipynb | ipython' sh 'cp dataset-stats/dataset-stats.html out/' } } stage('Combined report') { agent { docker { image 'gsscogs/databaker' reuseNode true alwaysPull true } } environment { AIRTABLE_TOKEN = credentials('airtable-token-alex') AIRTABLE_BASE = 'appb66460atpZjzMq' } steps { sh 'jupytext --to notebook combined/data_report.py' sh 'jupyter-nbconvert --output-dir=out --ExecutePreprocessor.timeout=None --execute combined/data_report.ipynb' } } } post { always { publishHTML([ allowMissing: true, alwaysLinkToLastBuild: true, keepAll: true, reportDir: 'out', reportFiles: 'dataset-dimensions.html,dataset-dimensions-geo.html,dataset-stats.html', reportName: 'Dataset Reports', reportTitles: 'Family Scope,Geo Scope,Size']) archiveArtifacts 'out/*.html' } } }