def call(csvs, String mapping=null, String oldLabel=null) { configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) { def config = readJSON(text: readFile(file: configfile)) String PMD = config['pmd_api'] String credentials = config['credentials'] String PIPELINE = config['pipeline_api'] String baseURI = config['base_uri'] if (!mapping) { if (fileExists('metadata/columns.csv')) { mapping = 'metadata/columns.csv' } else { mapping = config['default_mapping'] } } if (mapping.startsWith('http')) { def response = httpRequest( httpMode: 'GET', url: mapping) dir ('metadata') { writeFile file: 'columns.csv', text: response.content } mapping = 'metadata/columns.csv' } def drafts = drafter.listDraftsets(PMD, credentials, 'owned') def jobDraft = drafts.find { it['display-name'] == env.JOB_NAME } if (jobDraft) { drafter.deleteDraftset(PMD, credentials, jobDraft.id) } def newJobDraft = drafter.createDraftset(PMD, credentials, env.JOB_NAME) String datasetPath = util.slugise(env.JOB_NAME) String datasetGraph = "${baseURI}/graph/${datasetPath}" String metadataGraph = "${datasetGraph}/metadata" drafter.deleteGraph(PMD, credentials, newJobDraft.id, metadataGraph) drafter.deleteGraph(PMD, credentials, newJobDraft.id, datasetGraph) if (oldLabel) { echo "Deleting old graphs from label ${oldLabel}" String oldDatasetPath = util.slugise(oldLabel) String oldDatasetGraph = "${baseURI}/graph/${oldDatasetPath}" String oldMetadataGraph = "${oldDatasetGraph}/metadata" drafter.deleteGraph(PMD, credentials, newJobDraft.id, oldMetadataGraph) drafter.deleteGraph(PMD, credentials, newJobDraft.id, oldDatasetGraph) } drafter.addData(PMD, credentials, newJobDraft.id, readFile("out/dataset.trig"), "application/trig;charset=UTF-8") csvs.each { csv -> echo "Uploading ${csv}" runPipeline("${PIPELINE}/ons-table2qb.core/data-cube/import", newJobDraft.id, credentials, [ [name: 'observations-csv', file: [name: csv, type: 'text/csv;charset=UTF-8']], [name: 'dataset-name', value: ''], [name: 'dataset-slug', value: datasetPath], [name: 'columns-csv', file: [name: mapping, type: 'text/csv;charset=UTF-8']] ]) } } }