diff --git a/vars/dataset.groovy b/vars/dataset.groovy new file mode 100644 index 0000000..730b580 --- /dev/null +++ b/vars/dataset.groovy @@ -0,0 +1,18 @@ +def delete(String datasetLabel) { + echo "Deleting dataset graphs from label ${datasetLabel}" + + configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) { + def config = readJSON(text: readFile(file: configfile)) + String PMD = config['pmd_api'] + String credentials = config['credentials'] + String baseURI = config['base_uri'] + + def draftset = drafter.findDraftset(PMD, credentials, env.JOB_NAME) // assume it already exists + + String datasetPath = util.slugise(datasetLabel) + String datasetGraph = "${baseURI}/graph/${datasetPath}" + String metadataGraph = "${datasetGraph}/metadata" + drafter.deleteGraph(PMD, credentials, draftset.id, metadataGraph) + drafter.deleteGraph(PMD, credentials, draftset.id, datasetGraph) + } +} diff --git a/vars/drafter.groovy b/vars/drafter.groovy index 00d4b26..64b467f 100644 --- a/vars/drafter.groovy +++ b/vars/drafter.groovy @@ -11,6 +11,18 @@ } } +def findDraftset(String baseUrl, String, credentials, String displayName) { + echo "Finding draftset with display name '${displayName}'" + + def draftset = drafter.listDraftsets(baseUrl, credentials, 'owned') + def draftset = drafts.find { it['display-name'] == displayName } + if (draftset) { + draftset + } else { + error "Can't find draftset with the display-name '${displayName}'" + } +} + def deleteDraftset(String baseUrl, String credentials, String id) { echo "Deleting draftset ${id}" def response = httpRequest(acceptType: 'APPLICATION_JSON', diff --git a/vars/publishDraftset.groovy b/vars/publishDraftset.groovy index 14a9223..1c92783 100644 --- a/vars/publishDraftset.groovy +++ b/vars/publishDraftset.groovy @@ -3,12 +3,11 @@ def config = readJSON(text: readFile(file: configfile)) String PMD = config['pmd_api'] String credentials = config['credentials'] - def drafts = drafter.listDraftsets(PMD, credentials, 'owned') - def jobDraft = drafts.find { it['display-name'] == env.JOB_NAME } + def jobDraft = drafter.findDraftset(PMD, credentials, env.JOB_NAME) if (jobDraft) { drafter.publishDraftset(PMD, credentials, jobDraft.id) } else { error "Expecting a draftset for this job." } } -} \ No newline at end of file +} diff --git a/vars/uploadCube.groovy b/vars/uploadCube.groovy index 6b209f0..c73ca8a 100644 --- a/vars/uploadCube.groovy +++ b/vars/uploadCube.groovy @@ -4,17 +4,14 @@ String PMD = config['pmd_api'] String credentials = config['credentials'] String PIPELINE = config['pipeline_api'] - def drafts = drafter.listDraftsets(PMD, credentials, 'owned') - def jobDraft = drafts.find { it['display-name'] == env.JOB_NAME } + def jobDraft = drafts.findDraftset(env.JOB_NAME) if (jobDraft) { drafter.deleteDraftset(PMD, credentials, jobDraft.id as String) } def newJobDraft = drafter.createDraftset(PMD, credentials, env.JOB_NAME as String) String datasetPath = util.slugise(datasetLabel) - drafter.deleteGraph(PMD, credentials, newJobDraft.id as String, - "http://gss-data.org.uk/graph/${datasetPath}/metadata") - drafter.deleteGraph(PMD, credentials, newJobDraft.id as String, - "http://gss-data.org.uk/graph/${datasetPath}") + dataset.delete(datasetLabel) + drafter.addData(PMD, credentials, newJobDraft.id as String, readFile("out/dataset.trig"), "application/trig;charset=UTF-8") obslist.each { obsfile -> diff --git a/vars/uploadDraftset.groovy b/vars/uploadDraftset.groovy index a51bcb2..e1721ff 100644 --- a/vars/uploadDraftset.groovy +++ b/vars/uploadDraftset.groovy @@ -20,8 +20,7 @@ } mapping = 'metadata/columns.csv' } - def drafts = drafter.listDraftsets(PMD, credentials, 'owned') - def jobDraft = drafts.find { it['display-name'] == env.JOB_NAME } + def jobDraft = drafts.findDraftset(PMD, credentials, env.JOB_NAME) if (jobDraft) { drafter.deleteDraftset(PMD, credentials, jobDraft.id) } @@ -29,10 +28,7 @@ if (!datasetPath) { datasetPath = util.slugise(datasetLabel) } - drafter.deleteGraph(PMD, credentials, newJobDraft.id, - "http://gss-data.org.uk/graph/${datasetPath}/metadata") - drafter.deleteGraph(PMD, credentials, newJobDraft.id, - "http://gss-data.org.uk/graph/${datasetPath}") + dataset.delete(datasetLabel) drafter.addData(PMD, credentials, newJobDraft.id, readFile("out/dataset.trig"), "application/trig;charset=UTF-8") diff --git a/vars/uploadTidy.groovy b/vars/uploadTidy.groovy index 5e0901f..61d3629 100644 --- a/vars/uploadTidy.groovy +++ b/vars/uploadTidy.groovy @@ -21,28 +21,22 @@ } mapping = 'metadata/columns.csv' } - def drafts = drafter.listDraftsets(PMD, credentials, 'owned') - def jobDraft = drafts.find { it['display-name'] == env.JOB_NAME } - if (jobDraft) { - drafter.deleteDraftset(PMD, credentials, jobDraft.id) + + def oldJobDraft = drafter.findDraftset(env.JOB_NAME) + if (oldJobDraft) { + drafter.deleteDraftset(PMD, credentials, oldJobDraft.id) } + def newJobDraft = drafter.createDraftset(PMD, credentials, env.JOB_NAME) - String datasetPath = util.slugise(env.JOB_NAME) - String datasetGraph = "${baseURI}/graph/${datasetPath}" - String metadataGraph = "${datasetGraph}/metadata" - drafter.deleteGraph(PMD, credentials, newJobDraft.id, metadataGraph) - drafter.deleteGraph(PMD, credentials, newJobDraft.id, datasetGraph) + + dataset.delete(env.JOB_NAME) if (oldLabel) { - echo "Deleting old graphs from label ${oldLabel}" - String oldDatasetPath = util.slugise(oldLabel) - String oldDatasetGraph = "${baseURI}/graph/${oldDatasetPath}" - String oldMetadataGraph = "${oldDatasetGraph}/metadata" - drafter.deleteGraph(PMD, credentials, newJobDraft.id, oldMetadataGraph) - drafter.deleteGraph(PMD, credentials, newJobDraft.id, oldDatasetGraph) + dataset.delete(oldLabel) } drafter.addData(PMD, credentials, newJobDraft.id, readFile("out/dataset.trig"), "application/trig;charset=UTF-8") + String datasetPath = util.slugise(env.JOB_NAME) csvs.each { csv -> echo "Uploading ${csv}" runPipeline("${PIPELINE}/ons-table2qb.core/data-cube/import",