diff --git a/src/uk/org/floop/jenkins_pmd/Drafter.groovy b/src/uk/org/floop/jenkins_pmd/Drafter.groovy
index 735ab48..973250c 100644
--- a/src/uk/org/floop/jenkins_pmd/Drafter.groovy
+++ b/src/uk/org/floop/jenkins_pmd/Drafter.groovy
@@ -2,13 +2,17 @@
import groovy.json.JsonSlurper
import org.apache.http.HttpHost
+import org.apache.http.HttpResponse
import org.apache.http.client.fluent.Executor
import org.apache.http.client.fluent.Request
+import org.apache.http.entity.ContentType
+import org.apache.http.util.EntityUtils
class Drafter implements Serializable {
- private Executor exec
+ private PMD pmd
private URI apiBase
private HttpHost host
+ private String user, pass
enum Include {
ALL("all"), OWNED("owned"), CLAIMABLE("claimable")
@@ -18,22 +22,157 @@
}
}
- Drafter(String api, String user, String pass) {
- this.apiBase = new URI(api)
- this.host = new HttpHost(apiBase.getHost())
- exec = Executor.newInstance()
- .auth(this.host, user, pass)
- .authPreemptive(this.host)
+ Drafter(PMD pmd, String user, String pass) {
+ this.pmd = pmd
+ this.apiBase = new URI(pmd.config.pmd_api)
+ this.host = new HttpHost(apiBase.getHost(), (apiBase.getPort() != -1) ? apiBase.getPort() :
+ apiBase.getScheme() == "http" ? 80 : 443)
+ this.user = user
+ this.pass = pass
+ }
+
+ private Executor getExec() {
+ Executor.newInstance()
+ .auth(this.host, this.user, this.pass)
+ .authPreemptive(this.host)
}
def listDraftsets(Include include=Include.ALL) {
def js = new JsonSlurper()
- String path = (include == Include.ALL) ? "/v1/draftsets" : "/v1/draftsets?include=" + Include.value
+ String path = (include == Include.ALL) ? "/v1/draftsets" : "/v1/draftsets?include=" + include.value
def response = js.parse(
- exec.execute(
+ getExec().execute(
Request.Get(apiBase.resolve(path))
+ .addHeader("Accept", "application/json")
+ .userAgent(PMDConfig.UA)
).returnContent().asStream()
)
response
}
-}
\ No newline at end of file
+
+ private static String errorMsg(HttpResponse response) {
+ EntityUtils.consume(response.getEntity())
+ "${response.getStatusLine()} : ${EntityUtils.toString(response.getEntity())}"
+ }
+
+ def createDraftset(String label) {
+ String displayName = URLEncoder.encode(label, "UTF-8")
+ String path = "/v1/draftsets?display-name=${displayName}"
+ HttpResponse response = getExec().execute(
+ Request.Post(apiBase.resolve(path))
+ .addHeader("Accept", "application/json")
+ .userAgent(PMDConfig.UA)
+ ).returnResponse()
+ if (response.getStatusLine().statusCode == 200) {
+ return new JsonSlurper().parse(EntityUtils.toByteArray(response.getEntity()))
+ } else {
+ throw new DrafterException("Problem creating draftset ${errorMsg(response)}")
+ }
+ }
+
+ def deleteGraph(String draftsetId, String graph) {
+ String encGraph = URLEncoder.encode(graph, "UTF-8")
+ String path = "/v1/draftset/${draftsetId}/graph?graph=${encGraph}&silent=true"
+ HttpResponse response = getExec().execute(
+ Request.Delete(apiBase.resolve(path))
+ .addHeader("Accept", "application/json")
+ .userAgent(PMDConfig.UA)
+ ).returnResponse()
+ if (response.getStatusLine().statusCode == 200) {
+ return new JsonSlurper().parse(EntityUtils.toByteArray(response.getEntity()))
+ } else {
+ throw new DrafterException("Problem deleting graph ${errorMsg(response)}")
+ }
+ }
+
+ def deleteDraftset(String draftsetId) {
+ String path = "/v1/draftset/${draftsetId}"
+ HttpResponse response = getExec().execute(
+ Request.Delete(apiBase.resolve(path))
+ .addHeader("Accept", "application/json")
+ .userAgent(PMDConfig.UA)
+ ).returnResponse()
+ if (response.getStatusLine().statusCode == 202) {
+ def jobObj = new JsonSlurper().parse(EntityUtils.toByteArray(response.getEntity()))
+ waitForJob(apiBase.resolve(jobObj['finished-job'] as String), jobObj['restart-id'] as String)
+ } else {
+ throw new DrafterException("Problem deleting draftset ${jobObj['message']}")
+ }
+ }
+
+ def waitForJob(URI finishedJob, String restartId) {
+ Executor exec = getExec()
+ while (true) {
+ HttpResponse jobResponse = exec.execute(
+ Request.Get(finishedJob)
+ .setHeader("Accept", "application/json")
+ .userAgent(PMDConfig.UA)
+ ).returnResponse()
+ int status = jobResponse.getStatusLine().statusCode
+ def jobObj
+ try {
+ jobObj = new JsonSlurper().parse(EntityUtils.toByteArray(jobResponse.getEntity()))
+ } catch(e) {
+ throw new DrafterException("Failed waiting for job ${errorMsg(jobResponse)}.\n${e}")
+ }
+ if (status == 404) {
+ if (jobObj['restart-id'] != restartId) {
+ throw new DrafterException("Failed waiting for job to finish, no/different restart-id ${errorMsg(jobResponse)}")
+ } else {
+ sleep(10000)
+ }
+ } else if (status == 200) {
+ if (jobObj['restart-id'] != restartId) {
+ throw new DrafterException("Failed waiting for job to finish, restart-id is different.")
+ } else if (jobObj['type'] != 'ok') {
+ throw new DrafterException("Pipeline error in ${jobObj.details?.pipeline?.name}. ${jobObj.message}")
+ } else {
+ break
+ }
+ } else {
+ throw new DrafterException("Unexpected response waiting for job to complete: ${errorMsg(jobResponse)}")
+ }
+ }
+ }
+
+ def addData(String draftId, String fileName, String mimeType, String encoding, String graph=null) {
+ String path = "/v1/draftset/${draftId}/data"
+ if (graph) {
+ String encGraph = URLEncoder.encode(graph, "UTF-8")
+ path = path + "?graph=${encGraph}"
+ }
+ HttpResponse response = getExec().execute(
+ Request.Put(apiBase.resolve(path))
+ .addHeader("Accept", "application/json")
+ .userAgent(PMDConfig.UA)
+ .bodyFile(new File(fileName), ContentType.create(mimeType, encoding))
+ ).returnResponse()
+ if (response.getStatusLine().statusCode == 202) {
+ def jobObj = new JsonSlurper().parse(EntityUtils.toByteArray(response.getEntity()))
+ waitForJob(apiBase.resolve(jobObj['finished-job'] as String), jobObj['restart-id'] as String)
+ } else {
+ throw new DrafterException("Problem adding data ${errorMsg(response)}")
+ }
+ }
+
+ def findDraftset(String displayName) {
+ def drafts = listDraftsets(Include.OWNED)
+ def draftset = drafts.find { it['display-name'] == displayName }
+ if (draftset) {
+ draftset
+ } else {
+ throw new DrafterException("Can't find draftset with the display-name '${displayName}'")
+ }
+
+ }
+
+}
+
+class DrafterException extends Throwable {
+ String message
+
+ DrafterException(String message) {
+ this.message = message
+ }
+
+}
diff --git a/src/uk/org/floop/jenkins_pmd/PMD.groovy b/src/uk/org/floop/jenkins_pmd/PMD.groovy
new file mode 100644
index 0000000..d5d6872
--- /dev/null
+++ b/src/uk/org/floop/jenkins_pmd/PMD.groovy
@@ -0,0 +1,13 @@
+package uk.org.floop.jenkins_pmd
+
+class PMD implements Serializable {
+ PMDConfig config
+ Drafter drafter
+ Pipelines pipelines
+
+ PMD(PMDConfig config, String user, String pass) {
+ this.config = config
+ this.drafter = new Drafter(this, user, pass)
+ this.pipelines = new Pipelines(this, user, pass)
+ }
+}
diff --git a/src/uk/org/floop/jenkins_pmd/PMDConfig.groovy b/src/uk/org/floop/jenkins_pmd/PMDConfig.groovy
new file mode 100644
index 0000000..f49f5c7
--- /dev/null
+++ b/src/uk/org/floop/jenkins_pmd/PMDConfig.groovy
@@ -0,0 +1,11 @@
+package uk.org.floop.jenkins_pmd
+
+class PMDConfig implements Serializable {
+ static final String UA = "uk.org.floop.jenkins_pmd/0.1"
+ String pmd_api
+ String credentials
+ String pipeline_api
+ String default_mapping
+ String base_uri
+
+}
diff --git a/src/uk/org/floop/jenkins_pmd/Pipelines.groovy b/src/uk/org/floop/jenkins_pmd/Pipelines.groovy
new file mode 100644
index 0000000..2ba085e
--- /dev/null
+++ b/src/uk/org/floop/jenkins_pmd/Pipelines.groovy
@@ -0,0 +1,92 @@
+package uk.org.floop.jenkins_pmd
+
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+import org.apache.http.HttpHost
+import org.apache.http.HttpResponse
+import org.apache.http.client.fluent.Executor
+import org.apache.http.client.fluent.Request
+import org.apache.http.entity.ContentType
+import org.apache.http.entity.mime.MultipartEntityBuilder
+import org.apache.http.util.EntityUtils
+
+class Pipelines implements Serializable {
+ private PMD pmd
+ private URI apiBase
+ private HttpHost host
+ private String user, pass, basicAuth
+
+ Pipelines(PMD pmd, String user, String pass) {
+ this.pmd = pmd
+ this.apiBase = new URI(pmd.config.pipeline_api)
+ this.host = new HttpHost(apiBase.getHost(), (apiBase.getPort() != -1) ? apiBase.getPort() :
+ apiBase.getScheme() == "http" ? 80 : 443)
+ this.user = user
+ this.pass = pass
+ this.basicAuth = "${user}:${pass}".bytes.encodeBase64()
+ }
+
+ private Executor getExec() {
+ Executor.newInstance()
+ .auth(this.host, this.user, this.pass)
+ .authPreemptive(this.host)
+ }
+
+ private static String errorMsg(HttpResponse response) {
+ "${response.getStatusLine()} : ${EntityUtils.toString(response.getEntity())}"
+ }
+
+ def dataCube(String draftsetId, String observationsFilename, String datasetName, String datasetPath, String mapping) {
+ String path = "/v1/pipelines/ons-table2qb.core/data-cube/import"
+ MultipartEntityBuilder body = MultipartEntityBuilder.create()
+ body.addTextBody('__endpoint-type', 'grafter-server.destination/draftset-update')
+ body.addTextBody('__endpoint', JsonOutput.toJson([
+ url: "http://localhost:3001/v1/draftset/${draftsetId}/data",
+ headers: [Authorization: "Basic ${basicAuth}"]
+ ]))
+ body.addBinaryBody(
+ 'observations-csv',
+ new FileInputStream(observationsFilename),
+ ContentType.create('text/csv', 'UTF-8'),
+ observationsFilename
+ )
+ body.addTextBody('dataset-name', datasetName)
+ body.addTextBody('dataset-slug', datasetPath)
+ InputStream mappingStream
+ if (mapping.startsWith('http')) {
+ mappingStream = Request
+ .Get(mapping)
+ .userAgent(PMDConfig.UA)
+ .addHeader('Accept', 'text/csv')
+ .execute().returnContent().asStream()
+ } else {
+ mappingStream = new FileInputStream(mapping)
+ }
+ body.addBinaryBody(
+ 'columns-csv',
+ mappingStream,
+ ContentType.create('text/csv', 'UTF-8'),
+ mapping
+ )
+ HttpResponse response = getExec().execute(
+ Request.Post(apiBase.resolve(path))
+ .addHeader("Accept", "application/json")
+ .userAgent(PMDConfig.UA)
+ .body(body.build())
+ ).returnResponse()
+ if (response.getStatusLine().statusCode == 202) {
+ def jobObj = new JsonSlurper().parse(EntityUtils.toByteArray(response.getEntity()))
+ this.pmd.drafter.waitForJob(apiBase.resolve(jobObj['finished-job'] as String), jobObj['restart-id'] as String)
+ } else {
+ throw new PipelineException("Failed pipeline import: ${errorMsg(response)}")
+ }
+ }
+}
+
+class PipelineException extends Throwable {
+ String message
+
+ PipelineException(String message) {
+ this.message = message
+ }
+}
\ No newline at end of file
diff --git a/test/integration/groovy/uk/org/floop/jenkins_pmd/DrafterTests.groovy b/test/integration/groovy/uk/org/floop/jenkins_pmd/DrafterTests.groovy
index a82d280..cd48a59 100644
--- a/test/integration/groovy/uk/org/floop/jenkins_pmd/DrafterTests.groovy
+++ b/test/integration/groovy/uk/org/floop/jenkins_pmd/DrafterTests.groovy
@@ -6,6 +6,7 @@
import com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl
import com.github.tomakehurst.wiremock.core.WireMockConfiguration
import com.github.tomakehurst.wiremock.junit.WireMockClassRule
+import com.github.tomakehurst.wiremock.stubbing.Scenario
import org.jenkinsci.lib.configprovider.ConfigProvider
import org.jenkinsci.plugins.configfiles.ConfigFileStore
import org.jenkinsci.plugins.configfiles.GlobalConfigFiles
@@ -28,7 +29,8 @@
@ClassRule
public static WireMockClassRule wireMockRule = new WireMockClassRule(WireMockConfiguration.options()
- .dynamicPort()
+ //.dynamicPort()
+ .port(8123)
.usingFilesUnderClasspath("test/resources")
)
@@ -95,10 +97,15 @@
@Test
void "listDraftsets"() {
stubFor(get(urlMatching("/v1/draftsets.*"))
- .willReturn(ok().withBodyFile("listDraftsets.json")))
+ .withHeader("Accept", equalTo("application/json"))
+ .withBasicAuth("admin", "admin")
+ .willReturn(ok()
+ .withBodyFile("listDraftsets.json")
+ .withHeader("Content-Type", "application/json")))
final CpsFlowDefinition flow = new CpsFlowDefinition('''
node {
- echo drafter("pmd").listDraftsets()[0].id
+ def pmd = pmdConfig("pmd")
+ echo pmd.drafter.listDraftsets()[0].id
}'''.stripIndent(), true)
final WorkflowJob workflowJob = rule.createProject(WorkflowJob, 'project')
workflowJob.definition = flow
@@ -108,4 +115,81 @@
rule.assertLogContains('de305d54-75b4-431b-adb2-eb6b9e546014', firstResult)
}
+ @Test
+ void "uploadTidy"() {
+ stubFor(post("/v1/draftsets?display-name=project")
+ .withHeader("Accept", equalTo("application/json"))
+ .withBasicAuth("admin", "admin")
+ .willReturn(seeOther("/v1/draftset/4e376c57-6816-404a-8945-94849299f2a0")))
+ stubFor(get(urlMatching("/v1/draftsets.*"))
+ .withHeader("Accept", equalTo("application/json"))
+ .withBasicAuth("admin", "admin")
+ .willReturn(ok()
+ .withBodyFile("listDraftsets.json")
+ .withHeader("Content-Type", "application/json")))
+ stubFor(get("/v1/draftset/4e376c57-6816-404a-8945-94849299f2a0")
+ .withHeader("Accept", equalTo("application/json"))
+ .withBasicAuth("admin", "admin")
+ .willReturn(ok()
+ .withBodyFile("newDraftset.json")
+ .withHeader("Content-Type", "application/json")))
+ stubFor(delete("/v1/draftset/4e376c57-6816-404a-8945-94849299f2a0")
+ .withHeader("Accept", equalTo("application/json"))
+ .withBasicAuth("admin", "admin")
+ .willReturn(aResponse()
+ .withStatus(202)
+ .withBodyFile("deleteJob.json")
+ .withHeader("Content-Type", "application/json")))
+ stubFor(delete(urlMatching("/v1/draftset/4e376c57-6816-404a-8945-94849299f2a0/graph.*"))
+ .withHeader("Accept", equalTo("application/json"))
+ .withBasicAuth("admin", "admin")
+ .willReturn(aResponse()
+ .withStatus(200)
+ .withBodyFile("deleteGraph.json")
+ .withHeader("Content-Type", "application/json")))
+ stubFor(put("/v1/draftset/4e376c57-6816-404a-8945-94849299f2a0/data")
+ .withHeader("Accept", equalTo("application/json"))
+ .withBasicAuth("admin", "admin")
+ .willReturn(aResponse()
+ .withStatus(202)
+ .withBodyFile("addDataJob.json")
+ .withHeader("Content-Type", "application/json")))
+ stubFor(get("/v1/status/finished-jobs/2c4111e5-a299-4526-8327-bad5996de400").inScenario("Delete draftset")
+ .whenScenarioStateIs(Scenario.STARTED)
+ .withHeader("Accept", equalTo("application/json"))
+ .withBasicAuth("admin", "admin")
+ .willReturn(aResponse().withStatus(404).withBodyFile('notFinishedJob.json'))
+ .willSetStateTo("Finished"))
+ stubFor(get("/v1/status/finished-jobs/2c4111e5-a299-4526-8327-bad5996de400").inScenario("Delete draftset")
+ .whenScenarioStateIs("Finished")
+ .withHeader("Accept", equalTo("application/json"))
+ .withBasicAuth("admin", "admin")
+ .willReturn(ok()
+ .withBodyFile("finishedJobOk.json")))
+ stubFor(get("/columns.csv").willReturn(ok().withBodyFile('columns.csv')))
+ stubFor(post("/v1/pipelines/ons-table2qb.core/data-cube/import")
+ .withHeader('Accept', equalTo('application/json'))
+ .withBasicAuth('admin', 'admin')
+ .willReturn(aResponse().withStatus(202).withBodyFile('cubeImportJob.json')))
+ stubFor(get('/status/finished-jobs/4fc9ad42-f964-4f56-a1ab-a00bd622b84c')
+ .withHeader('Accept', equalTo('application/json'))
+ .withBasicAuth('admin', 'admin')
+ .willReturn(ok().withBodyFile('finishedImportJobOk.json')))
+
+ final CpsFlowDefinition flow = new CpsFlowDefinition("""
+ node {
+ dir('out') {
+ writeFile file:'dataset.trig', text:'dummy:data'
+ writeFile file:'observations.csv', text:'Dummy,CSV'
+ }
+ jobDraft.replace()
+ uploadTidy(['out/observations.csv'],
+ '${wireMockRule.baseUrl()}/columns.csv')
+ }""".stripIndent(), true)
+ final WorkflowJob workflowJob = rule.createProject(WorkflowJob, 'project')
+ workflowJob.definition = flow
+
+ final WorkflowRun firstResult = rule.buildAndAssertSuccess(workflowJob)
+ }
+
}
diff --git a/test/resources/__files/addDataJob.json b/test/resources/__files/addDataJob.json
new file mode 100644
index 0000000..b1e0a53
--- /dev/null
+++ b/test/resources/__files/addDataJob.json
@@ -0,0 +1,5 @@
+{
+ "type": "ok",
+ "restart-id": "b1b26596-2dca-4e52-883c-7fdcb8b4be97",
+ "finished-job": "/v1/status/finished-jobs/2c4111e5-a299-4526-8327-bad5996de400"
+}
\ No newline at end of file
diff --git a/test/resources/__files/columns.csv b/test/resources/__files/columns.csv
new file mode 100644
index 0000000..74c5416
--- /dev/null
+++ b/test/resources/__files/columns.csv
@@ -0,0 +1,31 @@
+title,name,component_attachment,property_template,value_template,datatype,value_transformation
+Age,age,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#age,http://gss-data.org.uk/def/concept/ages/{age},string,slugize
+Area of Destination or Origin,area_of_destination_or_origin,qb:dimension,http://gss-data.org.uk/def/dimension/area-of-destination-or-origin,http://statistics.data.gov.uk/id/statistical-geography/{area_of_destination_or_origin},string,
+CI,ci,qb:attribute,http://gss-data.org.uk/def/attribute/ci,,number,
+Citizenship,citizenship,qb:dimension,http://gss-data.org.uk/def/dimension/citizenship,http://gss-data.org.uk/def/concept/citizenship/{citizenship},string,slugize
+Count,count,qb:measure,http://gss-data.org.uk/def/measure/count,,number,
+Country of Residence,country_of_residence,qb:dimension,http://gss-data.org.uk/def/dimension/residence,http://gss-data.org.uk/def/concept/country-of-residence/{country_of_residence},string,slugize
+Flow,migration,qb:dimension,http://gss-data.org.uk/def/dimension/migration,http://gss-data.org.uk/def/concept/migration-directions/{migration},string,slugize
+Geography,geography,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#refArea,http://statistics.data.gov.uk/id/statistical-geography/{geography},string,
+Measure Type,measure_type,qb:dimension,http://purl.org/linked-data/cube#measureType,http://gss-data.org.uk/def/measure/{measure_type},string,slugize
+Mid Year,mid_year,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#refPeriod,http://reference.data.gov.uk/id/gregorian-interval/{+mid_year},string,
+Migration,migration_type,qb:dimension,http://gss-data.org.uk/def/dimension/migration-type,http://gss-data.org.uk/def/concept/migration-types/{migration_type},string,slugize
+Note,note,qb:attribute,http://purl.org/linked-data/sdmx/2009/attribute#comment,,string,
+Occupation,occupation,qb:dimension,http://gss-data.org.uk/def/dimension/occupation,http://gss-data.org.uk/def/concept/occupations/{occupation},string,slugize
+Registration Geography,registration_geography,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#refArea,http://gss-data.org.uk/def/concept/registration-geography/{registration_geography},string,slugize
+Period,period,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#refPeriod,http://reference.data.gov.uk/id/{+period},string,
+Reason for migration,reason_for_migration,qb:dimension,http://gss-data.org.uk/def/dimension/reason-for-migration,http://gss-data.org.uk/def/concept/reasons-for-migration/{reason_for_migration},string,slugize
+Revision,revision,qb:dimension,http://gss-data.org.uk/def/dimension/revision,http://gss-data.org.uk/def/concept/revisions/{revision},string,slugize
+Sex,sex,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#sex,http://purl.org/linked-data/sdmx/2009/code#sex-{sex},string,
+Unit,unit,qb:attribute,http://purl.org/linked-data/sdmx/2009/attribute#unitMeasure,http://gss-data.org.uk/def/concept/measurement-units/{unit},string,unitize
+Value,value,,http://gss-data.org.uk/def/measure/{measure_type},,number,
+Year,year,qb:dimension,http://purl.org/linked-data/sdmx/2009/dimension#refPeriod,http://reference.data.gov.uk/id/year/{year},string,
+Migration Flow,migration_flow,qb:dimension,http://gss-data.org.uk/def/dimension/migration,http://gss-data.org.uk/def/concept/migration-directions/{migration_flow},string,slugize
+Population Change Component,population_change_component,qb:dimension,http://gss-data.org.uk/def/dimension/population-change-component,http://gss-data.org.uk/def/concept/population-change-component/{population_change_component},string,slugize
+IPS Citizenship,ips_citizenship,qb:dimension,http://gss-data.org.uk/def/dimension/citizenship,http://gss-data.org.uk/def/concept/ips-citizenship/{ips_citizenship},string,
+LTIM Citizenship,ltim_citizenship,qb:dimension,http://gss-data.org.uk/def/dimension/citizenship,http://gss-data.org.uk/def/concept/ltim-citizenship/{ltim_citizenship},string,slugize
+Applicant type,applicant_type,qb:dimension,http://gss-data.org.uk/def/dimension/applicant-type,http://gss-data.org.uk/def/concept/ho-applicant-types/{applicant_type},string,slugize
+Application category,application_category,qb:dimension,http://gss-data.org.uk/def/dimension/application-category,http://gss-data.org.uk/def/concept/ho-application-category/{application_category},string,slugize
+Application type,application_type,qb:dimension,http://gss-data.org.uk/def/dimension/application-type,http://gss-data.org.uk/def/concept/ho-application-types/{application_type},string,slugize
+Country of nationality,ho_citizenship,qb:dimension,http://gss-data.org.uk/def/dimension/citizenship,http://gss-data.org.uk/def/concept/ho-citizenship/{ho_citizenship},string,slugize
+Resolution,resolution,qb:dimension,http://gss-data.org.uk/def/dimension/resolution,http://gss-data.org.uk/def/concept/ho-application-resolutions/{resolution},string,slugize
diff --git a/test/resources/__files/cubeImportJob.json b/test/resources/__files/cubeImportJob.json
new file mode 100644
index 0000000..60ed849
--- /dev/null
+++ b/test/resources/__files/cubeImportJob.json
@@ -0,0 +1 @@
+{"type":"ok","finished-job":"/status/finished-jobs/4fc9ad42-f964-4f56-a1ab-a00bd622b84c","restart-id":"a2a8f11d-96fd-466e-adaf-85e8d07d4956"}
\ No newline at end of file
diff --git a/test/resources/__files/deleteGraph.json b/test/resources/__files/deleteGraph.json
new file mode 100644
index 0000000..6ae0adb
--- /dev/null
+++ b/test/resources/__files/deleteGraph.json
@@ -0,0 +1,17 @@
+{
+ "id": "4e376c57-6816-404a-8945-94849299f2a0",
+ "display-name": "project",
+ "description": "Jenkins dataset pipeline",
+ "changes": {
+ "http://opendatacommunities.org/graph/homelessness/households-accommodated/temporary-housing-types": {
+ "status": "updated"
+ },
+ "http://opendatacommunities.org/data/labour-force/employment-rate/employment-rate-by-age": {
+ "status": "deleted"
+ }
+ },
+ "updated-at": "2016-01-04T13:35:21.000Z",
+ "created-at": "2016-01-01T13:35:21.000Z",
+ "current-owner": "admin@opendatacommunities.org",
+ "submitted-by": "editor@opendatacommunities.org"
+}
\ No newline at end of file
diff --git a/test/resources/__files/deleteJob.json b/test/resources/__files/deleteJob.json
new file mode 100644
index 0000000..b1e0a53
--- /dev/null
+++ b/test/resources/__files/deleteJob.json
@@ -0,0 +1,5 @@
+{
+ "type": "ok",
+ "restart-id": "b1b26596-2dca-4e52-883c-7fdcb8b4be97",
+ "finished-job": "/v1/status/finished-jobs/2c4111e5-a299-4526-8327-bad5996de400"
+}
\ No newline at end of file
diff --git a/test/resources/__files/finishedImportJobOk.json b/test/resources/__files/finishedImportJobOk.json
new file mode 100644
index 0000000..a701e5d
--- /dev/null
+++ b/test/resources/__files/finishedImportJobOk.json
@@ -0,0 +1,4 @@
+{
+ "type": "ok",
+ "restart-id": "a2a8f11d-96fd-466e-adaf-85e8d07d4956"
+}
\ No newline at end of file
diff --git a/test/resources/__files/finishedJobOk.json b/test/resources/__files/finishedJobOk.json
new file mode 100644
index 0000000..cf3033a
--- /dev/null
+++ b/test/resources/__files/finishedJobOk.json
@@ -0,0 +1,4 @@
+{
+ "type": "ok",
+ "restart-id": "b1b26596-2dca-4e52-883c-7fdcb8b4be97"
+}
\ No newline at end of file
diff --git a/test/resources/__files/listDraftsets.json b/test/resources/__files/listDraftsets.json
index 6741754..2a2f4bf 100644
--- a/test/resources/__files/listDraftsets.json
+++ b/test/resources/__files/listDraftsets.json
@@ -15,5 +15,13 @@
"created-at": "2016-01-01T13:35:21.000Z",
"current-owner": "admin@opendatacommunities.org",
"submitted-by": "editor@opendatacommunities.org"
+ }, {
+ "id": "4e376c57-6816-404a-8945-94849299f2a0",
+ "display-name": "project",
+ "description": "Jenkins dataset pipeline",
+ "updated-at": "2016-01-04T13:35:21.000Z",
+ "created-at": "2016-01-01T13:35:21.000Z",
+ "current-owner": "admin@opendatacommunities.org",
+ "submitted-by": "editor@opendatacommunities.org"
}
]
\ No newline at end of file
diff --git a/test/resources/__files/newDraftset.json b/test/resources/__files/newDraftset.json
new file mode 100644
index 0000000..ffc4476
--- /dev/null
+++ b/test/resources/__files/newDraftset.json
@@ -0,0 +1,9 @@
+{
+"id": "4e376c57-6816-404a-8945-94849299f2a0",
+"display-name": "project",
+"description": "Jenkins dataset pipeline",
+"updated-at": "2016-01-04T13:35:21.000Z",
+"created-at": "2016-01-01T13:35:21.000Z",
+"current-owner": "admin@opendatacommunities.org",
+"submitted-by": "editor@opendatacommunities.org"
+}
\ No newline at end of file
diff --git a/test/resources/__files/notFinishedJob.json b/test/resources/__files/notFinishedJob.json
new file mode 100644
index 0000000..97e7b6e
--- /dev/null
+++ b/test/resources/__files/notFinishedJob.json
@@ -0,0 +1,4 @@
+{
+ "type": "waiting",
+ "restart-id": "b1b26596-2dca-4e52-883c-7fdcb8b4be97"
+}
\ No newline at end of file
diff --git a/vars/dataset.groovy b/vars/dataset.groovy
index 3329507..08224e5 100644
--- a/vars/dataset.groovy
+++ b/vars/dataset.groovy
@@ -1,17 +1,14 @@
+import uk.org.floop.jenkins_pmd.PMD
+
def delete(String datasetPath) {
echo "Deleting data/ meta graphs for path ${datasetPath}"
- configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) {
- def config = readJSON(text: readFile(file: configfile))
- String PMD = config['pmd_api']
- String credentials = config['credentials']
- String baseURI = config['base_uri']
+ PMD pmd = pmdConfig("pmd")
- def draftset = jobDraft.find() // assume it already exists
+ def draftset = jobDraft.find() // assume it already exists
- String datasetGraph = "${baseURI}/graph/${datasetPath}"
- String metadataGraph = "${datasetGraph}/metadata"
- drafter.deleteGraph(PMD, credentials, draftset.id, metadataGraph)
- drafter.deleteGraph(PMD, credentials, draftset.id, datasetGraph)
- }
+ String datasetGraph = "${pmd.config.base_uri}/graph/${datasetPath}"
+ String metadataGraph = "${datasetGraph}/metadata"
+ pmd.drafter.deleteGraph(draftset.id, metadataGraph)
+ pmd.drafter.deleteGraph(draftset.id, datasetGraph)
}
diff --git a/vars/drafter.groovy b/vars/drafter.groovy
index 40ef169..2857355 100644
--- a/vars/drafter.groovy
+++ b/vars/drafter.groovy
@@ -1,70 +1,42 @@
import uk.org.floop.jenkins_pmd.Drafter
+import uk.org.floop.jenkins_pmd.PMD
+import uk.org.floop.jenkins_pmd.PMDConfig
def call(String configId) {
configFileProvider([configFile(fileId: configId, variable: 'configfile')]) {
- def config = readJSON(text: readFile(file: configfile))
- String PMD = config['pmd_api']
- String credentials = config['credentials']
- withCredentials([usernamePassword(credentialsId: credentials, usernameVariable: 'USER', passwordVariable: 'PASS')]) {
- return new Drafter(PMD, USER, PASS)
+ PMDConfig config = new PMDConfig(readJSON(text: readFile(file: configfile)))
+ withCredentials([usernamePassword(credentialsId: config.credentials, usernameVariable: 'USER', passwordVariable: 'PASS')]) {
+ return new PMD(config, USER as String, PASS as String).getDrafter()
}
}
}
def listDraftsets(String baseUrl, String credentials, String include) {
echo "Listing draftsets..."
-/* def response = httpRequest(acceptType: 'APPLICATION_JSON',
- authentication: credentials,
- httpMode: 'GET',
- url: "${baseUrl}/v1/draftsets?include=${include}")
- if (response.status == 200) {
- return readJSON(text: response.content)
- } else {
- error "Problem listing draftsets ${response.status} : ${response.content}"
- } */
- return Drafter.listDraftsets()
+
+ PMD pmd = pmdConfig("pmd")
+ return pmd.drafter.listDraftsets()
}
def findDraftset(String baseUrl, String credentials, String displayName) {
echo "Finding draftset with display name '${displayName}'"
- def drafts = listDraftsets(baseUrl, credentials, 'owned')
- def draftset = drafts.find { it['display-name'] == displayName }
- if (draftset) {
- draftset
- } else {
- error "Can't find draftset with the display-name '${displayName}'"
- }
+ PMD pmd = pmdConfig("pmd")
+ pmd.drafter.findDraftset(displayName)
}
def deleteDraftset(String baseUrl, String credentials, String id) {
echo "Deleting draftset ${id}"
- def response = httpRequest(acceptType: 'APPLICATION_JSON',
- authentication: credentials,
- httpMode: 'DELETE',
- url: "${baseUrl}/v1/draftset/${id}")
- if (response.status == 202) {
- def job = readJSON(text: response.content)
- waitForJob(
- "${baseUrl}${job['finished-job']}" as String,
- credentials, job['restart-id'] as String)
- } else {
- error "Problem deleting draftset ${response.status} : ${response.content}"
- }
+
+ PMD pmd = pmdConfig("pmd")
+ pmd.drafter.deleteDraftset(id)
}
def createDraftset(String baseUrl, String credentials, String label) {
echo "Creating draftset ${label}"
- String displayName = java.net.URLEncoder.encode(label, "UTF-8")
- def response = httpRequest(acceptType: 'APPLICATION_JSON',
- authentication: credentials,
- httpMode: 'POST',
- url: "${baseUrl}/v1/draftsets?display-name=${displayName}")
- if (response.status == 200) {
- return readJSON(text: response.content)
- } else {
- error "Problem creating draftset ${response.status} : ${response.content}"
- }
+
+ PMD pmd = pmdConfig("pmd")
+ pmd.drafter.createDraftset(label)
}
def queryDraftset(String baseUrl, String credentials, String id, String query, String type) {
@@ -84,16 +56,8 @@
def deleteGraph(String baseUrl, String credentials, String id, String graph) {
echo "Deleting graph <${graph}> from draftset ${id}"
- String encGraph = java.net.URLEncoder.encode(graph, "UTF-8")
- def response = httpRequest(acceptType: 'APPLICATION_JSON',
- authentication: credentials,
- httpMode: 'DELETE',
- url: "${baseUrl}/v1/draftset/${id}/graph?graph=${encGraph}&silent=true")
- if (response.status == 200) {
- return readJSON(text: response.content)
- } else {
- error "Problem deleting graph ${response.status} : ${response.content}"
- }
+ PMD pmd = pmdConfig("pmd")
+ pmd.drafter.deleteGraph(id, graph)
}
def addData(String baseUrl, String credentials, String id, data, String type, String graph=null) {
diff --git a/vars/jobDraft.groovy b/vars/jobDraft.groovy
index 6b690df..1c0fd7e 100644
--- a/vars/jobDraft.groovy
+++ b/vars/jobDraft.groovy
@@ -1,24 +1,18 @@
+import uk.org.floop.jenkins_pmd.PMD
+
def create() {
echo "Creating job draft"
- configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) {
- def config = readJSON(text: readFile(file: configfile))
- String PMD = config['pmd_api']
- String credentials = config['credentials']
- drafter.createDraftset(PMD, credentials, env.JOB_NAME)
- }
+ PMD pmd = pmdConfig("pmd")
+ pmd.drafter.createDraftset(env.JOB_NAME)
}
def delete() {
echo "Deleting job draft"
- configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) {
- def config = readJSON(text: readFile(file: configfile))
- String PMD = config['pmd_api']
- String credentials = config['credentials']
- def draft = drafter.findDraftset(PMD, credentials, env.JOB_NAME)
- drafter.deleteDraftset(PMD, credentials, draft.id)
- }
+ PMD pmd = pmdConfig("pmd")
+ def draft = pmd.drafter.findDraftset(env.JOB_NAME)
+ pmd.drafter.deleteDraftset(draft.id)
}
def replace() {
@@ -34,23 +28,15 @@
def find() {
echo "Finding job draft"
- configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) {
- def config = readJSON(text: readFile(file: configfile))
- String PMD = config['pmd_api']
- String credentials = config['credentials']
- drafter.findDraftset(PMD, credentials, env.JOB_NAME)
- }
+ PMD pmd = pmdConfig("pmd")
+ pmd.drafter.findDraftset(env.JOB_NAME)
}
def publish() {
echo "Publishing job draft"
- configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) {
- def config = readJSON(text: readFile(file: configfile))
- String PMD = config['pmd_api']
- String credentials = config['credentials']
- def draft = drafter.findDraftset(PMD, credentials, env.JOB_NAME)
- drafter.publishDraftset(PMD, credentials, draft.id)
- }
+ PMD pmd = pmdConfig("pmd")
+ def draft = pmd.drafter.findDraftset(env.JOB_NAME)
+ pmd.drafter.publishDraftset(draft.id)
}
diff --git a/vars/pmdConfig.groovy b/vars/pmdConfig.groovy
new file mode 100644
index 0000000..6a31755
--- /dev/null
+++ b/vars/pmdConfig.groovy
@@ -0,0 +1,12 @@
+import uk.org.floop.jenkins_pmd.Drafter
+import uk.org.floop.jenkins_pmd.PMD
+import uk.org.floop.jenkins_pmd.PMDConfig
+
+def call(String configId) {
+ configFileProvider([configFile(fileId: configId, variable: 'configfile')]) {
+ PMDConfig config = new PMDConfig(readJSON(text: readFile(file: configfile)))
+ withCredentials([usernamePassword(credentialsId: config.credentials, usernameVariable: 'USER', passwordVariable: 'PASS')]) {
+ return new PMD(config, USER as String, PASS as String)
+ }
+ }
+}
diff --git a/vars/publishDraftset.groovy b/vars/publishDraftset.groovy
index 1c92783..a2ee4de 100644
--- a/vars/publishDraftset.groovy
+++ b/vars/publishDraftset.groovy
@@ -1,5 +1,5 @@
def call() {
- configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) {
+ configFileProvider([configFile(fileId: 'pmdConfig', variable: 'configfile')]) {
def config = readJSON(text: readFile(file: configfile))
String PMD = config['pmd_api']
String credentials = config['credentials']
diff --git a/vars/uploadCodelist.groovy b/vars/uploadCodelist.groovy
index 1ddc6a6..4e1c5db 100644
--- a/vars/uploadCodelist.groovy
+++ b/vars/uploadCodelist.groovy
@@ -1,5 +1,5 @@
def call(String csv, String name) {
- configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) {
+ configFileProvider([configFile(fileId: 'pmdConfig', variable: 'configfile')]) {
def config = readJSON(text: readFile(file: configfile))
String PMD = config['pmd_api']
String credentials = config['credentials']
diff --git a/vars/uploadComponents.groovy b/vars/uploadComponents.groovy
index 3f431bd..eaa8621 100644
--- a/vars/uploadComponents.groovy
+++ b/vars/uploadComponents.groovy
@@ -1,5 +1,5 @@
def call(String csv) {
- configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) {
+ configFileProvider([configFile(fileId: 'pmdConfig', variable: 'configfile')]) {
def config = readJSON(text: readFile(file: configfile))
String PMD = config['pmd_api']
String credentials = config['credentials']
diff --git a/vars/uploadCube.groovy b/vars/uploadCube.groovy
index af66ac4..b7a87b3 100644
--- a/vars/uploadCube.groovy
+++ b/vars/uploadCube.groovy
@@ -1,5 +1,5 @@
def call(String datasetLabel, obslist) {
- configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) {
+ configFileProvider([configFile(fileId: 'pmdConfig', variable: 'configfile')]) {
def config = readJSON(text: readFile(file: configfile))
String PMD = config['pmd_api']
String credentials = config['credentials']
diff --git a/vars/uploadDraftset.groovy b/vars/uploadDraftset.groovy
index 184d8b7..3d044f7 100644
--- a/vars/uploadDraftset.groovy
+++ b/vars/uploadDraftset.groovy
@@ -1,5 +1,5 @@
def call(String datasetLabel, csvs, String mapping=null, String datasetPath=null) {
- configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) {
+ configFileProvider([configFile(fileId: 'pmdConfig', variable: 'configfile')]) {
def config = readJSON(text: readFile(file: configfile))
String PMD = config['pmd_api']
String credentials = config['credentials']
diff --git a/vars/uploadTidy.groovy b/vars/uploadTidy.groovy
index c5ddbef..442252b 100644
--- a/vars/uploadTidy.groovy
+++ b/vars/uploadTidy.groovy
@@ -1,47 +1,29 @@
+import uk.org.floop.jenkins_pmd.PMD
+
def call(csvs, String mapping=null, String datasetPath=null) {
if (!datasetPath) {
datasetPath = util.slugise(env.JOB_NAME)
}
- configFileProvider([configFile(fileId: 'pmd', variable: 'configfile')]) {
- def config = readJSON(text: readFile(file: configfile))
- String PMD = config['pmd_api']
- String credentials = config['credentials']
- String PIPELINE = config['pipeline_api']
- String baseURI = config['base_uri']
- if (!mapping) {
- if (fileExists('metadata/columns.csv')) {
- mapping = 'metadata/columns.csv'
- } else {
- mapping = config['default_mapping']
- }
- }
- if (mapping.startsWith('http')) {
- def response = httpRequest(
- httpMode: 'GET',
- url: mapping)
- dir ('metadata') {
- writeFile file: 'columns.csv', text: response.content
- }
+ PMD pmd = pmdConfig("pmd")
+ if (!mapping) {
+ if (fileExists('metadata/columns.csv')) {
mapping = 'metadata/columns.csv'
+ } else {
+ mapping = config['default_mapping']
}
+ }
+ if (!mapping.startsWith('http')) {
+ mapping = "${WORKSPACE}/${mapping}"
+ }
- dataset.delete(datasetPath)
+ dataset.delete(datasetPath)
- def draft = jobDraft.find()
+ def draft = jobDraft.find()
- drafter.addData(PMD, credentials, draft.id,
- readFile("out/dataset.trig"), "application/trig;charset=UTF-8")
+ pmd.drafter.addData(draft.id as String,"${WORKSPACE}/out/dataset.trig","application/trig","UTF-8")
- csvs.each { csv ->
- echo "Uploading ${csv}"
- runPipeline("${PIPELINE}/ons-table2qb.core/data-cube/import",
- draft.id, credentials, [
- [name: 'observations-csv',
- file: [name: csv, type: 'text/csv;charset=UTF-8']],
- [name: 'dataset-name', value: ''],
- [name: 'dataset-slug', value: datasetPath],
- [name: 'columns-csv', file: [name: mapping, type: 'text/csv;charset=UTF-8']]
- ])
- }
+ csvs.each { csv ->
+ echo "Uploading ${csv}"
+ pmd.pipelines.dataCube(draft.id as String, "${WORKSPACE}/${csv}", '', datasetPath, mapping)
}
}