| | #!/bin/env python3 |
|---|
| | import re |
|---|
| | import time |
|---|
| | from datetime import datetime |
|---|
| | |
|---|
| | from rdflib import Graph, URIRef, Literal, ConjunctiveGraph |
|---|
| | from rdflib.namespace import Namespace, DCTERMS, RDF |
|---|
| | from rdflib.store import NO_STORE |
|---|
| | from requests import Session |
|---|
| | from urllib.parse import urljoin |
|---|
| | |
|---|
| | DCAT = Namespace('http://www.w3.org/ns/dcat#') |
|---|
| | GDP = Namespace('http://gss-data.org.uk/def/gdp#') |
|---|
| | |
|---|
| | ds = ConjunctiveGraph('Sleepycat') |
|---|
| | if ds.open('datasets.db') == NO_STORE: |
|---|
| | ds.open('datasets.db', create=True) |
|---|
| | print(f'Datasets store has {len(ds)} triples') |
|---|
| | print(f' and {len(list(ds.triples((None, RDF.type, DCAT.Dataset))))} datasets.') |
|---|
| | |
|---|
| | gss = Graph('SPARQLStore', identifier='http://gss-data.org.uk') |
|---|
| | gss.open("http://gss-data.org.uk/sparql") |
|---|
| | |
|---|
| | orgs = {org.label.value: org.org for org in gss.query( |
|---|
| | """ |
|---|
| | PREFIX org: <http://www.w3.org/ns/org#> |
|---|
| | SELECT DISTINCT ?org ?label |
|---|
| | WHERE { |
|---|
| | ?org a org:Organization ; |
|---|
| | rdfs:label ?label . |
|---|
| | }""")} |
|---|
| | |
|---|
| | gss.close() |
|---|
| | |
|---|
| | datasets_url_base = 'https://www.gov.uk/government/statistics.json' |
|---|
| | |
|---|
| | s = Session() |
|---|
| | still_going = True |
|---|
| | datasets_url = datasets_url_base |
|---|
| | abbr_re = re.compile(r'<abbr title="([^"]+)">') |
|---|
| | collection_re = re.compile(r'Part of a collection: <a href="([^"]+)">') |
|---|
| | |
|---|
| | |
|---|
| | def fetch_carefully(url): |
|---|
| | tries = 0 |
|---|
| | holdoff = 5 |
|---|
| | while tries < 10: |
|---|
| | resp = s.get(url) |
|---|
| | if resp.status_code == 200: |
|---|
| | try: |
|---|
| | return resp.json() |
|---|
| | except: |
|---|
| | pass |
|---|
| | time.sleep(holdoff) |
|---|
| | tries = tries + 1 |
|---|
| | holdoff = holdoff * 2 |
|---|
| | |
|---|
| | |
|---|
| | while still_going: |
|---|
| | datasets = fetch_carefully(datasets_url) |
|---|
| | fresh_datasets = False |
|---|
| | for res_obj in datasets['results']: |
|---|
| | res = res_obj['result'] |
|---|
| | publisher = None |
|---|
| | issued = None |
|---|
| | collection = None |
|---|
| | abbr_match = abbr_re.match(res['organisations']) |
|---|
| | if abbr_match: |
|---|
| | publisher = orgs.get(abbr_match.group(1), None) |
|---|
| | else: |
|---|
| | publisher = orgs.get(res['organisations'], None) |
|---|
| | issued = datetime.fromisoformat(res['public_timestamp']) |
|---|
| | if 'publication_collections' in res and res['publication_collections'] is not None: |
|---|
| | coll_match = collection_re.match(res['publication_collections']) |
|---|
| | if coll_match: |
|---|
| | collection = coll_match.group(1) |
|---|
| | landingPage = URIRef(urljoin(datasets_url, res['url'])) |
|---|
| | prev_publish_date = ds.value(subject=landingPage, predicate=DCTERMS.issued, any=False) |
|---|
| | if True or prev_publish_date is None or prev_publish_date.value != issued: |
|---|
| | print(f"Updating {res['title']}") |
|---|
| | fresh_datasets = True |
|---|
| | ds.set((landingPage, RDF.type, DCAT.Dataset)) |
|---|
| | if res['display_type'] == 'National Statistics': |
|---|
| | ds.add((landingPage, RDF.type, GDP.NationalStatistics)) |
|---|
| | elif res['display_type'] == 'Official Statistics': |
|---|
| | ds.add((landingPage, RDF.type, GDP.OfficialStatistics)) |
|---|
| | ds.set((landingPage, DCTERMS.title, Literal(res['title'], lang="en-gb"))) |
|---|
| | if publisher is not None: |
|---|
| | ds.set((landingPage, DCTERMS.publisher, publisher)) |
|---|
| | if issued is not None: |
|---|
| | ds.set((landingPage, DCTERMS.issued, Literal(issued))) |
|---|
| | |
|---|
| | if fresh_datasets and 'next_page_url' in datasets: |
|---|
| | datasets_url = urljoin(datasets_url, datasets['next_page_url']) |
|---|
| | still_going = True |
|---|
| | else: |
|---|
| | still_going = False |
|---|
| | |
|---|
| | print(f'Datasets store has {len(ds)} triples.') |
|---|
| | ds.close() |
|---|
| | |
|---|
| | |
