import sys from urllib import request, parse from rdflib import Graph, term from lxml import etree if len(sys.argv) < 2: print('Usage: python {} '.format(sys.argv[0])) print('') print('Extract rdfa, microdata and json-ld annotations from a website') exit(1) url = sys.argv[1] g = Graph() g.parse(url, format='rdfa') g.parse(url, format='microdata') def sanitize_triple(t): """Function to remove bad URIs from the graph that would otherwise make the serialization fail.""" def sanitize_triple_item(item): if isinstance(item, term.URIRef) and '/' not in item: return term.URIRef(parse.quote(str(item))) return item return (sanitize_triple_item(t[0]), sanitize_triple_item(t[1]), sanitize_triple_item(t[2])) with request.urlopen(url) as response: # Get all json-ld objects embedded in the html file html = response.read().decode('utf-8') parser = etree.XMLParser(recover=True) root = etree.fromstring(html, parser=parser) for jsonld in root.findall(".//script[@type='application/ld+json']"): g.parse(data=jsonld.text, publicID=url, format='json-ld') fixedgraph = Graph() fixedgraph += [sanitize_triple(s) for s in g] print(g.serialize(format='turtle').decode('utf-8'))