unit-parser/main.py at master · shmdoc/unit-parser · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
# Created by Arno on 02/12/2019

# Source: https://stackoverflow.com/questions/3368969/find-string-between-two-substrings
import re

# Source: https://stackoverflow.com/questions/54059917/generate-all-length-n-permutations-of-true-false
import itertools

import sys

import escape_helpers


class Unit:
    name = ""
    notation = ""
    uri = ""
    definition = ""


supported_units = (
    "cm^3", "d", "degC", "degF", "degree_east", "degree_north", "g",
    "kg", "km", "l", "yr")

# html = open(sys.argv[1])  # The command line argument should point to this file: "units.html"
html = open("units.html")

unit_list = set()

lastReadUnit = None

sectionName = ""
prefPassed = False
failedlines = ""
failed = False
for line in html:
    if '<td class="conceptURI">' in line:
        lastReadUnit = Unit()
        parsed = re.search('<td class="conceptURI"><a href="(.*)">', line)
        try:
            lastReadUnit.uri = parsed.group(1)
        except:
            failed = True
            failedlines += line
    if '<td class="SN">' in line:
        parsed = re.search('<td class="SN">(.*)<a href=""></a></td>', line)
        try:
            lastReadUnit.definition = parsed.group(1)
        except:
            pass
    if '<td class="PREF">' in line:
        prefPassed = True
        parsed = re.search('<td class="PREF">(.*)<a href=""></a></td>', line)
        try:
            lastReadUnit.name = parsed.group(1)
        except:
            failed = True
            failedlines += line
    elif prefPassed and '<td class="ALT">' in line:
        prefPassed = False
        parsed = re.search('<td class="ALT">(.*)<a href=""></a></td>', line)
        try:
            lastReadUnit.notation = parsed.group(1)
        except:
            failed = True
            failedlines += line

        if not failed:
            if lastReadUnit.notation in supported_units:
                unit_list.add(lastReadUnit)
        else:
            failed = False

html.close()


def str_query(uri, relation, value):
    if value is not None:

        uri_relations = ("ext:unitUri")
        escaped_value = ""
        if relation in uri_relations:
            escaped_value = escape_helpers.sparql_escape_uri(value)
        else:
            escaped_value = escape_helpers.sparql_escape(value)

        if isinstance(value, bool):
            # Fix for weird problem with booleans
            escaped_value = escaped_value.replace("False", "false")
            escaped_value = escaped_value.replace("True", "true")
            escaped_value = escaped_value.replace("^^xsd:boolean",
                                                  "^^<http://mu.semte.ch/vocabularies/typed-literals/boolean>")

        return "\t\t{uri} {relation} {value} . \n".format(uri=uri, relation=relation,
                                                          value=escaped_value)
    return ""


# base_uri = "http://vocab.nerc.ac.uk/collection/P06/current/{id}/"

query_str = "INSERT DATA { \n"
query_str += "\tGRAPH {app_uri} {{ \n".format(
    app_uri=escape_helpers.sparql_escape_uri("http://mu.semte.ch/application"))
for unit in unit_list:
    uri = escape_helpers.sparql_escape_uri(unit.uri)
    print(unit.name, unit.notation, unit.uri)
    query_str += "\t\t{uri} a ext:Unit . \n".format(uri=uri)
    relation = "ext:unitName"
    query_str += str_query(uri, relation, unit.name)
    relation = "ext:unitNotation"
    query_str += str_query(uri, relation, unit.notation)
    relation = "ext:unitUri"
    query_str += str_query(uri, relation, unit.uri)
    relation = "ext:unitDefinition"
    query_str += str_query(uri, relation, unit.definition)
    relation = "mu:uuid"
    query_str += str_query(uri, relation, unit.uri[-5:-1])
query_str += "\t}\n"
query_str += "}\n"

prefixes = "PREFIX ext: {uri}\n".format(
    uri=escape_helpers.sparql_escape_uri("http://mu.semte.ch/vocabularies/ext/"))
prefixes += "PREFIX mu: {uri}\n".format(
    uri=escape_helpers.sparql_escape_uri("http://mu.semte.ch/vocabularies/core/"))
query_str = prefixes + query_str

# f = open(sys.argv[2], "w+")  # argv[2] should be cpp.json
f = open("query.sparql", "w+")
f.write(query_str)
f.close()

fails = open("error.log", "w+")
fails.write(failedlines)
fails.close()