vb@2
|
1 |
#! /usr/bin/env python3
|
vb@2
|
2 |
|
vb@2
|
3 |
from argparse import ArgumentParser
|
vb@2
|
4 |
from fileinput import FileInput, hook_encoded
|
vb@2
|
5 |
import re, itertools, sys
|
vb@2
|
6 |
|
vb@2
|
7 |
try:
|
vb@2
|
8 |
from math import log2
|
vb@2
|
9 |
except:
|
vb@2
|
10 |
from math import log
|
vb@2
|
11 |
def log2(x): return log(x) / log(2)
|
vb@2
|
12 |
|
vb@2
|
13 |
word = re.compile(r"(\S*?)(/|\s.*|$)")
|
vb@2
|
14 |
unwanted = re.compile(r"(^\d|[^']*')")
|
vb@2
|
15 |
space = re.compile(r'^\s')
|
vb@2
|
16 |
|
vb@2
|
17 |
p = ArgumentParser(description="create dictionary csv out of hunspell data")
|
vb@2
|
18 |
p.add_argument('--hunspell', '-H', type=str, default="/usr/share/hunspell",
|
vb@2
|
19 |
help='directory where hunspell dictionary files reside (default: /usr/share/hunspell)')
|
vb@2
|
20 |
p.add_argument('--lang', '-l', type=str, default="en_US",
|
vb@2
|
21 |
help='use dictionary for language LANG (default: en_US)')
|
vb@2
|
22 |
p.add_argument('--encoding', '-e', type=str, default="utf-8",
|
vb@2
|
23 |
help='file encoding (default: utf-8)')
|
vb@2
|
24 |
|
vb@2
|
25 |
args = p.parse_args()
|
vb@2
|
26 |
|
vb@120
|
27 |
try:
|
vb@120
|
28 |
from icu import UnicodeString, Locale
|
vb@120
|
29 |
except ImportError:
|
vb@120
|
30 |
print("warning: PyICU not installed, using fallback", file=sys.stderr)
|
vb@120
|
31 |
def upper(x):
|
vb@120
|
32 |
return x.upper();
|
vb@120
|
33 |
else:
|
vb@120
|
34 |
locale = Locale(args.lang)
|
vb@120
|
35 |
def upper(x):
|
vb@120
|
36 |
u = UnicodeString(x)
|
vb@120
|
37 |
return str(u.toUpper(locale))
|
vb@120
|
38 |
|
vb@2
|
39 |
_all = (
|
vb@120
|
40 |
upper(word.match(line).group(1))
|
vb@2
|
41 |
for line in FileInput(
|
vb@2
|
42 |
args.hunspell + "/" + args.lang + ".dic",
|
vb@2
|
43 |
openhook=hook_encoded(args.encoding)
|
vb@2
|
44 |
)
|
vb@2
|
45 |
if not space.match(line)
|
vb@2
|
46 |
)
|
vb@2
|
47 |
_words = [w for w in _all if len(w) > 2 and not unwanted.match(w)]
|
vb@2
|
48 |
_words.sort()
|
vb@2
|
49 |
_words = [w for w, g in itertools.groupby(_words)]
|
vb@2
|
50 |
|
vb@120
|
51 |
while len(_words) > 65536 * 2:
|
vb@120
|
52 |
_words = _words[::2]
|
vb@120
|
53 |
|
vb@2
|
54 |
if len(_words) > 65536:
|
vb@2
|
55 |
_words = _words[:65536]
|
vb@2
|
56 |
elif len(_words) < 65536:
|
vb@2
|
57 |
sys.stderr.write(
|
vb@2
|
58 |
"warning for {}: only {:.2f} bit in wordlist, that makes {:.2f} bit for 5 words\n".format(
|
vb@2
|
59 |
args.lang,
|
vb@2
|
60 |
log2(len(_words)),
|
vb@2
|
61 |
log2(len(_words))*5
|
vb@2
|
62 |
)
|
vb@2
|
63 |
)
|
vb@2
|
64 |
_words.extend(_words[:65536-len(_words)])
|
vb@2
|
65 |
|
vb@2
|
66 |
assert len(_words) == 65536, "lenght is {}".format(len(_words))
|
vb@2
|
67 |
|
vb@2
|
68 |
for i, w in enumerate(_words):
|
Edouard@4
|
69 |
print("{l},{i},{w},0".format(l=args.lang[:2], i=i, w=w))
|