49 lines
998 B
Python
49 lines
998 B
Python
"""
|
|
this file holds a class, which implements a "detector" - language classifier -
|
|
that learns from predefined, frequency-analysed sets of ngrams
|
|
"""
|
|
|
|
|
|
class da_detector:
|
|
def __init__(self, langs_to_check=["sk"]):
|
|
# langs to check
|
|
# to be picked from ["cz", "sk", "de", "en", "fr"]
|
|
self.langs_to_check = langs_to_check
|
|
|
|
def rm_interpunction(data):
|
|
from string import punctuation
|
|
for ngram in data:
|
|
try:
|
|
ngram = ngram.translate(str.maketrans('', '', punctuation))
|
|
except Exception as e:
|
|
raise e
|
|
return data
|
|
|
|
def rm_digits(data):
|
|
from string import digits
|
|
try:
|
|
for ngram in data:
|
|
ngram = ngram.translate(None, digits)
|
|
except Exception as e:
|
|
raise e
|
|
return data
|
|
|
|
def parse_freqs(self, path):
|
|
import json
|
|
|
|
fullpath = freqs_folder + path
|
|
try:
|
|
with open(fullpath, 'r') as f:
|
|
j_data = f.read()
|
|
except Exception as e:
|
|
raise e
|
|
|
|
try:
|
|
obj = json.loads(j_data)
|
|
except Exception as e:
|
|
raise e
|
|
return obj
|
|
|
|
|
|
# vim: ff=unix noexpandtab
|