su_task3-nlp/da_detector.py

"""
this file holds a class, which implements a "detector" - language classifier -
that learns from predefined, frequency-analysed sets of ngrams
"""


class da_detector:
	def __init__(self, langs_to_check=["sk"]):
		# langs to check
		# to be picked from ["cz", "sk", "de", "en", "fr"]
		self.langs_to_check = langs_to_check

	def rm_interpunction(data):
		from string import punctuation
		for ngram in data:
			try:
				ngram = ngram.translate(str.maketrans('', '', punctuation))
			except Exception as e:
				raise e
		return data

	def rm_digits(data):
		from string import digits
		try:
			for ngram in data:
				ngram = ngram.translate(None, digits)
		except Exception as e:
			raise e
		return data

	def parse_freqs(self, path):
		import json

		fullpath = freqs_folder + path
		try:
			with open(fullpath, 'r') as f:
				j_data = f.read()
		except Exception as e:
			raise e

		try:
			obj = json.loads(j_data)
		except Exception as e:
			raise e
		return obj


# vim: ff=unix noexpandtab