76 lines
2.4 KiB
Python
76 lines
2.4 KiB
Python
|
import re
|
|||
|
|
import nltk
|
|||
|
|
from bs4 import BeautifulSoup
|
|||
|
|
from nltk.corpus import stopwords
|
|||
|
|
from pymystem3 import Mystem
|
|||
|
|
from string import punctuation
|
|||
|
|
from transformers import BertTokenizer
|
|||
|
|
|
|||
|
|
nltk.download("stopwords")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def get_clear_text(text):
|
|||
|
|
soup = BeautifulSoup(text, 'html.parser')
|
|||
|
|
|
|||
|
|
# extract the plain text from the HTML document without tags
|
|||
|
|
clear_text = ''
|
|||
|
|
for tag in soup.find_all():
|
|||
|
|
clear_text += tag.string or ''
|
|||
|
|
|
|||
|
|
clear_text = re.sub(pattern='[\u202F\u00A0\n]+', repl=' ', string=clear_text)
|
|||
|
|
|
|||
|
|
# only words
|
|||
|
|
clear_text = re.sub(pattern='[^A-ZА-ЯЁ -]', repl='', string=clear_text, flags=re.IGNORECASE)
|
|||
|
|
|
|||
|
|
clear_text = re.sub(pattern='\s+', repl=' ', string=clear_text)
|
|||
|
|
|
|||
|
|
clear_text = clear_text.lower()
|
|||
|
|
|
|||
|
|
mystem = Mystem()
|
|||
|
|
russian_stopwords = stopwords.words("russian")
|
|||
|
|
|
|||
|
|
tokens = mystem.lemmatize(clear_text)
|
|||
|
|
tokens = [token for token in tokens if token not in russian_stopwords \
|
|||
|
|
and token != " " \
|
|||
|
|
and token.strip() not in punctuation]
|
|||
|
|
|
|||
|
|
clear_text = " ".join(tokens)
|
|||
|
|
|
|||
|
|
return clear_text
|
|||
|
|
|
|||
|
|
|
|||
|
|
# if __name__ == '__main__':
|
|||
|
|
#
|
|||
|
|
# # initialize the tokenizer with the pre-trained BERT model and vocabulary
|
|||
|
|
# tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')
|
|||
|
|
#
|
|||
|
|
# # split each text into smaller segments of maximum length 512
|
|||
|
|
# max_length = 512
|
|||
|
|
# segmented_texts = []
|
|||
|
|
# for text in [clear_text1, clear_text2]:
|
|||
|
|
# segmented_text = []
|
|||
|
|
# for i in range(0, len(text), max_length):
|
|||
|
|
# segment = text[i:i+max_length]
|
|||
|
|
# segmented_text.append(segment)
|
|||
|
|
# segmented_texts.append(segmented_text)
|
|||
|
|
#
|
|||
|
|
# # tokenize each segment using the BERT tokenizer
|
|||
|
|
# tokenized_texts = []
|
|||
|
|
# for segmented_text in segmented_texts:
|
|||
|
|
# tokenized_text = []
|
|||
|
|
# for segment in segmented_text:
|
|||
|
|
# segment_tokens = tokenizer.tokenize(segment)
|
|||
|
|
# segment_tokens = ['[CLS]'] + segment_tokens + ['[SEP]']
|
|||
|
|
# tokenized_text.append(segment_tokens)
|
|||
|
|
# tokenized_texts.append(tokenized_text)
|
|||
|
|
#
|
|||
|
|
# input_ids = []
|
|||
|
|
# for tokenized_text in tokenized_texts:
|
|||
|
|
# input_id = []
|
|||
|
|
# for segment_tokens in tokenized_text:
|
|||
|
|
# segment_id = tokenizer.convert_tokens_to_ids(segment_tokens)
|
|||
|
|
# input_id.append(segment_id)
|
|||
|
|
# input_ids.append(input_id)
|
|||
|
|
#
|
|||
|
|
# print(input_ids)
|