import numpy as np
from tqdm import tqdm

processed_data = []

texts = df.text.to_list()
titles = df.title.to_list()

data_tuples = list(zip(texts, titles))

with nlp.select_pipes(enable=["tok2vec", "attribute_ruler", "lemmatizer"]): 
    for i, doc in enumerate(tqdm(nlp.pipe(texts, batch_size=50, n_process=-1), total=len(texts))):

        valid_tokens = [
            t for t in doc if not t.is_stop and t.is_alpha
        ]

        if valid_tokens:
            valid_vectors = [t.vector for t in valid_tokens]
            doc_vector = np.mean(valid_vectors, axis=0)
        else:
            doc_vector = np.zeros(nlp.vocab.vectors_length)

        processed_data.append(
            {
                "title": titles[i],
                "text": texts[i],
                "tokens": [t.lemma_.lower() for t in valid_tokens],  
                "vector": doc_vector,
            }
        )