main.py

import transformers
import evaluate
from transformers import AutoTokenizer

van = """
wsldfjkghbjsfkiljfsdsf
"""

if __name__ == '__main__':
    tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
    print(tokenizer(van, truncation=True))