import subprocess import sys import tiktoken def test_encoding_for_model(): enc = tiktoken.encoding_for_model("gpt2") assert enc.name == "gpt2" enc = tiktoken.encoding_for_model("text-davinci-003") assert enc.name == "p50k_base" enc = tiktoken.encoding_for_model("text-davinci-edit-001") assert enc.name == "p50k_edit" enc = tiktoken.encoding_for_model("gpt-3.5-turbo-0301") assert enc.name == "cl100k_base" def test_optional_blobfile_dependency(): prog = """ import tiktoken import sys assert "blobfile" not in sys.modules """ subprocess.check_call([sys.executable, "-c", prog])