|
[project] |
|
name = "tiktoken" |
|
version = "0.6.0" |
|
description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" |
|
readme = "README.md" |
|
license = {file = "LICENSE"} |
|
authors = [{name = "Shantanu Jain"}, {email = "[email protected]"}] |
|
dependencies = ["regex>=2022.1.18", "requests>=2.26.0"] |
|
optional-dependencies = {blobfile = ["blobfile>=2"]} |
|
requires-python = ">=3.8" |
|
|
|
[project.urls] |
|
homepage = "https://github.com/openai/tiktoken" |
|
repository = "https://github.com/openai/tiktoken" |
|
changelog = "https://github.com/openai/tiktoken/blob/main/CHANGELOG.md" |
|
|
|
[build-system] |
|
build-backend = "setuptools.build_meta" |
|
requires = ["setuptools>=62.4", "wheel", "setuptools-rust>=1.5.2"] |
|
|
|
[tool.cibuildwheel] |
|
build-frontend = "build" |
|
build-verbosity = 1 |
|
|
|
linux.before-all = "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y" |
|
linux.environment = { PATH = "$PATH:$HOME/.cargo/bin" } |
|
macos.before-all = "rustup target add aarch64-apple-darwin" |
|
|
|
skip = [ |
|
"*-manylinux_i686", |
|
"*-musllinux_i686", |
|
"*-win32", |
|
] |
|
macos.archs = ["x86_64", "arm64"] |
|
|
|
|
|
test-skip = "*-macosx_arm64" |
|
|
|
before-test = "pip install pytest hypothesis" |
|
test-command = "pytest {project}/tests --import-mode=append" |
|
|
|
[[tool.cibuildwheel.overrides]] |
|
select = "*linux_aarch64" |
|
test-command = """python -c 'import tiktoken; enc = tiktoken.get_encoding("gpt2"); assert enc.encode("hello world") == [31373, 995]'""" |
|
|
|
|
