{ "id": "veil-persian-tokenizer-efficiency", "title": "Persian tokenizer efficiency across models", "version": "1.0.0", "measuredAt": "2026-08-19", "license": "CC-BY-4.0", "measures": "Tokens spent on the same meaning in Persian versus English, per model tokenizer. This is a cost and context-window measurement. It is not a measure of model quality, Persian fluency, or reasoning.", "method": "Every passage in the paired corpus is encoded with the model's own tokenizer file, read from HuggingFace at the recorded commit. Counts are exact, not estimated. A model whose encode/decode round-trip does not reproduce the input is dropped rather than reported.", "corpus": { "id": "veil-persian-tokenizer-corpus", "version": "1.0.0", "items": 28, "faCharacters": 3065, "enCharacters": 3592 }, "harness": "scripts/tokenizer-bench.mjs", "verification": { "roundTrip": "every published row reproduces all 56 passages exactly", "reference": "cross-checked against huggingface/tokenizers via scripts/verify-tokenizer.py; see verifiedAgainstReference" }, "models": [ { "id": "persianmind", "label": "PersianMind", "vendor": "University of Tehran", "repo": "universitytehran/PersianMind-v1.0", "mirror": false, "persianSpecific": true, "note": "Llama-2 with a Persian-extended vocabulary. On this board to answer the obvious question: does extending the vocabulary for Persian actually pay?", "revision": "af603eeb074138e2a613fbc95d89f018afbd3041", "fileSha256": "add5ca15f72824c65d84025377ab32a53096095bafe8dcd55ce0871b8af1f91e", "measuredAt": "2026-08-19", "vocabSize": 41510, "fingerprint": "5d3f077a633414ae", "items": [ { "id": "news-01", "register": "news", "faTokens": 35, "enTokens": 34, "ratio": 1.029 }, { "id": "news-02", "register": "news", "faTokens": 36, "enTokens": 38, "ratio": 0.947 }, { "id": "news-03", "register": "news", "faTokens": 33, "enTokens": 27, "ratio": 1.222 }, { "id": "technical-01", "register": "technical", "faTokens": 35, "enTokens": 29, "ratio": 1.207 }, { "id": "technical-02", "register": "technical", "faTokens": 32, "enTokens": 24, "ratio": 1.333 }, { "id": "technical-03", "register": "technical", "faTokens": 31, "enTokens": 24, "ratio": 1.292 }, { "id": "technical-04", "register": "technical", "faTokens": 32, "enTokens": 27, "ratio": 1.185 }, { "id": "policy-01", "register": "policy", "faTokens": 36, "enTokens": 27, "ratio": 1.333 }, { "id": "policy-02", "register": "policy", "faTokens": 33, "enTokens": 22, "ratio": 1.5 }, { "id": "conversational-01", "register": "conversational", "faTokens": 26, "enTokens": 27, "ratio": 0.963 }, { "id": "conversational-02", "register": "conversational", "faTokens": 31, "enTokens": 24, "ratio": 1.292 }, { "id": "conversational-03", "register": "conversational", "faTokens": 27, "enTokens": 26, "ratio": 1.038 }, { "id": "scientific-01", "register": "scientific", "faTokens": 40, "enTokens": 31, "ratio": 1.29 }, { "id": "scientific-02", "register": "scientific", "faTokens": 25, "enTokens": 28, "ratio": 0.893 }, { "id": "scientific-03", "register": "scientific", "faTokens": 35, "enTokens": 26, "ratio": 1.346 }, { "id": "product-01", "register": "product", "faTokens": 24, "enTokens": 25, "ratio": 0.96 }, { "id": "product-02", "register": "product", "faTokens": 30, "enTokens": 24, "ratio": 1.25 }, { "id": "finance-01", "register": "finance", "faTokens": 28, "enTokens": 26, "ratio": 1.077 }, { "id": "finance-02", "register": "finance", "faTokens": 23, "enTokens": 22, "ratio": 1.045 }, { "id": "education-01", "register": "education", "faTokens": 31, "enTokens": 27, "ratio": 1.148 }, { "id": "education-02", "register": "education", "faTokens": 18, "enTokens": 25, "ratio": 0.72 }, { "id": "legal-01", "register": "legal", "faTokens": 29, "enTokens": 34, "ratio": 0.853 }, { "id": "legal-02", "register": "legal", "faTokens": 21, "enTokens": 22, "ratio": 0.955 }, { "id": "literary-01", "register": "literary", "faTokens": 34, "enTokens": 26, "ratio": 1.308 }, { "id": "literary-02", "register": "literary", "faTokens": 31, "enTokens": 20, "ratio": 1.55 }, { "id": "medical-01", "register": "medical", "faTokens": 23, "enTokens": 22, "ratio": 1.045 }, { "id": "instruction-01", "register": "instruction", "faTokens": 32, "enTokens": 23, "ratio": 1.391 }, { "id": "instruction-02", "register": "instruction", "faTokens": 28, "enTokens": 27, "ratio": 1.037 } ], "faTokens": 839, "enTokens": 737, "ratio": 1.138, "faTokensPerChar": 0.2737, "enTokensPerChar": 0.2052, "faCharsPerToken": 3.653, "rank": 1, "tied": false }, { "id": "gemma", "label": "Gemma", "vendor": "Google", "repo": "Xenova/gemma-tokenizer", "mirror": true, "persianSpecific": false, "note": "256k vocabulary — the largest on this board.", "revision": "fbc8c39d11899b177f1fe8f1d25bd25fbee66fad", "fileSha256": "05e97791a5e007260de1db7e1692e53150e08cea481e2bf25435553380c147ee", "measuredAt": "2026-08-19", "vocabSize": 256000, "fingerprint": "36f857605252c885", "items": [ { "id": "news-01", "register": "news", "faTokens": 38, "enTokens": 34, "ratio": 1.118 }, { "id": "news-02", "register": "news", "faTokens": 36, "enTokens": 35, "ratio": 1.029 }, { "id": "news-03", "register": "news", "faTokens": 37, "enTokens": 24, "ratio": 1.542 }, { "id": "technical-01", "register": "technical", "faTokens": 37, "enTokens": 28, "ratio": 1.321 }, { "id": "technical-02", "register": "technical", "faTokens": 34, "enTokens": 22, "ratio": 1.545 }, { "id": "technical-03", "register": "technical", "faTokens": 34, "enTokens": 24, "ratio": 1.417 }, { "id": "technical-04", "register": "technical", "faTokens": 33, "enTokens": 26, "ratio": 1.269 }, { "id": "policy-01", "register": "policy", "faTokens": 37, "enTokens": 23, "ratio": 1.609 }, { "id": "policy-02", "register": "policy", "faTokens": 34, "enTokens": 18, "ratio": 1.889 }, { "id": "conversational-01", "register": "conversational", "faTokens": 29, "enTokens": 25, "ratio": 1.16 }, { "id": "conversational-02", "register": "conversational", "faTokens": 29, "enTokens": 22, "ratio": 1.318 }, { "id": "conversational-03", "register": "conversational", "faTokens": 27, "enTokens": 25, "ratio": 1.08 }, { "id": "scientific-01", "register": "scientific", "faTokens": 42, "enTokens": 30, "ratio": 1.4 }, { "id": "scientific-02", "register": "scientific", "faTokens": 34, "enTokens": 26, "ratio": 1.308 }, { "id": "scientific-03", "register": "scientific", "faTokens": 35, "enTokens": 25, "ratio": 1.4 }, { "id": "product-01", "register": "product", "faTokens": 34, "enTokens": 24, "ratio": 1.417 }, { "id": "product-02", "register": "product", "faTokens": 34, "enTokens": 20, "ratio": 1.7 }, { "id": "finance-01", "register": "finance", "faTokens": 30, "enTokens": 24, "ratio": 1.25 }, { "id": "finance-02", "register": "finance", "faTokens": 29, "enTokens": 20, "ratio": 1.45 }, { "id": "education-01", "register": "education", "faTokens": 35, "enTokens": 26, "ratio": 1.346 }, { "id": "education-02", "register": "education", "faTokens": 23, "enTokens": 24, "ratio": 0.958 }, { "id": "legal-01", "register": "legal", "faTokens": 31, "enTokens": 32, "ratio": 0.969 }, { "id": "legal-02", "register": "legal", "faTokens": 25, "enTokens": 21, "ratio": 1.19 }, { "id": "literary-01", "register": "literary", "faTokens": 33, "enTokens": 26, "ratio": 1.269 }, { "id": "literary-02", "register": "literary", "faTokens": 29, "enTokens": 20, "ratio": 1.45 }, { "id": "medical-01", "register": "medical", "faTokens": 31, "enTokens": 19, "ratio": 1.632 }, { "id": "instruction-01", "register": "instruction", "faTokens": 29, "enTokens": 23, "ratio": 1.261 }, { "id": "instruction-02", "register": "instruction", "faTokens": 29, "enTokens": 26, "ratio": 1.115 } ], "faTokens": 908, "enTokens": 692, "ratio": 1.312, "faTokensPerChar": 0.2962, "enTokensPerChar": 0.1927, "faCharsPerToken": 3.376, "rank": 2, "tied": true }, { "id": "gemma-2", "label": "Gemma 2", "vendor": "Google", "repo": "Xenova/gemma2-tokenizer", "mirror": true, "persianSpecific": false, "note": "", "revision": "550a07caea23182155d4803a36a81c6f33040024", "fileSha256": "3f289bc05132635a8bc7aca7aa21255efd5e18f3710f43e3cdb96bcd41be4922", "measuredAt": "2026-08-19", "vocabSize": 256000, "fingerprint": "5267192a42d67ba7", "items": [ { "id": "news-01", "register": "news", "faTokens": 38, "enTokens": 34, "ratio": 1.118 }, { "id": "news-02", "register": "news", "faTokens": 36, "enTokens": 35, "ratio": 1.029 }, { "id": "news-03", "register": "news", "faTokens": 37, "enTokens": 24, "ratio": 1.542 }, { "id": "technical-01", "register": "technical", "faTokens": 37, "enTokens": 28, "ratio": 1.321 }, { "id": "technical-02", "register": "technical", "faTokens": 34, "enTokens": 22, "ratio": 1.545 }, { "id": "technical-03", "register": "technical", "faTokens": 34, "enTokens": 24, "ratio": 1.417 }, { "id": "technical-04", "register": "technical", "faTokens": 33, "enTokens": 26, "ratio": 1.269 }, { "id": "policy-01", "register": "policy", "faTokens": 37, "enTokens": 23, "ratio": 1.609 }, { "id": "policy-02", "register": "policy", "faTokens": 34, "enTokens": 18, "ratio": 1.889 }, { "id": "conversational-01", "register": "conversational", "faTokens": 29, "enTokens": 25, "ratio": 1.16 }, { "id": "conversational-02", "register": "conversational", "faTokens": 29, "enTokens": 22, "ratio": 1.318 }, { "id": "conversational-03", "register": "conversational", "faTokens": 27, "enTokens": 25, "ratio": 1.08 }, { "id": "scientific-01", "register": "scientific", "faTokens": 42, "enTokens": 30, "ratio": 1.4 }, { "id": "scientific-02", "register": "scientific", "faTokens": 34, "enTokens": 26, "ratio": 1.308 }, { "id": "scientific-03", "register": "scientific", "faTokens": 35, "enTokens": 25, "ratio": 1.4 }, { "id": "product-01", "register": "product", "faTokens": 34, "enTokens": 24, "ratio": 1.417 }, { "id": "product-02", "register": "product", "faTokens": 34, "enTokens": 20, "ratio": 1.7 }, { "id": "finance-01", "register": "finance", "faTokens": 30, "enTokens": 24, "ratio": 1.25 }, { "id": "finance-02", "register": "finance", "faTokens": 29, "enTokens": 20, "ratio": 1.45 }, { "id": "education-01", "register": "education", "faTokens": 35, "enTokens": 26, "ratio": 1.346 }, { "id": "education-02", "register": "education", "faTokens": 23, "enTokens": 24, "ratio": 0.958 }, { "id": "legal-01", "register": "legal", "faTokens": 31, "enTokens": 32, "ratio": 0.969 }, { "id": "legal-02", "register": "legal", "faTokens": 25, "enTokens": 21, "ratio": 1.19 }, { "id": "literary-01", "register": "literary", "faTokens": 33, "enTokens": 26, "ratio": 1.269 }, { "id": "literary-02", "register": "literary", "faTokens": 29, "enTokens": 20, "ratio": 1.45 }, { "id": "medical-01", "register": "medical", "faTokens": 31, "enTokens": 19, "ratio": 1.632 }, { "id": "instruction-01", "register": "instruction", "faTokens": 29, "enTokens": 23, "ratio": 1.261 }, { "id": "instruction-02", "register": "instruction", "faTokens": 29, "enTokens": 26, "ratio": 1.115 } ], "faTokens": 908, "enTokens": 692, "ratio": 1.312, "faTokensPerChar": 0.2962, "enTokensPerChar": 0.1927, "faCharsPerToken": 3.376, "rank": 2, "tied": true }, { "id": "mistral-small-3", "label": "Mistral Small 3", "vendor": "Mistral AI", "repo": "mistralai/Mistral-Small-24B-Instruct-2501", "mirror": false, "persianSpecific": false, "note": "Tekken — the larger vocabulary Mistral moved to.", "revision": "9527884be6e5616bdd54de542f9ae13384489724", "fileSha256": "b76085f9923309d873994d444989f7eb6ec074b06f25b58f1e8d7b7741070949", "measuredAt": "2026-08-19", "vocabSize": 131072, "fingerprint": "c0eabb636dbd2b86", "items": [ { "id": "news-01", "register": "news", "faTokens": 41, "enTokens": 34, "ratio": 1.206 }, { "id": "news-02", "register": "news", "faTokens": 38, "enTokens": 35, "ratio": 1.086 }, { "id": "news-03", "register": "news", "faTokens": 36, "enTokens": 25, "ratio": 1.44 }, { "id": "technical-01", "register": "technical", "faTokens": 40, "enTokens": 29, "ratio": 1.379 }, { "id": "technical-02", "register": "technical", "faTokens": 33, "enTokens": 23, "ratio": 1.435 }, { "id": "technical-03", "register": "technical", "faTokens": 35, "enTokens": 24, "ratio": 1.458 }, { "id": "technical-04", "register": "technical", "faTokens": 38, "enTokens": 25, "ratio": 1.52 }, { "id": "policy-01", "register": "policy", "faTokens": 39, "enTokens": 26, "ratio": 1.5 }, { "id": "policy-02", "register": "policy", "faTokens": 33, "enTokens": 18, "ratio": 1.833 }, { "id": "conversational-01", "register": "conversational", "faTokens": 33, "enTokens": 25, "ratio": 1.32 }, { "id": "conversational-02", "register": "conversational", "faTokens": 29, "enTokens": 22, "ratio": 1.318 }, { "id": "conversational-03", "register": "conversational", "faTokens": 28, "enTokens": 25, "ratio": 1.12 }, { "id": "scientific-01", "register": "scientific", "faTokens": 47, "enTokens": 30, "ratio": 1.567 }, { "id": "scientific-02", "register": "scientific", "faTokens": 34, "enTokens": 25, "ratio": 1.36 }, { "id": "scientific-03", "register": "scientific", "faTokens": 36, "enTokens": 25, "ratio": 1.44 }, { "id": "product-01", "register": "product", "faTokens": 29, "enTokens": 24, "ratio": 1.208 }, { "id": "product-02", "register": "product", "faTokens": 31, "enTokens": 21, "ratio": 1.476 }, { "id": "finance-01", "register": "finance", "faTokens": 34, "enTokens": 24, "ratio": 1.417 }, { "id": "finance-02", "register": "finance", "faTokens": 32, "enTokens": 21, "ratio": 1.524 }, { "id": "education-01", "register": "education", "faTokens": 37, "enTokens": 27, "ratio": 1.37 }, { "id": "education-02", "register": "education", "faTokens": 26, "enTokens": 24, "ratio": 1.083 }, { "id": "legal-01", "register": "legal", "faTokens": 35, "enTokens": 31, "ratio": 1.129 }, { "id": "legal-02", "register": "legal", "faTokens": 24, "enTokens": 21, "ratio": 1.143 }, { "id": "literary-01", "register": "literary", "faTokens": 39, "enTokens": 26, "ratio": 1.5 }, { "id": "literary-02", "register": "literary", "faTokens": 32, "enTokens": 20, "ratio": 1.6 }, { "id": "medical-01", "register": "medical", "faTokens": 32, "enTokens": 19, "ratio": 1.684 }, { "id": "instruction-01", "register": "instruction", "faTokens": 33, "enTokens": 23, "ratio": 1.435 }, { "id": "instruction-02", "register": "instruction", "faTokens": 31, "enTokens": 26, "ratio": 1.192 } ], "faTokens": 955, "enTokens": 698, "ratio": 1.368, "faTokensPerChar": 0.3116, "enTokensPerChar": 0.1943, "faCharsPerToken": 3.209, "rank": 4, "tied": false }, { "id": "llama-3", "label": "Llama 3", "vendor": "Meta", "repo": "Xenova/llama-3-tokenizer", "mirror": true, "persianSpecific": false, "note": "Weights repo is licence-gated; the tokenizer is not.", "revision": "72bff9ee09897a16b3b4b2b9995fecb0bfa7dbe6", "fileSha256": "c05a3c2174e9edd5be19dc5a0748c42a9037bec2811ce062728bfd71f8702d78", "measuredAt": "2026-08-19", "vocabSize": 128256, "fingerprint": "68ca3b8ce39c96b2", "items": [ { "id": "news-01", "register": "news", "faTokens": 40, "enTokens": 34, "ratio": 1.176 }, { "id": "news-02", "register": "news", "faTokens": 37, "enTokens": 34, "ratio": 1.088 }, { "id": "news-03", "register": "news", "faTokens": 37, "enTokens": 24, "ratio": 1.542 }, { "id": "technical-01", "register": "technical", "faTokens": 39, "enTokens": 29, "ratio": 1.345 }, { "id": "technical-02", "register": "technical", "faTokens": 35, "enTokens": 22, "ratio": 1.591 }, { "id": "technical-03", "register": "technical", "faTokens": 34, "enTokens": 24, "ratio": 1.417 }, { "id": "technical-04", "register": "technical", "faTokens": 36, "enTokens": 25, "ratio": 1.44 }, { "id": "policy-01", "register": "policy", "faTokens": 39, "enTokens": 25, "ratio": 1.56 }, { "id": "policy-02", "register": "policy", "faTokens": 36, "enTokens": 18, "ratio": 2 }, { "id": "conversational-01", "register": "conversational", "faTokens": 35, "enTokens": 25, "ratio": 1.4 }, { "id": "conversational-02", "register": "conversational", "faTokens": 29, "enTokens": 22, "ratio": 1.318 }, { "id": "conversational-03", "register": "conversational", "faTokens": 29, "enTokens": 25, "ratio": 1.16 }, { "id": "scientific-01", "register": "scientific", "faTokens": 47, "enTokens": 28, "ratio": 1.679 }, { "id": "scientific-02", "register": "scientific", "faTokens": 35, "enTokens": 25, "ratio": 1.4 }, { "id": "scientific-03", "register": "scientific", "faTokens": 36, "enTokens": 25, "ratio": 1.44 }, { "id": "product-01", "register": "product", "faTokens": 33, "enTokens": 24, "ratio": 1.375 }, { "id": "product-02", "register": "product", "faTokens": 36, "enTokens": 20, "ratio": 1.8 }, { "id": "finance-01", "register": "finance", "faTokens": 32, "enTokens": 24, "ratio": 1.333 }, { "id": "finance-02", "register": "finance", "faTokens": 30, "enTokens": 20, "ratio": 1.5 }, { "id": "education-01", "register": "education", "faTokens": 38, "enTokens": 26, "ratio": 1.462 }, { "id": "education-02", "register": "education", "faTokens": 23, "enTokens": 24, "ratio": 0.958 }, { "id": "legal-01", "register": "legal", "faTokens": 35, "enTokens": 31, "ratio": 1.129 }, { "id": "legal-02", "register": "legal", "faTokens": 27, "enTokens": 21, "ratio": 1.286 }, { "id": "literary-01", "register": "literary", "faTokens": 37, "enTokens": 26, "ratio": 1.423 }, { "id": "literary-02", "register": "literary", "faTokens": 29, "enTokens": 20, "ratio": 1.45 }, { "id": "medical-01", "register": "medical", "faTokens": 33, "enTokens": 19, "ratio": 1.737 }, { "id": "instruction-01", "register": "instruction", "faTokens": 32, "enTokens": 23, "ratio": 1.391 }, { "id": "instruction-02", "register": "instruction", "faTokens": 30, "enTokens": 26, "ratio": 1.154 } ], "faTokens": 959, "enTokens": 689, "ratio": 1.392, "faTokensPerChar": 0.3129, "enTokensPerChar": 0.1918, "faCharsPerToken": 3.196, "rank": 5, "tied": true }, { "id": "llama-3-1", "label": "Llama 3.1", "vendor": "Meta", "repo": "Xenova/Meta-Llama-3.1-Tokenizer", "mirror": true, "persianSpecific": false, "note": "Listed separately so an identical fingerprint to Llama 3 is a published finding rather than an assumption.", "revision": "d49ea33e96b3fcf8738bb79ff9d85da81742fa82", "fileSha256": "79e3e522635f3171300913bb421464a87de6222182a0570b9b2ccba2a964b2b4", "measuredAt": "2026-08-19", "vocabSize": 128256, "fingerprint": "68ca3b8ce39c96b2", "items": [ { "id": "news-01", "register": "news", "faTokens": 40, "enTokens": 34, "ratio": 1.176 }, { "id": "news-02", "register": "news", "faTokens": 37, "enTokens": 34, "ratio": 1.088 }, { "id": "news-03", "register": "news", "faTokens": 37, "enTokens": 24, "ratio": 1.542 }, { "id": "technical-01", "register": "technical", "faTokens": 39, "enTokens": 29, "ratio": 1.345 }, { "id": "technical-02", "register": "technical", "faTokens": 35, "enTokens": 22, "ratio": 1.591 }, { "id": "technical-03", "register": "technical", "faTokens": 34, "enTokens": 24, "ratio": 1.417 }, { "id": "technical-04", "register": "technical", "faTokens": 36, "enTokens": 25, "ratio": 1.44 }, { "id": "policy-01", "register": "policy", "faTokens": 39, "enTokens": 25, "ratio": 1.56 }, { "id": "policy-02", "register": "policy", "faTokens": 36, "enTokens": 18, "ratio": 2 }, { "id": "conversational-01", "register": "conversational", "faTokens": 35, "enTokens": 25, "ratio": 1.4 }, { "id": "conversational-02", "register": "conversational", "faTokens": 29, "enTokens": 22, "ratio": 1.318 }, { "id": "conversational-03", "register": "conversational", "faTokens": 29, "enTokens": 25, "ratio": 1.16 }, { "id": "scientific-01", "register": "scientific", "faTokens": 47, "enTokens": 28, "ratio": 1.679 }, { "id": "scientific-02", "register": "scientific", "faTokens": 35, "enTokens": 25, "ratio": 1.4 }, { "id": "scientific-03", "register": "scientific", "faTokens": 36, "enTokens": 25, "ratio": 1.44 }, { "id": "product-01", "register": "product", "faTokens": 33, "enTokens": 24, "ratio": 1.375 }, { "id": "product-02", "register": "product", "faTokens": 36, "enTokens": 20, "ratio": 1.8 }, { "id": "finance-01", "register": "finance", "faTokens": 32, "enTokens": 24, "ratio": 1.333 }, { "id": "finance-02", "register": "finance", "faTokens": 30, "enTokens": 20, "ratio": 1.5 }, { "id": "education-01", "register": "education", "faTokens": 38, "enTokens": 26, "ratio": 1.462 }, { "id": "education-02", "register": "education", "faTokens": 23, "enTokens": 24, "ratio": 0.958 }, { "id": "legal-01", "register": "legal", "faTokens": 35, "enTokens": 31, "ratio": 1.129 }, { "id": "legal-02", "register": "legal", "faTokens": 27, "enTokens": 21, "ratio": 1.286 }, { "id": "literary-01", "register": "literary", "faTokens": 37, "enTokens": 26, "ratio": 1.423 }, { "id": "literary-02", "register": "literary", "faTokens": 29, "enTokens": 20, "ratio": 1.45 }, { "id": "medical-01", "register": "medical", "faTokens": 33, "enTokens": 19, "ratio": 1.737 }, { "id": "instruction-01", "register": "instruction", "faTokens": 32, "enTokens": 23, "ratio": 1.391 }, { "id": "instruction-02", "register": "instruction", "faTokens": 30, "enTokens": 26, "ratio": 1.154 } ], "faTokens": 959, "enTokens": 689, "ratio": 1.392, "faTokensPerChar": 0.3129, "enTokensPerChar": 0.1918, "faCharsPerToken": 3.196, "sameTokenizerAs": "llama-3", "rank": 5, "tied": true }, { "id": "gpt-4o", "label": "GPT-4o", "vendor": "OpenAI", "repo": "Xenova/gpt-4o", "mirror": true, "persianSpecific": false, "note": "o200k_base — also used by GPT-4.1 and o-series models.", "revision": "7956d98f2a83b2751a98ea7136fdf7fe6cf54e69", "fileSha256": "43a3ad4618a6a938f8c2614154ea10c31ad53a62d5683ae0b0e6133575cef07e", "measuredAt": "2026-08-19", "vocabSize": 200000, "fingerprint": "e4a1818eed0c8d85", "items": [ { "id": "news-01", "register": "news", "faTokens": 39, "enTokens": 34, "ratio": 1.147 }, { "id": "news-02", "register": "news", "faTokens": 38, "enTokens": 34, "ratio": 1.118 }, { "id": "news-03", "register": "news", "faTokens": 39, "enTokens": 24, "ratio": 1.625 }, { "id": "technical-01", "register": "technical", "faTokens": 38, "enTokens": 29, "ratio": 1.31 }, { "id": "technical-02", "register": "technical", "faTokens": 37, "enTokens": 22, "ratio": 1.682 }, { "id": "technical-03", "register": "technical", "faTokens": 36, "enTokens": 24, "ratio": 1.5 }, { "id": "technical-04", "register": "technical", "faTokens": 36, "enTokens": 25, "ratio": 1.44 }, { "id": "policy-01", "register": "policy", "faTokens": 39, "enTokens": 25, "ratio": 1.56 }, { "id": "policy-02", "register": "policy", "faTokens": 36, "enTokens": 18, "ratio": 2 }, { "id": "conversational-01", "register": "conversational", "faTokens": 31, "enTokens": 25, "ratio": 1.24 }, { "id": "conversational-02", "register": "conversational", "faTokens": 28, "enTokens": 23, "ratio": 1.217 }, { "id": "conversational-03", "register": "conversational", "faTokens": 29, "enTokens": 25, "ratio": 1.16 }, { "id": "scientific-01", "register": "scientific", "faTokens": 47, "enTokens": 28, "ratio": 1.679 }, { "id": "scientific-02", "register": "scientific", "faTokens": 35, "enTokens": 25, "ratio": 1.4 }, { "id": "scientific-03", "register": "scientific", "faTokens": 37, "enTokens": 25, "ratio": 1.48 }, { "id": "product-01", "register": "product", "faTokens": 33, "enTokens": 24, "ratio": 1.375 }, { "id": "product-02", "register": "product", "faTokens": 35, "enTokens": 20, "ratio": 1.75 }, { "id": "finance-01", "register": "finance", "faTokens": 32, "enTokens": 24, "ratio": 1.333 }, { "id": "finance-02", "register": "finance", "faTokens": 31, "enTokens": 20, "ratio": 1.55 }, { "id": "education-01", "register": "education", "faTokens": 36, "enTokens": 26, "ratio": 1.385 }, { "id": "education-02", "register": "education", "faTokens": 26, "enTokens": 24, "ratio": 1.083 }, { "id": "legal-01", "register": "legal", "faTokens": 34, "enTokens": 30, "ratio": 1.133 }, { "id": "legal-02", "register": "legal", "faTokens": 25, "enTokens": 21, "ratio": 1.19 }, { "id": "literary-01", "register": "literary", "faTokens": 38, "enTokens": 26, "ratio": 1.462 }, { "id": "literary-02", "register": "literary", "faTokens": 34, "enTokens": 20, "ratio": 1.7 }, { "id": "medical-01", "register": "medical", "faTokens": 33, "enTokens": 19, "ratio": 1.737 }, { "id": "instruction-01", "register": "instruction", "faTokens": 34, "enTokens": 23, "ratio": 1.478 }, { "id": "instruction-02", "register": "instruction", "faTokens": 33, "enTokens": 26, "ratio": 1.269 } ], "faTokens": 969, "enTokens": 689, "ratio": 1.406, "faTokensPerChar": 0.3162, "enTokensPerChar": 0.1918, "faCharsPerToken": 3.163, "rank": 7, "tied": true }, { "id": "gpt-oss-20b", "label": "gpt-oss 20B", "vendor": "OpenAI", "repo": "openai/gpt-oss-20b", "mirror": false, "persianSpecific": false, "note": "Open-weights, published by the vendor.", "revision": "6cee5e81ee83917806bbde320786a8fb61efebee", "fileSha256": "0614fe83cadab421296e664e1f48f4261fa8fef6e03e63bb75c20f38e37d07d3", "measuredAt": "2026-08-19", "vocabSize": 200019, "fingerprint": "f5c4cab9908b0d33", "items": [ { "id": "news-01", "register": "news", "faTokens": 39, "enTokens": 34, "ratio": 1.147 }, { "id": "news-02", "register": "news", "faTokens": 38, "enTokens": 34, "ratio": 1.118 }, { "id": "news-03", "register": "news", "faTokens": 39, "enTokens": 24, "ratio": 1.625 }, { "id": "technical-01", "register": "technical", "faTokens": 38, "enTokens": 29, "ratio": 1.31 }, { "id": "technical-02", "register": "technical", "faTokens": 37, "enTokens": 22, "ratio": 1.682 }, { "id": "technical-03", "register": "technical", "faTokens": 36, "enTokens": 24, "ratio": 1.5 }, { "id": "technical-04", "register": "technical", "faTokens": 36, "enTokens": 25, "ratio": 1.44 }, { "id": "policy-01", "register": "policy", "faTokens": 39, "enTokens": 25, "ratio": 1.56 }, { "id": "policy-02", "register": "policy", "faTokens": 36, "enTokens": 18, "ratio": 2 }, { "id": "conversational-01", "register": "conversational", "faTokens": 31, "enTokens": 25, "ratio": 1.24 }, { "id": "conversational-02", "register": "conversational", "faTokens": 28, "enTokens": 23, "ratio": 1.217 }, { "id": "conversational-03", "register": "conversational", "faTokens": 29, "enTokens": 25, "ratio": 1.16 }, { "id": "scientific-01", "register": "scientific", "faTokens": 47, "enTokens": 28, "ratio": 1.679 }, { "id": "scientific-02", "register": "scientific", "faTokens": 35, "enTokens": 25, "ratio": 1.4 }, { "id": "scientific-03", "register": "scientific", "faTokens": 37, "enTokens": 25, "ratio": 1.48 }, { "id": "product-01", "register": "product", "faTokens": 33, "enTokens": 24, "ratio": 1.375 }, { "id": "product-02", "register": "product", "faTokens": 35, "enTokens": 20, "ratio": 1.75 }, { "id": "finance-01", "register": "finance", "faTokens": 32, "enTokens": 24, "ratio": 1.333 }, { "id": "finance-02", "register": "finance", "faTokens": 31, "enTokens": 20, "ratio": 1.55 }, { "id": "education-01", "register": "education", "faTokens": 36, "enTokens": 26, "ratio": 1.385 }, { "id": "education-02", "register": "education", "faTokens": 26, "enTokens": 24, "ratio": 1.083 }, { "id": "legal-01", "register": "legal", "faTokens": 34, "enTokens": 30, "ratio": 1.133 }, { "id": "legal-02", "register": "legal", "faTokens": 25, "enTokens": 21, "ratio": 1.19 }, { "id": "literary-01", "register": "literary", "faTokens": 38, "enTokens": 26, "ratio": 1.462 }, { "id": "literary-02", "register": "literary", "faTokens": 34, "enTokens": 20, "ratio": 1.7 }, { "id": "medical-01", "register": "medical", "faTokens": 33, "enTokens": 19, "ratio": 1.737 }, { "id": "instruction-01", "register": "instruction", "faTokens": 34, "enTokens": 23, "ratio": 1.478 }, { "id": "instruction-02", "register": "instruction", "faTokens": 33, "enTokens": 26, "ratio": 1.269 } ], "faTokens": 969, "enTokens": 689, "ratio": 1.406, "faTokensPerChar": 0.3162, "enTokensPerChar": 0.1918, "faCharsPerToken": 3.163, "rank": 7, "tied": true }, { "id": "glm-4-5", "label": "GLM-4.5", "vendor": "Z.ai", "repo": "zai-org/GLM-4.5", "mirror": false, "persianSpecific": false, "note": "", "revision": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "fileSha256": "9340665016419c825c4bdabbcc9acc43b7ca2c68ce142724afa829abb1be5efd", "measuredAt": "2026-08-19", "vocabSize": 151365, "fingerprint": "3c4bed0010b8e7ab", "items": [ { "id": "news-01", "register": "news", "faTokens": 51, "enTokens": 34, "ratio": 1.5 }, { "id": "news-02", "register": "news", "faTokens": 47, "enTokens": 34, "ratio": 1.382 }, { "id": "news-03", "register": "news", "faTokens": 46, "enTokens": 24, "ratio": 1.917 }, { "id": "technical-01", "register": "technical", "faTokens": 51, "enTokens": 29, "ratio": 1.759 }, { "id": "technical-02", "register": "technical", "faTokens": 44, "enTokens": 22, "ratio": 2 }, { "id": "technical-03", "register": "technical", "faTokens": 46, "enTokens": 24, "ratio": 1.917 }, { "id": "technical-04", "register": "technical", "faTokens": 46, "enTokens": 25, "ratio": 1.84 }, { "id": "policy-01", "register": "policy", "faTokens": 47, "enTokens": 25, "ratio": 1.88 }, { "id": "policy-02", "register": "policy", "faTokens": 45, "enTokens": 18, "ratio": 2.5 }, { "id": "conversational-01", "register": "conversational", "faTokens": 45, "enTokens": 25, "ratio": 1.8 }, { "id": "conversational-02", "register": "conversational", "faTokens": 35, "enTokens": 22, "ratio": 1.591 }, { "id": "conversational-03", "register": "conversational", "faTokens": 35, "enTokens": 25, "ratio": 1.4 }, { "id": "scientific-01", "register": "scientific", "faTokens": 53, "enTokens": 28, "ratio": 1.893 }, { "id": "scientific-02", "register": "scientific", "faTokens": 43, "enTokens": 25, "ratio": 1.72 }, { "id": "scientific-03", "register": "scientific", "faTokens": 46, "enTokens": 25, "ratio": 1.84 }, { "id": "product-01", "register": "product", "faTokens": 35, "enTokens": 24, "ratio": 1.458 }, { "id": "product-02", "register": "product", "faTokens": 41, "enTokens": 20, "ratio": 2.05 }, { "id": "finance-01", "register": "finance", "faTokens": 42, "enTokens": 24, "ratio": 1.75 }, { "id": "finance-02", "register": "finance", "faTokens": 35, "enTokens": 20, "ratio": 1.75 }, { "id": "education-01", "register": "education", "faTokens": 49, "enTokens": 26, "ratio": 1.885 }, { "id": "education-02", "register": "education", "faTokens": 32, "enTokens": 24, "ratio": 1.333 }, { "id": "legal-01", "register": "legal", "faTokens": 45, "enTokens": 31, "ratio": 1.452 }, { "id": "legal-02", "register": "legal", "faTokens": 29, "enTokens": 21, "ratio": 1.381 }, { "id": "literary-01", "register": "literary", "faTokens": 46, "enTokens": 26, "ratio": 1.769 }, { "id": "literary-02", "register": "literary", "faTokens": 39, "enTokens": 20, "ratio": 1.95 }, { "id": "medical-01", "register": "medical", "faTokens": 37, "enTokens": 19, "ratio": 1.947 }, { "id": "instruction-01", "register": "instruction", "faTokens": 45, "enTokens": 23, "ratio": 1.957 }, { "id": "instruction-02", "register": "instruction", "faTokens": 36, "enTokens": 26, "ratio": 1.385 } ], "faTokens": 1191, "enTokens": 689, "ratio": 1.729, "faTokensPerChar": 0.3886, "enTokensPerChar": 0.1918, "faCharsPerToken": 2.573, "rank": 9, "tied": false }, { "id": "deepseek-v3", "label": "DeepSeek-V3", "vendor": "DeepSeek", "repo": "deepseek-ai/DeepSeek-V3", "mirror": false, "persianSpecific": false, "note": "", "revision": "e815299b0bcbac849fa540c768ef21845365c9eb", "fileSha256": "621ac2e32d0dba658404412318818aaa8ce8cda492e59830109d8da6b517fb41", "measuredAt": "2026-08-19", "vocabSize": 128815, "fingerprint": "b49c0fe0d563eb09", "items": [ { "id": "news-01", "register": "news", "faTokens": 54, "enTokens": 34, "ratio": 1.588 }, { "id": "news-02", "register": "news", "faTokens": 46, "enTokens": 34, "ratio": 1.353 }, { "id": "news-03", "register": "news", "faTokens": 49, "enTokens": 24, "ratio": 2.042 }, { "id": "technical-01", "register": "technical", "faTokens": 51, "enTokens": 29, "ratio": 1.759 }, { "id": "technical-02", "register": "technical", "faTokens": 42, "enTokens": 23, "ratio": 1.826 }, { "id": "technical-03", "register": "technical", "faTokens": 50, "enTokens": 24, "ratio": 2.083 }, { "id": "technical-04", "register": "technical", "faTokens": 49, "enTokens": 25, "ratio": 1.96 }, { "id": "policy-01", "register": "policy", "faTokens": 55, "enTokens": 26, "ratio": 2.115 }, { "id": "policy-02", "register": "policy", "faTokens": 48, "enTokens": 18, "ratio": 2.667 }, { "id": "conversational-01", "register": "conversational", "faTokens": 42, "enTokens": 25, "ratio": 1.68 }, { "id": "conversational-02", "register": "conversational", "faTokens": 37, "enTokens": 22, "ratio": 1.682 }, { "id": "conversational-03", "register": "conversational", "faTokens": 35, "enTokens": 25, "ratio": 1.4 }, { "id": "scientific-01", "register": "scientific", "faTokens": 56, "enTokens": 30, "ratio": 1.867 }, { "id": "scientific-02", "register": "scientific", "faTokens": 40, "enTokens": 25, "ratio": 1.6 }, { "id": "scientific-03", "register": "scientific", "faTokens": 52, "enTokens": 25, "ratio": 2.08 }, { "id": "product-01", "register": "product", "faTokens": 34, "enTokens": 24, "ratio": 1.417 }, { "id": "product-02", "register": "product", "faTokens": 43, "enTokens": 20, "ratio": 2.15 }, { "id": "finance-01", "register": "finance", "faTokens": 42, "enTokens": 24, "ratio": 1.75 }, { "id": "finance-02", "register": "finance", "faTokens": 33, "enTokens": 20, "ratio": 1.65 }, { "id": "education-01", "register": "education", "faTokens": 44, "enTokens": 26, "ratio": 1.692 }, { "id": "education-02", "register": "education", "faTokens": 29, "enTokens": 24, "ratio": 1.208 }, { "id": "legal-01", "register": "legal", "faTokens": 43, "enTokens": 31, "ratio": 1.387 }, { "id": "legal-02", "register": "legal", "faTokens": 29, "enTokens": 21, "ratio": 1.381 }, { "id": "literary-01", "register": "literary", "faTokens": 48, "enTokens": 26, "ratio": 1.846 }, { "id": "literary-02", "register": "literary", "faTokens": 40, "enTokens": 20, "ratio": 2 }, { "id": "medical-01", "register": "medical", "faTokens": 35, "enTokens": 19, "ratio": 1.842 }, { "id": "instruction-01", "register": "instruction", "faTokens": 42, "enTokens": 23, "ratio": 1.826 }, { "id": "instruction-02", "register": "instruction", "faTokens": 35, "enTokens": 26, "ratio": 1.346 } ], "faTokens": 1203, "enTokens": 693, "ratio": 1.736, "faTokensPerChar": 0.3925, "enTokensPerChar": 0.1929, "faCharsPerToken": 2.548, "rank": 10, "tied": false }, { "id": "qwen-2-5", "label": "Qwen2.5", "vendor": "Alibaba", "repo": "Qwen/Qwen2.5-7B-Instruct", "mirror": false, "persianSpecific": false, "note": "", "revision": "a09a35458c702b33eeacc393d103063234e8bc28", "fileSha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", "measuredAt": "2026-08-19", "vocabSize": 151665, "fingerprint": "ab095f79e58ac767", "items": [ { "id": "news-01", "register": "news", "faTokens": 87, "enTokens": 34, "ratio": 2.559 }, { "id": "news-02", "register": "news", "faTokens": 80, "enTokens": 34, "ratio": 2.353 }, { "id": "news-03", "register": "news", "faTokens": 79, "enTokens": 24, "ratio": 3.292 }, { "id": "technical-01", "register": "technical", "faTokens": 81, "enTokens": 29, "ratio": 2.793 }, { "id": "technical-02", "register": "technical", "faTokens": 68, "enTokens": 22, "ratio": 3.091 }, { "id": "technical-03", "register": "technical", "faTokens": 69, "enTokens": 24, "ratio": 2.875 }, { "id": "technical-04", "register": "technical", "faTokens": 67, "enTokens": 25, "ratio": 2.68 }, { "id": "policy-01", "register": "policy", "faTokens": 69, "enTokens": 25, "ratio": 2.76 }, { "id": "policy-02", "register": "policy", "faTokens": 65, "enTokens": 18, "ratio": 3.611 }, { "id": "conversational-01", "register": "conversational", "faTokens": 55, "enTokens": 25, "ratio": 2.2 }, { "id": "conversational-02", "register": "conversational", "faTokens": 46, "enTokens": 22, "ratio": 2.091 }, { "id": "conversational-03", "register": "conversational", "faTokens": 53, "enTokens": 25, "ratio": 2.12 }, { "id": "scientific-01", "register": "scientific", "faTokens": 81, "enTokens": 28, "ratio": 2.893 }, { "id": "scientific-02", "register": "scientific", "faTokens": 56, "enTokens": 25, "ratio": 2.24 }, { "id": "scientific-03", "register": "scientific", "faTokens": 75, "enTokens": 25, "ratio": 3 }, { "id": "product-01", "register": "product", "faTokens": 59, "enTokens": 24, "ratio": 2.458 }, { "id": "product-02", "register": "product", "faTokens": 63, "enTokens": 20, "ratio": 3.15 }, { "id": "finance-01", "register": "finance", "faTokens": 62, "enTokens": 24, "ratio": 2.583 }, { "id": "finance-02", "register": "finance", "faTokens": 60, "enTokens": 20, "ratio": 3 }, { "id": "education-01", "register": "education", "faTokens": 66, "enTokens": 26, "ratio": 2.538 }, { "id": "education-02", "register": "education", "faTokens": 50, "enTokens": 24, "ratio": 2.083 }, { "id": "legal-01", "register": "legal", "faTokens": 66, "enTokens": 31, "ratio": 2.129 }, { "id": "legal-02", "register": "legal", "faTokens": 40, "enTokens": 21, "ratio": 1.905 }, { "id": "literary-01", "register": "literary", "faTokens": 63, "enTokens": 26, "ratio": 2.423 }, { "id": "literary-02", "register": "literary", "faTokens": 49, "enTokens": 20, "ratio": 2.45 }, { "id": "medical-01", "register": "medical", "faTokens": 56, "enTokens": 19, "ratio": 2.947 }, { "id": "instruction-01", "register": "instruction", "faTokens": 63, "enTokens": 23, "ratio": 2.739 }, { "id": "instruction-02", "register": "instruction", "faTokens": 49, "enTokens": 26, "ratio": 1.885 } ], "faTokens": 1777, "enTokens": 689, "ratio": 2.579, "faTokensPerChar": 0.5798, "enTokensPerChar": 0.1918, "faCharsPerToken": 1.725, "rank": 11, "tied": true }, { "id": "qwen-3", "label": "Qwen3", "vendor": "Alibaba", "repo": "Qwen/Qwen3-8B", "mirror": false, "persianSpecific": false, "note": "", "revision": "b968826d9c46dd6066d109eabc6255188de91218", "fileSha256": "aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4", "measuredAt": "2026-08-19", "vocabSize": 151669, "fingerprint": "ab095f79e58ac767", "items": [ { "id": "news-01", "register": "news", "faTokens": 87, "enTokens": 34, "ratio": 2.559 }, { "id": "news-02", "register": "news", "faTokens": 80, "enTokens": 34, "ratio": 2.353 }, { "id": "news-03", "register": "news", "faTokens": 79, "enTokens": 24, "ratio": 3.292 }, { "id": "technical-01", "register": "technical", "faTokens": 81, "enTokens": 29, "ratio": 2.793 }, { "id": "technical-02", "register": "technical", "faTokens": 68, "enTokens": 22, "ratio": 3.091 }, { "id": "technical-03", "register": "technical", "faTokens": 69, "enTokens": 24, "ratio": 2.875 }, { "id": "technical-04", "register": "technical", "faTokens": 67, "enTokens": 25, "ratio": 2.68 }, { "id": "policy-01", "register": "policy", "faTokens": 69, "enTokens": 25, "ratio": 2.76 }, { "id": "policy-02", "register": "policy", "faTokens": 65, "enTokens": 18, "ratio": 3.611 }, { "id": "conversational-01", "register": "conversational", "faTokens": 55, "enTokens": 25, "ratio": 2.2 }, { "id": "conversational-02", "register": "conversational", "faTokens": 46, "enTokens": 22, "ratio": 2.091 }, { "id": "conversational-03", "register": "conversational", "faTokens": 53, "enTokens": 25, "ratio": 2.12 }, { "id": "scientific-01", "register": "scientific", "faTokens": 81, "enTokens": 28, "ratio": 2.893 }, { "id": "scientific-02", "register": "scientific", "faTokens": 56, "enTokens": 25, "ratio": 2.24 }, { "id": "scientific-03", "register": "scientific", "faTokens": 75, "enTokens": 25, "ratio": 3 }, { "id": "product-01", "register": "product", "faTokens": 59, "enTokens": 24, "ratio": 2.458 }, { "id": "product-02", "register": "product", "faTokens": 63, "enTokens": 20, "ratio": 3.15 }, { "id": "finance-01", "register": "finance", "faTokens": 62, "enTokens": 24, "ratio": 2.583 }, { "id": "finance-02", "register": "finance", "faTokens": 60, "enTokens": 20, "ratio": 3 }, { "id": "education-01", "register": "education", "faTokens": 66, "enTokens": 26, "ratio": 2.538 }, { "id": "education-02", "register": "education", "faTokens": 50, "enTokens": 24, "ratio": 2.083 }, { "id": "legal-01", "register": "legal", "faTokens": 66, "enTokens": 31, "ratio": 2.129 }, { "id": "legal-02", "register": "legal", "faTokens": 40, "enTokens": 21, "ratio": 1.905 }, { "id": "literary-01", "register": "literary", "faTokens": 63, "enTokens": 26, "ratio": 2.423 }, { "id": "literary-02", "register": "literary", "faTokens": 49, "enTokens": 20, "ratio": 2.45 }, { "id": "medical-01", "register": "medical", "faTokens": 56, "enTokens": 19, "ratio": 2.947 }, { "id": "instruction-01", "register": "instruction", "faTokens": 63, "enTokens": 23, "ratio": 2.739 }, { "id": "instruction-02", "register": "instruction", "faTokens": 49, "enTokens": 26, "ratio": 1.885 } ], "faTokens": 1777, "enTokens": 689, "ratio": 2.579, "faTokensPerChar": 0.5798, "enTokensPerChar": 0.1918, "faCharsPerToken": 1.725, "sameTokenizerAs": "qwen-2-5", "rank": 11, "tied": true }, { "id": "gpt-4", "label": "GPT-4 / GPT-3.5 Turbo", "vendor": "OpenAI", "repo": "Xenova/gpt-4", "mirror": true, "persianSpecific": false, "note": "cl100k_base — the previous generation, kept for the size of the change.", "revision": "1d9f1f1b1fae88c0e4df1dab0a397f8de6229075", "fileSha256": "239eb2359f79c38497476671aaa835e01fb43d42743c612a8514a0dfa2ac93a2", "measuredAt": "2026-08-19", "vocabSize": 100263, "fingerprint": "390a47963496e4ba", "items": [ { "id": "news-01", "register": "news", "faTokens": 104, "enTokens": 34, "ratio": 3.059 }, { "id": "news-02", "register": "news", "faTokens": 108, "enTokens": 34, "ratio": 3.176 }, { "id": "news-03", "register": "news", "faTokens": 96, "enTokens": 24, "ratio": 4 }, { "id": "technical-01", "register": "technical", "faTokens": 94, "enTokens": 29, "ratio": 3.241 }, { "id": "technical-02", "register": "technical", "faTokens": 77, "enTokens": 22, "ratio": 3.5 }, { "id": "technical-03", "register": "technical", "faTokens": 85, "enTokens": 24, "ratio": 3.542 }, { "id": "technical-04", "register": "technical", "faTokens": 78, "enTokens": 25, "ratio": 3.12 }, { "id": "policy-01", "register": "policy", "faTokens": 88, "enTokens": 25, "ratio": 3.52 }, { "id": "policy-02", "register": "policy", "faTokens": 82, "enTokens": 18, "ratio": 4.556 }, { "id": "conversational-01", "register": "conversational", "faTokens": 73, "enTokens": 25, "ratio": 2.92 }, { "id": "conversational-02", "register": "conversational", "faTokens": 64, "enTokens": 22, "ratio": 2.909 }, { "id": "conversational-03", "register": "conversational", "faTokens": 66, "enTokens": 25, "ratio": 2.64 }, { "id": "scientific-01", "register": "scientific", "faTokens": 100, "enTokens": 28, "ratio": 3.571 }, { "id": "scientific-02", "register": "scientific", "faTokens": 71, "enTokens": 25, "ratio": 2.84 }, { "id": "scientific-03", "register": "scientific", "faTokens": 90, "enTokens": 25, "ratio": 3.6 }, { "id": "product-01", "register": "product", "faTokens": 67, "enTokens": 24, "ratio": 2.792 }, { "id": "product-02", "register": "product", "faTokens": 72, "enTokens": 20, "ratio": 3.6 }, { "id": "finance-01", "register": "finance", "faTokens": 80, "enTokens": 24, "ratio": 3.333 }, { "id": "finance-02", "register": "finance", "faTokens": 77, "enTokens": 20, "ratio": 3.85 }, { "id": "education-01", "register": "education", "faTokens": 85, "enTokens": 26, "ratio": 3.269 }, { "id": "education-02", "register": "education", "faTokens": 61, "enTokens": 24, "ratio": 2.542 }, { "id": "legal-01", "register": "legal", "faTokens": 85, "enTokens": 31, "ratio": 2.742 }, { "id": "legal-02", "register": "legal", "faTokens": 57, "enTokens": 21, "ratio": 2.714 }, { "id": "literary-01", "register": "literary", "faTokens": 83, "enTokens": 26, "ratio": 3.192 }, { "id": "literary-02", "register": "literary", "faTokens": 66, "enTokens": 20, "ratio": 3.3 }, { "id": "medical-01", "register": "medical", "faTokens": 71, "enTokens": 19, "ratio": 3.737 }, { "id": "instruction-01", "register": "instruction", "faTokens": 79, "enTokens": 23, "ratio": 3.435 }, { "id": "instruction-02", "register": "instruction", "faTokens": 57, "enTokens": 26, "ratio": 2.192 } ], "faTokens": 2216, "enTokens": 689, "ratio": 3.216, "faTokensPerChar": 0.723, "enTokensPerChar": 0.1918, "faCharsPerToken": 1.383, "rank": 13, "tied": true }, { "id": "phi-4", "label": "Phi-4", "vendor": "Microsoft", "repo": "microsoft/phi-4", "mirror": false, "persianSpecific": false, "note": "", "revision": "2db69c1c3e91a05d2c64a3185acfbaf36f744e25", "fileSha256": "9f38d05d9d25756bb2f181ab5a0cebcd59e638df10336fc7ed1010f7296d0298", "measuredAt": "2026-08-19", "vocabSize": 100352, "fingerprint": "66e60352a989976f", "items": [ { "id": "news-01", "register": "news", "faTokens": 104, "enTokens": 34, "ratio": 3.059 }, { "id": "news-02", "register": "news", "faTokens": 108, "enTokens": 34, "ratio": 3.176 }, { "id": "news-03", "register": "news", "faTokens": 96, "enTokens": 24, "ratio": 4 }, { "id": "technical-01", "register": "technical", "faTokens": 94, "enTokens": 29, "ratio": 3.241 }, { "id": "technical-02", "register": "technical", "faTokens": 77, "enTokens": 22, "ratio": 3.5 }, { "id": "technical-03", "register": "technical", "faTokens": 85, "enTokens": 24, "ratio": 3.542 }, { "id": "technical-04", "register": "technical", "faTokens": 78, "enTokens": 25, "ratio": 3.12 }, { "id": "policy-01", "register": "policy", "faTokens": 88, "enTokens": 25, "ratio": 3.52 }, { "id": "policy-02", "register": "policy", "faTokens": 82, "enTokens": 18, "ratio": 4.556 }, { "id": "conversational-01", "register": "conversational", "faTokens": 73, "enTokens": 25, "ratio": 2.92 }, { "id": "conversational-02", "register": "conversational", "faTokens": 64, "enTokens": 22, "ratio": 2.909 }, { "id": "conversational-03", "register": "conversational", "faTokens": 66, "enTokens": 25, "ratio": 2.64 }, { "id": "scientific-01", "register": "scientific", "faTokens": 100, "enTokens": 28, "ratio": 3.571 }, { "id": "scientific-02", "register": "scientific", "faTokens": 71, "enTokens": 25, "ratio": 2.84 }, { "id": "scientific-03", "register": "scientific", "faTokens": 90, "enTokens": 25, "ratio": 3.6 }, { "id": "product-01", "register": "product", "faTokens": 67, "enTokens": 24, "ratio": 2.792 }, { "id": "product-02", "register": "product", "faTokens": 72, "enTokens": 20, "ratio": 3.6 }, { "id": "finance-01", "register": "finance", "faTokens": 80, "enTokens": 24, "ratio": 3.333 }, { "id": "finance-02", "register": "finance", "faTokens": 77, "enTokens": 20, "ratio": 3.85 }, { "id": "education-01", "register": "education", "faTokens": 85, "enTokens": 26, "ratio": 3.269 }, { "id": "education-02", "register": "education", "faTokens": 61, "enTokens": 24, "ratio": 2.542 }, { "id": "legal-01", "register": "legal", "faTokens": 85, "enTokens": 31, "ratio": 2.742 }, { "id": "legal-02", "register": "legal", "faTokens": 57, "enTokens": 21, "ratio": 2.714 }, { "id": "literary-01", "register": "literary", "faTokens": 83, "enTokens": 26, "ratio": 3.192 }, { "id": "literary-02", "register": "literary", "faTokens": 66, "enTokens": 20, "ratio": 3.3 }, { "id": "medical-01", "register": "medical", "faTokens": 71, "enTokens": 19, "ratio": 3.737 }, { "id": "instruction-01", "register": "instruction", "faTokens": 79, "enTokens": 23, "ratio": 3.435 }, { "id": "instruction-02", "register": "instruction", "faTokens": 57, "enTokens": 26, "ratio": 2.192 } ], "faTokens": 2216, "enTokens": 689, "ratio": 3.216, "faTokensPerChar": 0.723, "enTokensPerChar": 0.1918, "faCharsPerToken": 1.383, "rank": 13, "tied": true }, { "id": "olmo-2", "label": "OLMo 2", "vendor": "Allen Institute for AI", "repo": "allenai/OLMo-2-1124-7B", "mirror": false, "persianSpecific": false, "note": "Fully open training data, which makes its Persian coverage inspectable.", "revision": "7df9a82518afdecae4e8c026b27adccc8c1f0032", "fileSha256": "73fd5254624f39a88e3faac6a8e11300fc3c735ed37880d4f4f08db898eaecca", "measuredAt": "2026-08-19", "vocabSize": 100278, "fingerprint": "ab742b330b7b43f7", "items": [ { "id": "news-01", "register": "news", "faTokens": 104, "enTokens": 34, "ratio": 3.059 }, { "id": "news-02", "register": "news", "faTokens": 108, "enTokens": 34, "ratio": 3.176 }, { "id": "news-03", "register": "news", "faTokens": 96, "enTokens": 24, "ratio": 4 }, { "id": "technical-01", "register": "technical", "faTokens": 94, "enTokens": 29, "ratio": 3.241 }, { "id": "technical-02", "register": "technical", "faTokens": 77, "enTokens": 22, "ratio": 3.5 }, { "id": "technical-03", "register": "technical", "faTokens": 85, "enTokens": 24, "ratio": 3.542 }, { "id": "technical-04", "register": "technical", "faTokens": 78, "enTokens": 25, "ratio": 3.12 }, { "id": "policy-01", "register": "policy", "faTokens": 88, "enTokens": 25, "ratio": 3.52 }, { "id": "policy-02", "register": "policy", "faTokens": 82, "enTokens": 18, "ratio": 4.556 }, { "id": "conversational-01", "register": "conversational", "faTokens": 73, "enTokens": 25, "ratio": 2.92 }, { "id": "conversational-02", "register": "conversational", "faTokens": 64, "enTokens": 22, "ratio": 2.909 }, { "id": "conversational-03", "register": "conversational", "faTokens": 66, "enTokens": 25, "ratio": 2.64 }, { "id": "scientific-01", "register": "scientific", "faTokens": 100, "enTokens": 28, "ratio": 3.571 }, { "id": "scientific-02", "register": "scientific", "faTokens": 71, "enTokens": 25, "ratio": 2.84 }, { "id": "scientific-03", "register": "scientific", "faTokens": 90, "enTokens": 25, "ratio": 3.6 }, { "id": "product-01", "register": "product", "faTokens": 67, "enTokens": 24, "ratio": 2.792 }, { "id": "product-02", "register": "product", "faTokens": 72, "enTokens": 20, "ratio": 3.6 }, { "id": "finance-01", "register": "finance", "faTokens": 80, "enTokens": 24, "ratio": 3.333 }, { "id": "finance-02", "register": "finance", "faTokens": 77, "enTokens": 20, "ratio": 3.85 }, { "id": "education-01", "register": "education", "faTokens": 85, "enTokens": 26, "ratio": 3.269 }, { "id": "education-02", "register": "education", "faTokens": 61, "enTokens": 24, "ratio": 2.542 }, { "id": "legal-01", "register": "legal", "faTokens": 85, "enTokens": 31, "ratio": 2.742 }, { "id": "legal-02", "register": "legal", "faTokens": 57, "enTokens": 21, "ratio": 2.714 }, { "id": "literary-01", "register": "literary", "faTokens": 83, "enTokens": 26, "ratio": 3.192 }, { "id": "literary-02", "register": "literary", "faTokens": 66, "enTokens": 20, "ratio": 3.3 }, { "id": "medical-01", "register": "medical", "faTokens": 71, "enTokens": 19, "ratio": 3.737 }, { "id": "instruction-01", "register": "instruction", "faTokens": 79, "enTokens": 23, "ratio": 3.435 }, { "id": "instruction-02", "register": "instruction", "faTokens": 57, "enTokens": 26, "ratio": 2.192 } ], "faTokens": 2216, "enTokens": 689, "ratio": 3.216, "faTokensPerChar": 0.723, "enTokensPerChar": 0.1918, "faCharsPerToken": 1.383, "rank": 13, "tied": true }, { "id": "mistral-v3", "label": "Mistral v3", "vendor": "Mistral AI", "repo": "Xenova/mistral-tokenizer-v3", "mirror": true, "persianSpecific": false, "note": "32k SentencePiece vocabulary.", "revision": "b251f946a5b6b16c36963ea8848611bce67dfbf1", "fileSha256": "e553af6fff7d7ad76e830608b218c5c0b0822998d5a1a96099a74cd3c1cb1a49", "measuredAt": "2026-08-19", "vocabSize": 32768, "fingerprint": "c557e2b72ac82990", "items": [ { "id": "news-01", "register": "news", "faTokens": 147, "enTokens": 34, "ratio": 4.324 }, { "id": "news-02", "register": "news", "faTokens": 141, "enTokens": 36, "ratio": 3.917 }, { "id": "news-03", "register": "news", "faTokens": 123, "enTokens": 27, "ratio": 4.556 }, { "id": "technical-01", "register": "technical", "faTokens": 126, "enTokens": 29, "ratio": 4.345 }, { "id": "technical-02", "register": "technical", "faTokens": 112, "enTokens": 24, "ratio": 4.667 }, { "id": "technical-03", "register": "technical", "faTokens": 120, "enTokens": 25, "ratio": 4.8 }, { "id": "technical-04", "register": "technical", "faTokens": 109, "enTokens": 27, "ratio": 4.037 }, { "id": "policy-01", "register": "policy", "faTokens": 119, "enTokens": 29, "ratio": 4.103 }, { "id": "policy-02", "register": "policy", "faTokens": 119, "enTokens": 20, "ratio": 5.95 }, { "id": "conversational-01", "register": "conversational", "faTokens": 96, "enTokens": 26, "ratio": 3.692 }, { "id": "conversational-02", "register": "conversational", "faTokens": 79, "enTokens": 24, "ratio": 3.292 }, { "id": "conversational-03", "register": "conversational", "faTokens": 86, "enTokens": 25, "ratio": 3.44 }, { "id": "scientific-01", "register": "scientific", "faTokens": 132, "enTokens": 30, "ratio": 4.4 }, { "id": "scientific-02", "register": "scientific", "faTokens": 109, "enTokens": 27, "ratio": 4.037 }, { "id": "scientific-03", "register": "scientific", "faTokens": 123, "enTokens": 26, "ratio": 4.731 }, { "id": "product-01", "register": "product", "faTokens": 101, "enTokens": 24, "ratio": 4.208 }, { "id": "product-02", "register": "product", "faTokens": 100, "enTokens": 22, "ratio": 4.545 }, { "id": "finance-01", "register": "finance", "faTokens": 105, "enTokens": 24, "ratio": 4.375 }, { "id": "finance-02", "register": "finance", "faTokens": 100, "enTokens": 21, "ratio": 4.762 }, { "id": "education-01", "register": "education", "faTokens": 117, "enTokens": 28, "ratio": 4.179 }, { "id": "education-02", "register": "education", "faTokens": 86, "enTokens": 24, "ratio": 3.583 }, { "id": "legal-01", "register": "legal", "faTokens": 119, "enTokens": 33, "ratio": 3.606 }, { "id": "legal-02", "register": "legal", "faTokens": 82, "enTokens": 23, "ratio": 3.565 }, { "id": "literary-01", "register": "literary", "faTokens": 103, "enTokens": 26, "ratio": 3.962 }, { "id": "literary-02", "register": "literary", "faTokens": 83, "enTokens": 20, "ratio": 4.15 }, { "id": "medical-01", "register": "medical", "faTokens": 98, "enTokens": 19, "ratio": 5.158 }, { "id": "instruction-01", "register": "instruction", "faTokens": 102, "enTokens": 23, "ratio": 4.435 }, { "id": "instruction-02", "register": "instruction", "faTokens": 81, "enTokens": 27, "ratio": 3 } ], "faTokens": 3018, "enTokens": 723, "ratio": 4.174, "faTokensPerChar": 0.9847, "enTokensPerChar": 0.2013, "faCharsPerToken": 1.016, "rank": 16, "tied": true }, { "id": "maral-7b", "label": "Maral 7B", "vendor": "MaralGPT", "repo": "MaralGPT/Maral-7B-alpha-1", "mirror": false, "persianSpecific": true, "note": "A Persian community model built on Mistral.", "revision": "2ab5ca2a0d1a4454a78b4ca911e595bb9da2fe2f", "fileSha256": "fc4f0bd70b3709312d9d1d9e5ba674794b6bc5abc17429897a540f93882f25fc", "measuredAt": "2026-08-19", "vocabSize": 32000, "fingerprint": "3e48030f4beccfad", "items": [ { "id": "news-01", "register": "news", "faTokens": 147, "enTokens": 34, "ratio": 4.324 }, { "id": "news-02", "register": "news", "faTokens": 141, "enTokens": 36, "ratio": 3.917 }, { "id": "news-03", "register": "news", "faTokens": 123, "enTokens": 27, "ratio": 4.556 }, { "id": "technical-01", "register": "technical", "faTokens": 126, "enTokens": 29, "ratio": 4.345 }, { "id": "technical-02", "register": "technical", "faTokens": 112, "enTokens": 24, "ratio": 4.667 }, { "id": "technical-03", "register": "technical", "faTokens": 120, "enTokens": 25, "ratio": 4.8 }, { "id": "technical-04", "register": "technical", "faTokens": 109, "enTokens": 27, "ratio": 4.037 }, { "id": "policy-01", "register": "policy", "faTokens": 119, "enTokens": 29, "ratio": 4.103 }, { "id": "policy-02", "register": "policy", "faTokens": 119, "enTokens": 20, "ratio": 5.95 }, { "id": "conversational-01", "register": "conversational", "faTokens": 96, "enTokens": 26, "ratio": 3.692 }, { "id": "conversational-02", "register": "conversational", "faTokens": 79, "enTokens": 24, "ratio": 3.292 }, { "id": "conversational-03", "register": "conversational", "faTokens": 86, "enTokens": 25, "ratio": 3.44 }, { "id": "scientific-01", "register": "scientific", "faTokens": 132, "enTokens": 30, "ratio": 4.4 }, { "id": "scientific-02", "register": "scientific", "faTokens": 109, "enTokens": 27, "ratio": 4.037 }, { "id": "scientific-03", "register": "scientific", "faTokens": 123, "enTokens": 26, "ratio": 4.731 }, { "id": "product-01", "register": "product", "faTokens": 101, "enTokens": 24, "ratio": 4.208 }, { "id": "product-02", "register": "product", "faTokens": 100, "enTokens": 22, "ratio": 4.545 }, { "id": "finance-01", "register": "finance", "faTokens": 105, "enTokens": 24, "ratio": 4.375 }, { "id": "finance-02", "register": "finance", "faTokens": 100, "enTokens": 21, "ratio": 4.762 }, { "id": "education-01", "register": "education", "faTokens": 117, "enTokens": 28, "ratio": 4.179 }, { "id": "education-02", "register": "education", "faTokens": 86, "enTokens": 24, "ratio": 3.583 }, { "id": "legal-01", "register": "legal", "faTokens": 119, "enTokens": 33, "ratio": 3.606 }, { "id": "legal-02", "register": "legal", "faTokens": 82, "enTokens": 23, "ratio": 3.565 }, { "id": "literary-01", "register": "literary", "faTokens": 103, "enTokens": 26, "ratio": 3.962 }, { "id": "literary-02", "register": "literary", "faTokens": 83, "enTokens": 20, "ratio": 4.15 }, { "id": "medical-01", "register": "medical", "faTokens": 98, "enTokens": 19, "ratio": 5.158 }, { "id": "instruction-01", "register": "instruction", "faTokens": 102, "enTokens": 23, "ratio": 4.435 }, { "id": "instruction-02", "register": "instruction", "faTokens": 81, "enTokens": 27, "ratio": 3 } ], "faTokens": 3018, "enTokens": 723, "ratio": 4.174, "faTokensPerChar": 0.9847, "enTokensPerChar": 0.2013, "faCharsPerToken": 1.016, "rank": 16, "tied": true }, { "id": "claude", "label": "Claude", "vendor": "Anthropic", "repo": "Xenova/claude-tokenizer", "mirror": true, "persianSpecific": false, "note": "Anthropic publishes no tokenizer file; this is the community mirror in common use.", "revision": "cae688821ea05490de49a6d3faa36468a4672fad", "fileSha256": "c241737df24b4e7f7c9af4fdcee29a0ca903dcb288a8b753bc346a3092911767", "measuredAt": "2026-08-19", "vocabSize": 65000, "fingerprint": "4773fa1b568b99b8", "items": [ { "id": "news-01", "register": "news", "faTokens": 145, "enTokens": 34, "ratio": 4.265 }, { "id": "news-02", "register": "news", "faTokens": 146, "enTokens": 34, "ratio": 4.294 }, { "id": "news-03", "register": "news", "faTokens": 127, "enTokens": 25, "ratio": 5.08 }, { "id": "technical-01", "register": "technical", "faTokens": 129, "enTokens": 29, "ratio": 4.448 }, { "id": "technical-02", "register": "technical", "faTokens": 114, "enTokens": 23, "ratio": 4.957 }, { "id": "technical-03", "register": "technical", "faTokens": 120, "enTokens": 24, "ratio": 5 }, { "id": "technical-04", "register": "technical", "faTokens": 114, "enTokens": 26, "ratio": 4.385 }, { "id": "policy-01", "register": "policy", "faTokens": 121, "enTokens": 26, "ratio": 4.654 }, { "id": "policy-02", "register": "policy", "faTokens": 119, "enTokens": 18, "ratio": 6.611 }, { "id": "conversational-01", "register": "conversational", "faTokens": 95, "enTokens": 25, "ratio": 3.8 }, { "id": "conversational-02", "register": "conversational", "faTokens": 80, "enTokens": 22, "ratio": 3.636 }, { "id": "conversational-03", "register": "conversational", "faTokens": 86, "enTokens": 25, "ratio": 3.44 }, { "id": "scientific-01", "register": "scientific", "faTokens": 138, "enTokens": 30, "ratio": 4.6 }, { "id": "scientific-02", "register": "scientific", "faTokens": 105, "enTokens": 26, "ratio": 4.038 }, { "id": "scientific-03", "register": "scientific", "faTokens": 126, "enTokens": 25, "ratio": 5.04 }, { "id": "product-01", "register": "product", "faTokens": 101, "enTokens": 24, "ratio": 4.208 }, { "id": "product-02", "register": "product", "faTokens": 106, "enTokens": 21, "ratio": 5.048 }, { "id": "finance-01", "register": "finance", "faTokens": 104, "enTokens": 24, "ratio": 4.333 }, { "id": "finance-02", "register": "finance", "faTokens": 103, "enTokens": 20, "ratio": 5.15 }, { "id": "education-01", "register": "education", "faTokens": 115, "enTokens": 26, "ratio": 4.423 }, { "id": "education-02", "register": "education", "faTokens": 82, "enTokens": 24, "ratio": 3.417 }, { "id": "legal-01", "register": "legal", "faTokens": 120, "enTokens": 31, "ratio": 3.871 }, { "id": "legal-02", "register": "legal", "faTokens": 79, "enTokens": 21, "ratio": 3.762 }, { "id": "literary-01", "register": "literary", "faTokens": 108, "enTokens": 26, "ratio": 4.154 }, { "id": "literary-02", "register": "literary", "faTokens": 87, "enTokens": 20, "ratio": 4.35 }, { "id": "medical-01", "register": "medical", "faTokens": 96, "enTokens": 19, "ratio": 5.053 }, { "id": "instruction-01", "register": "instruction", "faTokens": 104, "enTokens": 23, "ratio": 4.522 }, { "id": "instruction-02", "register": "instruction", "faTokens": 83, "enTokens": 27, "ratio": 3.074 } ], "faTokens": 3053, "enTokens": 698, "ratio": 4.374, "faTokensPerChar": 0.9961, "enTokensPerChar": 0.1943, "faCharsPerToken": 1.004, "rank": 18, "tied": false }, { "id": "grok-1", "label": "Grok-1", "vendor": "xAI", "repo": "Xenova/grok-1-tokenizer", "mirror": true, "persianSpecific": false, "note": "", "revision": "40ee9ae4aed428f623d105e0d85afbbb7985c4b6", "fileSha256": "f9ea0625debc3e7ee0f0b3c00e933e230ad9c88afe16369f865e8fde3d836faa", "measuredAt": "2026-08-19", "vocabSize": 131072, "fingerprint": "d6ed57b72a18068b", "items": [ { "id": "news-01", "register": "news", "faTokens": 141, "enTokens": 34, "ratio": 4.147 }, { "id": "news-02", "register": "news", "faTokens": 146, "enTokens": 35, "ratio": 4.171 }, { "id": "news-03", "register": "news", "faTokens": 128, "enTokens": 24, "ratio": 5.333 }, { "id": "technical-01", "register": "technical", "faTokens": 128, "enTokens": 28, "ratio": 4.571 }, { "id": "technical-02", "register": "technical", "faTokens": 115, "enTokens": 22, "ratio": 5.227 }, { "id": "technical-03", "register": "technical", "faTokens": 114, "enTokens": 24, "ratio": 4.75 }, { "id": "technical-04", "register": "technical", "faTokens": 119, "enTokens": 26, "ratio": 4.577 }, { "id": "policy-01", "register": "policy", "faTokens": 130, "enTokens": 23, "ratio": 5.652 }, { "id": "policy-02", "register": "policy", "faTokens": 113, "enTokens": 18, "ratio": 6.278 }, { "id": "conversational-01", "register": "conversational", "faTokens": 96, "enTokens": 25, "ratio": 3.84 }, { "id": "conversational-02", "register": "conversational", "faTokens": 81, "enTokens": 22, "ratio": 3.682 }, { "id": "conversational-03", "register": "conversational", "faTokens": 86, "enTokens": 25, "ratio": 3.44 }, { "id": "scientific-01", "register": "scientific", "faTokens": 140, "enTokens": 30, "ratio": 4.667 }, { "id": "scientific-02", "register": "scientific", "faTokens": 102, "enTokens": 26, "ratio": 3.923 }, { "id": "scientific-03", "register": "scientific", "faTokens": 128, "enTokens": 25, "ratio": 5.12 }, { "id": "product-01", "register": "product", "faTokens": 90, "enTokens": 24, "ratio": 3.75 }, { "id": "product-02", "register": "product", "faTokens": 105, "enTokens": 20, "ratio": 5.25 }, { "id": "finance-01", "register": "finance", "faTokens": 110, "enTokens": 24, "ratio": 4.583 }, { "id": "finance-02", "register": "finance", "faTokens": 104, "enTokens": 20, "ratio": 5.2 }, { "id": "education-01", "register": "education", "faTokens": 110, "enTokens": 26, "ratio": 4.231 }, { "id": "education-02", "register": "education", "faTokens": 80, "enTokens": 24, "ratio": 3.333 }, { "id": "legal-01", "register": "legal", "faTokens": 120, "enTokens": 32, "ratio": 3.75 }, { "id": "legal-02", "register": "legal", "faTokens": 77, "enTokens": 21, "ratio": 3.667 }, { "id": "literary-01", "register": "literary", "faTokens": 116, "enTokens": 26, "ratio": 4.462 }, { "id": "literary-02", "register": "literary", "faTokens": 95, "enTokens": 20, "ratio": 4.75 }, { "id": "medical-01", "register": "medical", "faTokens": 96, "enTokens": 19, "ratio": 5.053 }, { "id": "instruction-01", "register": "instruction", "faTokens": 105, "enTokens": 23, "ratio": 4.565 }, { "id": "instruction-02", "register": "instruction", "faTokens": 79, "enTokens": 26, "ratio": 3.038 } ], "faTokens": 3054, "enTokens": 692, "ratio": 4.413, "faTokensPerChar": 0.9964, "enTokensPerChar": 0.1927, "faCharsPerToken": 1.004, "rank": 19, "tied": false } ], "skipped": [] }