Upload folder using huggingface_hub
Browse files- README.md +2 -2
- merges.txt +0 -50
- special_tokens_map.json +1 -2
- tokenizer.json +0 -0
- tokenizer_config.json +0 -0
- vocab.json +0 -0
README.md
CHANGED
|
@@ -20,7 +20,7 @@ A **Byte-Level BPE** tokenizer trained on **dan_Latn** data from Fineweb-2-HQ.
|
|
| 20 |
| Algorithm | Byte-Level BPE |
|
| 21 |
| Language | `dan_Latn` |
|
| 22 |
| Target Vocab Size | 32,000 |
|
| 23 |
-
| Final Vocab Size | 32,
|
| 24 |
| Pre-tokenizer | custom:dan_Latn |
|
| 25 |
| Number handling | ltr_3digit |
|
| 26 |
| Contraction handling | True |
|
|
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
|
|
| 46 |
## Sample Encoding
|
| 47 |
| Text | Tokens | Token IDs |
|
| 48 |
|------|--------|-----------|
|
| 49 |
-
| `Hello, world! 12345 This is a test. こんにちは` | `H, ello, ,, Ġworld, !, Ġ, 123, 45, ĠThis, Ġis, Ġa, Ġtest, ., Ġ, ãģ, ĵ, ãĤ, ĵ, ãģ, «` | `
|
|
|
|
| 20 |
| Algorithm | Byte-Level BPE |
|
| 21 |
| Language | `dan_Latn` |
|
| 22 |
| Target Vocab Size | 32,000 |
|
| 23 |
+
| Final Vocab Size | 32,865 |
|
| 24 |
| Pre-tokenizer | custom:dan_Latn |
|
| 25 |
| Number handling | ltr_3digit |
|
| 26 |
| Contraction handling | True |
|
|
|
|
| 46 |
## Sample Encoding
|
| 47 |
| Text | Tokens | Token IDs |
|
| 48 |
|------|--------|-----------|
|
| 49 |
+
| `Hello, world! 12345 This is a test. こんにちは` | `H, ello, ,, Ġworld, !, Ġ, 123, 45, ĠThis, Ġis, Ġa, Ġtest, ., Ġ, ãģ, ĵ, ãĤ, ĵ, ãģ, «` | `42, 12891, 14, 20025, 3, 223, 23512, 3909, 17088, 998, 263, 1922, 16, 223, 7445, 244, 16059, 244, 7445, 107` |
|
merges.txt
CHANGED
|
@@ -31740,53 +31740,3 @@
|
|
| 31740 |
['ag', 'land']
|
| 31741 |
['Ġop', 'vask']
|
| 31742 |
['Ġbl', 'æst']
|
| 31743 |
-
['sp', 'o']
|
| 31744 |
-
['ĠOver', 'trædelse']
|
| 31745 |
-
['kurren', 'cer']
|
| 31746 |
-
['ig', 'terne']
|
| 31747 |
-
['Ġmed', 'følgende']
|
| 31748 |
-
['Ġdu', 'r']
|
| 31749 |
-
['kl', 'æringen']
|
| 31750 |
-
['bær', 'ende']
|
| 31751 |
-
['værdi', 'erne']
|
| 31752 |
-
['A', 'z']
|
| 31753 |
-
['ĠB', 'yr']
|
| 31754 |
-
['Ġslag', 'smÃ¥l']
|
| 31755 |
-
['Ġreger', 'ede']
|
| 31756 |
-
['ag', 'tet']
|
| 31757 |
-
['ĠL', 'er']
|
| 31758 |
-
['ĠFor', 'an']
|
| 31759 |
-
['aler', 'en']
|
| 31760 |
-
['ĠhÃ¥nd', 'vask']
|
| 31761 |
-
['Ġudt', 'aget']
|
| 31762 |
-
['smod', 'eller']
|
| 31763 |
-
['unn', 'el']
|
| 31764 |
-
['Ġover', 'bel']
|
| 31765 |
-
['ĠSp', 'encer']
|
| 31766 |
-
['Ġpot', 'te']
|
| 31767 |
-
['Ġetik', 'etter']
|
| 31768 |
-
['k', 'ede']
|
| 31769 |
-
['in', 'frastruktur']
|
| 31770 |
-
['ol', 'as']
|
| 31771 |
-
['Ġbe', 'holdt']
|
| 31772 |
-
['assi', 'fikation']
|
| 31773 |
-
['bryd', 'else']
|
| 31774 |
-
['Ġnom', 'in']
|
| 31775 |
-
['Ġkonfron', 'teret']
|
| 31776 |
-
['ĠS', 'ax']
|
| 31777 |
-
['ĠI', 'gn']
|
| 31778 |
-
['Ġgod', 'kender']
|
| 31779 |
-
['bef', 'ly']
|
| 31780 |
-
['esp', 'or']
|
| 31781 |
-
['Ġerstat', 'tede']
|
| 31782 |
-
['kø', 'ling']
|
| 31783 |
-
['st', 'ati']
|
| 31784 |
-
['ast', 'on']
|
| 31785 |
-
['Ġrutinem', 'æssig']
|
| 31786 |
-
['S', 'ygdom']
|
| 31787 |
-
['w', 'h']
|
| 31788 |
-
['is', 'tika']
|
| 31789 |
-
['Ġved', 'tagelsen']
|
| 31790 |
-
['Ġunder', 'holde']
|
| 31791 |
-
['Ġare', 'alet']
|
| 31792 |
-
['Ġudbet', 'alt']
|
|
|
|
| 31740 |
['ag', 'land']
|
| 31741 |
['Ġop', 'vask']
|
| 31742 |
['Ġbl', 'æst']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
special_tokens_map.json
CHANGED
|
@@ -1,6 +1,5 @@
|
|
| 1 |
{
|
| 2 |
"bos_token": "<s>",
|
| 3 |
"eos_token": "</s>",
|
| 4 |
-
"pad_token": "<pad>"
|
| 5 |
-
"unk_token": "<unk>"
|
| 6 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"bos_token": "<s>",
|
| 3 |
"eos_token": "</s>",
|
| 4 |
+
"pad_token": "<pad>"
|
|
|
|
| 5 |
}
|
tokenizer.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
vocab.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|