gsaltintas commited on
Commit
d4fcef5
·
verified ·
1 Parent(s): c6c6c2e

Upload folder using huggingface_hub

Browse files
Files changed (6) hide show
  1. README.md +2 -2
  2. merges.txt +0 -50
  3. special_tokens_map.json +1 -2
  4. tokenizer.json +0 -0
  5. tokenizer_config.json +0 -0
  6. vocab.json +0 -0
README.md CHANGED
@@ -20,7 +20,7 @@ A **Byte-Level BPE** tokenizer trained on **dan_Latn** data from Fineweb-2-HQ.
20
  | Algorithm | Byte-Level BPE |
21
  | Language | `dan_Latn` |
22
  | Target Vocab Size | 32,000 |
23
- | Final Vocab Size | 32,000 |
24
  | Pre-tokenizer | custom:dan_Latn |
25
  | Number handling | ltr_3digit |
26
  | Contraction handling | True |
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
- | `Hello, world! 12345 This is a test. こんにちは` | `H, ello, ,, Ġworld, !, Ġ, 123, 45, ĠThis, Ġis, Ġa, Ġtest, ., Ġ, ãģ, ĵ, ãĤ, ĵ, ãģ, «` | `43, 12841, 15, 19975, 4, 174, 23462, 3859, 17038, 948, 213, 1872, 17, 174, 7395, 194, 16009, 194, 7395, 108` |
 
20
  | Algorithm | Byte-Level BPE |
21
  | Language | `dan_Latn` |
22
  | Target Vocab Size | 32,000 |
23
+ | Final Vocab Size | 32,865 |
24
  | Pre-tokenizer | custom:dan_Latn |
25
  | Number handling | ltr_3digit |
26
  | Contraction handling | True |
 
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
+ | `Hello, world! 12345 This is a test. こんにちは` | `H, ello, ,, Ġworld, !, Ġ, 123, 45, ĠThis, Ġis, Ġa, Ġtest, ., Ġ, ãģ, ĵ, ãĤ, ĵ, ãģ, «` | `42, 12891, 14, 20025, 3, 223, 23512, 3909, 17088, 998, 263, 1922, 16, 223, 7445, 244, 16059, 244, 7445, 107` |
merges.txt CHANGED
@@ -31740,53 +31740,3 @@
31740
  ['ag', 'land']
31741
  ['Ġop', 'vask']
31742
  ['Ġbl', 'æst']
31743
- ['sp', 'o']
31744
- ['ĠOver', 'trædelse']
31745
- ['kurren', 'cer']
31746
- ['ig', 'terne']
31747
- ['Ġmed', 'følgende']
31748
- ['Ġdu', 'r']
31749
- ['kl', 'æringen']
31750
- ['bær', 'ende']
31751
- ['værdi', 'erne']
31752
- ['A', 'z']
31753
- ['ĠB', 'yr']
31754
- ['Ġslag', 'smÃ¥l']
31755
- ['Ġreger', 'ede']
31756
- ['ag', 'tet']
31757
- ['ĠL', 'er']
31758
- ['ĠFor', 'an']
31759
- ['aler', 'en']
31760
- ['ĠhÃ¥nd', 'vask']
31761
- ['Ġudt', 'aget']
31762
- ['smod', 'eller']
31763
- ['unn', 'el']
31764
- ['Ġover', 'bel']
31765
- ['ĠSp', 'encer']
31766
- ['Ġpot', 'te']
31767
- ['Ġetik', 'etter']
31768
- ['k', 'ede']
31769
- ['in', 'frastruktur']
31770
- ['ol', 'as']
31771
- ['Ġbe', 'holdt']
31772
- ['assi', 'fikation']
31773
- ['bryd', 'else']
31774
- ['Ġnom', 'in']
31775
- ['Ġkonfron', 'teret']
31776
- ['ĠS', 'ax']
31777
- ['ĠI', 'gn']
31778
- ['Ġgod', 'kender']
31779
- ['bef', 'ly']
31780
- ['esp', 'or']
31781
- ['Ġerstat', 'tede']
31782
- ['kø', 'ling']
31783
- ['st', 'ati']
31784
- ['ast', 'on']
31785
- ['Ġrutinem', 'æssig']
31786
- ['S', 'ygdom']
31787
- ['w', 'h']
31788
- ['is', 'tika']
31789
- ['Ġved', 'tagelsen']
31790
- ['Ġunder', 'holde']
31791
- ['Ġare', 'alet']
31792
- ['Ġudbet', 'alt']
 
31740
  ['ag', 'land']
31741
  ['Ġop', 'vask']
31742
  ['Ġbl', 'æst']
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
special_tokens_map.json CHANGED
@@ -1,6 +1,5 @@
1
  {
2
  "bos_token": "<s>",
3
  "eos_token": "</s>",
4
- "pad_token": "<pad>",
5
- "unk_token": "<unk>"
6
  }
 
1
  {
2
  "bos_token": "<s>",
3
  "eos_token": "</s>",
4
+ "pad_token": "<pad>"
 
5
  }
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
The diff for this file is too large to render. See raw diff
 
vocab.json CHANGED
The diff for this file is too large to render. See raw diff