ktoken
common
ktoken
/
com.aallam.ktoken
/
EncodingConfig
/
EncodingConfig
Encoding
Config
constructor
(
pattern
:
Regex
,
mergeableRanks
:
Map
<
ByteString
,
Int
>
,
specialTokens
:
Map
<
ByteString
,
Int
>
,
explicitNVocab
:
Int
?
=
null
)