Package io.github.kirstenali.deepj.tokenizers.bpe

Class BPETrainer

java.lang.Object

io.github.kirstenali.deepj.tokenizers.bpe.BPETrainer

public final class BPETrainer
extends Object

Field Summary

Fields

Modifier and Type	Field	Description
static final List<String>	DEFAULT_SPECIAL_TOKENS

Constructor Summary

Constructors

Constructor	Description
BPETrainer()

Method Summary

Modifier and Type	Method	Description
BPEModel	train(String text, int targetVocabSize)
BPEModel	train(String text, int targetVocabSize, List<String> specialTokens)
BPEModel	trainFromFile(Path path, int targetVocabSize, List<String> specialTokens)
BPEModel	trainFromFile(Path path, int targetVocabSize, List<String> specialTokens, int sampleChars)
BPETokenizer	trainTokenizer(String text, int targetVocabSize)
BPETokenizer	trainTokenizer(String text, int targetVocabSize, List<String> specialTokens)
BPETokenizer	trainTokenizerFromFile(Path path, int targetVocabSize)
BPETokenizer	trainTokenizerFromFile(Path path, int targetVocabSize, List<String> specialTokens)
BPETokenizer	trainTokenizerWithDefaults(String text, int targetVocabSize)
BPETokenizer	trainTokenizerWithDefaultsFromFile(Path path, int targetVocabSize)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

DEFAULT_SPECIAL_TOKENS

public static final List<String> DEFAULT_SPECIAL_TOKENS

Constructor Details

BPETrainer

public BPETrainer()

Method Details

train

public BPEModel train(String text,
 int targetVocabSize)

train

public BPEModel train(String text,
 int targetVocabSize,
 List<String> specialTokens)

trainFromFile

public BPEModel trainFromFile(Path path,
 int targetVocabSize,
 List<String> specialTokens)
                       throws IOException

Throws:

IOException

trainFromFile

public BPEModel trainFromFile(Path path,
 int targetVocabSize,
 List<String> specialTokens,
 int sampleChars)
                       throws IOException

Throws:

IOException

trainTokenizer

public BPETokenizer trainTokenizer(String text,
 int targetVocabSize)

trainTokenizer

public BPETokenizer trainTokenizer(String text,
 int targetVocabSize,
 List<String> specialTokens)

trainTokenizerWithDefaults

public BPETokenizer trainTokenizerWithDefaults(String text,
 int targetVocabSize)

trainTokenizerWithDefaultsFromFile

public BPETokenizer trainTokenizerWithDefaultsFromFile(Path path,
 int targetVocabSize)
                                                throws IOException

Throws:

IOException

trainTokenizerFromFile

public BPETokenizer trainTokenizerFromFile(Path path,
 int targetVocabSize)
                                    throws IOException

Throws:

IOException

trainTokenizerFromFile

public BPETokenizer trainTokenizerFromFile(Path path,
 int targetVocabSize,
 List<String> specialTokens)
                                    throws IOException

Throws:

IOException