Constructor MlChunkTokenizer

Namespace

Qavren.Edge.Ingestion

Assembly

Qavren.Edge.Ingestion.dll

MlChunkTokenizer(Tokenizer, int, int, string?)

public MlChunkTokenizer(Tokenizer tokenizer, int maxSequenceLength, int specialTokenOverhead, string? id = null)

Parameters

tokenizer Tokenizer

maxSequenceLength int

specialTokenOverhead int

id string