min = $min; $this->max = $max; $this->wordTokenizer = $wordTokenizer ?? new Word(); $this->sentenceTokenizer = new Sentence(); } /** * Tokenize a blob of text. * * @internal * * @param string $text * @return list */ public function tokenize(string $text) : array { $sentences = $this->sentenceTokenizer->tokenize($text); $nGrams = []; foreach ($sentences as $sentence) { $words = $this->wordTokenizer->tokenize($sentence); $n = count($words); foreach ($words as $i => $word) { $p = min($n - $i, $this->max); for ($j = $this->min; $j <= $p; ++$j) { $nGram = $word; for ($k = 1; $k < $j; ++$k) { $nGram .= self::SEPARATOR . $words[$i + $k]; } $nGrams[] = $nGram; } } } return $nGrams; } /** * Return the string representation of the object. * * @internal * * @return string */ public function __toString() : string { return "N-Gram (min: {$this->min}, max: {$this->max}, word tokenizer: {$this->wordTokenizer})"; } }