min = $min; $this->max = $max; $this->skip = $skip; $this->wordTokenizer = $wordTokenizer ?? new Word(); $this->sentenceTokenizer = new Sentence(); } /** * Tokenize a blob of text. * * @param string $text * @return list */ public function tokenize(string $text) : array { $sentences = $this->sentenceTokenizer->tokenize($text); $skipGrams = []; foreach ($sentences as $sentence) { $words = $this->wordTokenizer->tokenize($sentence); $n = count($words); foreach ($words as $i => $word) { for ($j = $this->min; $j <= $this->max; ++$j) { $p = min($n - ($i + $j), $this->skip); for ($k = 0; $k <= $p; ++$k) { $skipGram = $word; for ($l = 1; $l < $j; ++$l) { $skipGram .= self::SEPARATOR . $words[$i + $k + $l]; } $skipGrams[] = $skipGram; } } } } return $skipGrams; } /** * Return the string representation of the object. * * @internal * * @return string */ public function __toString() : string { return "K-Skip-N-Gram (min: {$this->min}, max: {$this->max}, skip: {$this->skip}, word tokenizer: {$this->wordTokenizer})"; } }