tokenizer = new WordStemmer('english'); } /** * @test */ public function build() : void { $this->assertInstanceOf(WordStemmer::class, $this->tokenizer); $this->assertInstanceOf(Word::class, $this->tokenizer); $this->assertInstanceOf(Tokenizer::class, $this->tokenizer); } /** * @test * @dataProvider tokenizeProvider * * @param string $text * @param list $expected */ public function tokenize(string $text, array $expected) : void { $tokens = $this->tokenizer->tokenize($text); $this->assertEquals($expected, $tokens); } /** * @return Generator */ public function tokenizeProvider() : Generator { /** * English */ yield [ "If something's important enough, you should try. Even if - the probable outcome is failure.", [ 'If', 'someth', 'import', 'enough', 'you', 'should', 'tri', 'even', 'if', '-', 'the', 'probabl', 'outcom', 'is', 'failur', ], ]; } }