*/ protected int $dimensions; /** * The tokenizer used to extract tokens from blobs of text. * * @var Tokenizer */ protected Tokenizer $tokenizer; /** * The hash function that accepts a string token and returns an integer. * * @var callable(string):int */ protected $hashFn; /** * The 32-bit MurmurHash3 hashing function. * * @param string $input * @return int */ public static function murmur3(string $input) : int { return intval(hash('murmur3a', $input), 16); } /** * The 32-bit FNV1a hashing function. * * @param string $input * @return int */ public static function fnv1(string $input) : int { return intval(hash('fnv1a32', $input), 16); } /** * @param int $dimensions * @param Tokenizer|null $tokenizer * @param callable(string):int|null $hashFn * @throws InvalidArgumentException */ public function __construct(int $dimensions, ?Tokenizer $tokenizer = null, ?callable $hashFn = null) { if ($dimensions < 1 or $dimensions > self::MAX_DIMENSIONS) { throw new InvalidArgumentException('Dimensions must be' . ' between 0 and ' . self::MAX_DIMENSIONS . ", $dimensions given."); } $this->dimensions = $dimensions; $this->tokenizer = $tokenizer ?? new Word(); $this->hashFn = $hashFn ?? self::CRC32; } /** * Return the data types that this transformer is compatible with. * * @return DataType[] */ public function compatibility() : array { return DataType::all(); } /** * Transform the dataset in place. * * @param array $samples */ public function transform(array &$samples) : void { array_walk($samples, [$this, 'vectorize']); } /** * Vectorize the text features of a sample. * * @param list $sample */ public function vectorize(array &$sample) : void { $vectors = []; foreach ($sample as $column => $value) { if (is_string($value)) { $template = array_fill(0, $this->dimensions, 0); $tokens = $this->tokenizer->tokenize($value); $counts = array_count_values($tokens); foreach ($counts as $token => $count) { $offset = call_user_func($this->hashFn, $token); $offset %= $this->dimensions; $template[$offset] += $count; } $vectors[] = $template; unset($sample[$column]); } } $sample = array_merge($sample, ...$vectors); } /** * Return the string representation of the object. * * @internal * * @return string */ public function __toString() : string { return "Token Hashing Vectorizer (dimensions: {$this->dimensions}, tokenizer: {$this->tokenizer})"; } }