| Current Path : C:/xampp/htdocs/moodle/lib/mlbackend/php/phpml/src/Phpml/Tokenization/ |
| Current File : C:/xampp/htdocs/moodle/lib/mlbackend/php/phpml/src/Phpml/Tokenization/WordTokenizer.php |
<?php
declare(strict_types=1);
namespace Phpml\Tokenization;
class WordTokenizer implements Tokenizer
{
public function tokenize(string $text): array
{
$tokens = [];
preg_match_all('/\w\w+/u', $text, $tokens);
return $tokens[0];
}
}