Your IP : 192.168.165.1


Current Path : C:/xampp/htdocs/moodle/lib/mlbackend/php/phpml/src/Phpml/Tokenization/
Upload File :
Current File : C:/xampp/htdocs/moodle/lib/mlbackend/php/phpml/src/Phpml/Tokenization/WhitespaceTokenizer.php

<?php

declare(strict_types=1);

namespace Phpml\Tokenization;

use Phpml\Exception\InvalidArgumentException;

class WhitespaceTokenizer implements Tokenizer
{
    public function tokenize(string $text): array
    {
        $substrings = preg_split('/[\pZ\pC]+/u', $text, -1, PREG_SPLIT_NO_EMPTY);
        if ($substrings === false) {
            throw new InvalidArgumentException('preg_split failed on: '.$text);
        }

        return $substrings;
    }
}