public function run($input)
    {
        if ($input instanceof TokenRegistry) {
            // Tokenize a TokenRegistry
            $f = clone($input);
            foreach ($input as $token) {
                if (! $token instanceof WordToken) {
                    continue;
                }
                $newTokens = $this->tokenize($token->get());
                $f->replace($token, $newTokens);
            }

            return $f ;
        }

        // Tokenize a simple string.
        $array =  $this->tokenize($input);
        $registry = new TokenRegistry();
        foreach ($array as $item) {
            $registry->add($item);
        }

        return $registry;
    }
public function run($input)
    {
        if ($input instanceof TokenRegistry) {
            // Tokenize a TokenRegistry
            $f = clone($input);
            foreach ($input as $token) {
                if (! $token instanceof WordToken) {
                    continue;
                }
                $newTokens = $this->tokenize($token->get());
                $f->replace($token, $newTokens);
            }

            return $f ;
        }

        // Tokenize a simple string.
        $array =  $this->tokenize($input);
        $registry = new TokenRegistry();
        foreach ($array as $item) {
            $registry->add($item);
        }

        return $registry;
    }
public function run ( $ input ) { if ( $ input instanceof TokenRegistry ) { // Tokenize a TokenRegistry $ f = clone ( $ input ) ; foreach ( $ input as $ token ) { if ( ! $ token instanceof WordToken ) { continue ; } $ newTokens = $ this -> tokenize ( $ token -> get ( ) ) ; $ f -> replace ( $ token , $ newTokens ) ; } return $ f ; } // Tokenize a simple string. $ array = $ this -> tokenize ( $ input ) ; $ registry = new TokenRegistry ( ) ; foreach ( $ array as $ item ) { $ registry -> add ( $ item ) ; } return $ registry ; }
Split the given input into tokens using punktuation marks as splitter

The input can be a string or a tokenRegistry. If the input is a
TokenRegistry, each item will be tokenized.

@param string|\Org\Heigl\Hyphenator\Tokenizer\TokenRegistry $input The
input to be tokenized

@return \Org\Heigl\Hyphenator\Tokenizer\TokenRegistry
Split the given input into tokens using punktuation marks as splitter