Review

class TokenStream {
  /**
   * Returns the next token in the stream, or null at EOS. When possible, the
   * input Token should be used as the returned Token (this gives fastest
   * tokenization performance), but this is not required and a new Token may be
   * returned. Callers may re-use a single Token instance for successive calls
   * to this method.
   * <p>
   * This implicitly defines a "contract" between consumers (callers of this
   * method) and producers (implementations of this method that are the source
   * for tokens):
   * <ul>
   * <li>A consumer must fully consume the previously returned {@link Token}
   * before calling this method again.</li>
   * <li>A producer must call {@link Token#clear()} before setting the fields in
   * it and returning it</li>
   * </ul>
   * Also, the producer must make no assumptions about a {@link Token} after it
   * has been returned: the caller may arbitrarily change it. If the producer
   * needs to hold onto the {@link Token} for subsequent calls, it must clone()
   * it before storing it. Note that a {@link TokenFilter} is considered a
   * consumer.
   * 
   * @param reusableToken a {@link Token} that may or may not be used to return;
   *        this parameter should never be null (the callee is not required to
   *        check for null before using it, but it is a good idea to assert that
   *        it is not null.)
   * @return next {@link Token} in the stream or null if end-of-stream was hit
   * @deprecated The new {@link #incrementToken()} and {@link AttributeSource}
   *             APIs should be used instead.
   */
  public Token next(final Token reusableToken) throws IOException {
    assert reusableToken != null;
    
    if (tokenWrapper == null)
      throw new UnsupportedOperationException("This TokenStream only supports the new Attributes API.");
    
    if (supportedMethods.hasIncrementToken) {
      tokenWrapper.delegate = reusableToken;
      return incrementToken() ? tokenWrapper.delegate : null;
    } else {
      assert supportedMethods.hasNext;
      final Token token = next();
      if (token == null) return null;
      tokenWrapper.delegate = token;
      return token;
    }
  }

}