import java.util.*;
import java.io.*;


class Test {
  public static void main(String[] args) {
    try {
      BufferedReader f=new BufferedReader(
          new FileReader("Test.java"));
      StreamTokenizer st = new StreamTokenizer(f);
      st.ordinaryChar('.');
      st.wordChars('=', '=');
      st.whitespaceChars('=', '=');

      FileWriter tf = new FileWriter("tokens.txt");

      // the main loop:
      while (st.nextToken() != StreamTokenizer.TT_EOF) {
	printToken(st, tf);
      }

      tf.close();
    } catch (Exception e) {
      System.out.println("Test failed: " + e.getMessage());
    }
  }

  private static void printToken(StreamTokenizer st, Writer w) 
      throws IOException {
    String wStr = "No token: " + (char)st.ttype + '\n';
    if (st.ttype == StreamTokenizer.TT_EOL) 
      wStr = "TT_EOL\n";
    else if (st.ttype == StreamTokenizer.TT_NUMBER)
      wStr = "TT_NUMBER " + st.nval + '\n';
    else if (st.ttype == StreamTokenizer.TT_WORD)
      wStr = "TT_WORD " + st.sval + '\n';
    else if (st.ttype == '"')
      wStr = "STRING " + st.sval + '\n';
    w.write(wStr, 0, wStr.length());
  }
}
