tokenizer.cjs 1.4 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859
  1. 'use strict';
  2. const SyntaxError = require('./SyntaxError.cjs');
  3. const TAB = 9;
  4. const N = 10;
  5. const F = 12;
  6. const R = 13;
  7. const SPACE = 32;
  8. class Tokenizer {
  9. constructor(str) {
  10. this.str = str;
  11. this.pos = 0;
  12. }
  13. charCodeAt(pos) {
  14. return pos < this.str.length ? this.str.charCodeAt(pos) : 0;
  15. }
  16. charCode() {
  17. return this.charCodeAt(this.pos);
  18. }
  19. nextCharCode() {
  20. return this.charCodeAt(this.pos + 1);
  21. }
  22. nextNonWsCode(pos) {
  23. return this.charCodeAt(this.findWsEnd(pos));
  24. }
  25. skipWs() {
  26. this.pos = this.findWsEnd(this.pos);
  27. }
  28. findWsEnd(pos) {
  29. for (; pos < this.str.length; pos++) {
  30. const code = this.str.charCodeAt(pos);
  31. if (code !== R && code !== N && code !== F && code !== SPACE && code !== TAB) {
  32. break;
  33. }
  34. }
  35. return pos;
  36. }
  37. substringToPos(end) {
  38. return this.str.substring(this.pos, this.pos = end);
  39. }
  40. eat(code) {
  41. if (this.charCode() !== code) {
  42. this.error('Expect `' + String.fromCharCode(code) + '`');
  43. }
  44. this.pos++;
  45. }
  46. peek() {
  47. return this.pos < this.str.length ? this.str.charAt(this.pos++) : '';
  48. }
  49. error(message) {
  50. throw new SyntaxError.SyntaxError(message, this.str, this.pos);
  51. }
  52. }
  53. exports.Tokenizer = Tokenizer;