fix lexer & cover with tests

Author: ichiriac

src/lexer.js

@@ -28,10 +28,11 @@ var lexerWhiteSpace = [' ', '\t', '\r', '\n'];
  */
 Lexer.prototype.read = function (input) {
  this._input = input;
- this.line = 0;
+ this.line = 1;
  this.offset = 0;
  this.text = '';
  this.token = null;
+ this.backup = null;
 };
 
 /**
@@ -89,6 +90,7 @@ Lexer.prototype.unlex = function (state) {
  this.token = state.token;
  this.line = state.line;
  }
+ this.backup = null;
  return this.token;
 };
 

test/lexer.js

@@ -0,0 +1,103 @@
+/*!
+ * Copyright (C) 2017 Glayzzle (BSD3 License)
+ * @authors https://github.com/glayzzle/docblock-parser/graphs/contributors
+ * @url http://glayzzle.com/docblock-parser
+ */
+
+// eslint-disable-next-line no-unused-vars
+var should = require('should');
+var tokens = require('../src/token');
+var Lexer = require('../src/lexer');
+
+describe('Test lexer', function () {
+ var reader = new Lexer(tokens);
+ describe('Test windows lines', function () {
+ reader.read([
+ 'hello',
+ 'world'
+ ].join('\r\n'));
+
+ reader.lex().should.be.exactly(tokens.T_STRING);
+ reader.text.should.be.exactly('hello');
+ reader.line.should.be.exactly(1);
+
+ reader.next().should.be.exactly(tokens.T_WHITESPACE);
+ reader.text.should.be.exactly('\r\n');
+ reader.line.should.be.exactly(2);
+
+ reader.unput();
+ reader.line.should.be.exactly(1);
+
+ reader.lex().should.be.exactly(tokens.T_STRING);
+ reader.text.should.be.exactly('world');
+ reader.line.should.be.exactly(2);
+ });
+ describe('Test mac lines', function () {
+ reader.read([
+ 'hello',
+ 'world'
+ ].join('\r'));
+
+ reader.lex().should.be.exactly(tokens.T_STRING);
+ reader.text.should.be.exactly('hello');
+ reader.line.should.be.exactly(1);
+
+ reader.lex().should.be.exactly(tokens.T_STRING);
+ reader.text.should.be.exactly('world');
+ reader.line.should.be.exactly(2);
+ });
+ describe('Test unlex', function () {
+ reader.read([
+ 'hello',
+ 'world'
+ ].join('\n'));
+
+ // read
+ var start = reader.state();
+ reader.unlex();
+ reader.offset.should.be.exactly(0);
+
+ // next
+ reader.lex().should.be.exactly(tokens.T_STRING);
+ reader.lex().should.be.exactly(tokens.T_STRING);
+ reader.line.should.be.exactly(2);
+
+ // go back
+ reader.unlex(start);
+ reader.lex().should.be.exactly(tokens.T_STRING);
+ reader.line.should.be.exactly(1);
+ reader.text.should.be.exactly('hello');
+
+ // next
+ reader.lex().should.be.exactly(tokens.T_STRING);
+ reader.line.should.be.exactly(2);
+
+ // back
+ reader.unlex();
+ reader.line.should.be.exactly(1);
+ reader.text.should.be.exactly('hello');
+
+ });
+ describe('Test float', function () {
+ reader.read('1.2.3');
+ reader.lex().should.be.exactly(tokens.T_NUM);
+ reader.text.should.be.exactly('1.2');
+ reader.lex().should.be.exactly(tokens.T_NUM);
+ reader.text.should.be.exactly('.3');
+ });
+ describe('Test assign', function () {
+ reader.read('foo => a, bar: b');
+ reader.lex().should.be.exactly(tokens.T_STRING);
+ reader.lex().should.be.exactly('=>');
+ reader.lex().should.be.exactly(tokens.T_STRING);
+ reader.lex().should.be.exactly(',');
+ reader.lex().should.be.exactly(tokens.T_STRING);
+ reader.lex().should.be.exactly('=>');
+ reader.lex().should.be.exactly(tokens.T_STRING);
+ });
+ describe('Test text', function () {
+ reader.read('\'aze\\\'rty\'');
+ reader.lex().should.be.exactly(tokens.T_TEXT);
+ reader.text.should.be.exactly('\'aze\\\'rty\'');
+ });
+});