Skip to content

Commit 633d09c

Browse files
baransubaransu
committed
Make dot operator working without tokenizer
1 parent 00df71d commit 633d09c

File tree

8 files changed

+72
-84
lines changed

8 files changed

+72
-84
lines changed

‎src/Syntax.js

-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ const Syntax = {
1313
Punctuator: "Punctuator",
1414
Identifier: "Identifier",
1515
ArraySubscript: "ArraySubscript",
16-
AccessIdentifier: "AccessIdentifier",
1716
Constant: "Constant",
1817
Type: "Type",
1918
Declaration: "Declaration",

‎src/parser/array-subscript.js

+7-7
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
// @flow
2-
import invariant from "invariant";
3-
import Syntax from "../Syntax";
2+
import invariant from 'invariant';
3+
import Syntax from '../Syntax';
44
import metadata, {
55
OBJECT_KEY_TYPES,
66
TYPE_OBJECT,
77
TYPE_ARRAY,
88
TYPE_USER
9-
} from "./metadata";
10-
import { findLocalIndex, findGlobalIndex } from "./introspection";
9+
} from './metadata';
10+
import { findLocalIndex, findGlobalIndex } from './introspection';
1111

12-
import type { TokenType, NodeType, MetadataType } from "../flow/types";
13-
import type Context from "./context";
12+
import type { TokenType, NodeType, MetadataType } from '../flow/types';
13+
import type Context from './context';
1414

1515
export const nodeMetaType = (targetNode: NodeType): ?MetadataType =>
1616
metadata.get(TYPE_USER, targetNode) || metadata.get(TYPE_ARRAY, targetNode);
@@ -54,7 +54,7 @@ export const patchStringSubscript = (
5454
const absoluteByteOffset = byteOffsetsByKey[key];
5555
return [
5656
params[0],
57-
ctx.makeNode({ value: absoluteByteOffset, type: "i32" }, Syntax.Constant)
57+
ctx.makeNode({ value: absoluteByteOffset, type: 'i32' }, Syntax.Constant)
5858
];
5959
}
6060
return params;

‎src/parser/expression.js

+27-27
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,9 @@ import Syntax from "../Syntax";
33
import operator from "./operator";
44
import constant from "./constant";
55
import stringLiteral from "./string-literal";
6-
import accessIdentifier from "./access-identifier";
76
import builtInType from "./builtin-type";
87
import { getAssociativty, getPrecedence } from "./introspection";
9-
import maybeIdentifier from "./maybe-identifier";
8+
import maybeIdentifier, { accessIdentifier } from "./maybe-identifier";
109
import { PRECEDENCE_FUNCTION_CALL } from "./precedence";
1110
import type Context from "./context";
1211
import type { Node, Token } from "../flow/types";
@@ -22,6 +21,7 @@ const isLBracket = valueIs("(");
2221
const isLSqrBracket = valueIs("[");
2322
const isTStart = valueIs("?");
2423
const isBlockStart = valueIs("{");
24+
2525
export const isPunctuatorAndNotBracket = (t: ?Token) =>
2626
t && t.type === Syntax.Punctuator && t.value !== "]" && t.value !== ")";
2727

@@ -63,15 +63,18 @@ const expression = (
6363
getAssociativty(previous) === "left"
6464
) {
6565
if (value === "," && previous.type === Syntax.FunctionCall) {
66-
break;
67-
}
66+
break;
67+
}
6868
// if (value === ":" && previous.type === Syntax.Pair) break;
6969
consume();
7070
}
7171
};
7272

7373
const processPunctuator = () => {
7474
switch (ctx.token.value) {
75+
case ".":
76+
operators.push(ctx.token);
77+
break;
7578
case "(":
7679
depth++;
7780
// Function call.
@@ -92,12 +95,12 @@ break;
9295
operands.push(expr);
9396
}
9497
return false;
95-
}
96-
if (ctx.token.value === "?") {
97-
inTernary = true;
98-
}
99-
operators.push(ctx.token);
100-
98+
}
99+
if (ctx.token.value === "?") {
100+
inTernary = true;
101+
}
102+
operators.push(ctx.token);
103+
101104
break;
102105
case "[":
103106
depth++;
@@ -172,21 +175,18 @@ break;
172175
operands.push(constant(ctx));
173176
break;
174177
case Syntax.Identifier:
175-
eatFunctionCall = true;
176-
operands.push(maybeIdentifier(ctx));
177-
break;
178-
case Syntax.AccessIdentifier:
179-
// We're manually creating StringLiteral from AccessIdentifier
180-
operators.push({
181-
type: "Punctuator",
182-
value: "[",
183-
start: {},
184-
end: {}
185-
});
186-
operands.push(accessIdentifier(ctx));
187-
eatUntil(isLSqrBracket);
188-
consume();
189-
eatFunctionCall = false;
178+
const prev = last(operators);
179+
if (prev && prev.value === ".") {
180+
operators.pop();
181+
operators.push({ ...prev, value: "[" });
182+
operands.push(accessIdentifier(ctx));
183+
eatUntil(isLSqrBracket);
184+
consume();
185+
eatFunctionCall = false;
186+
} else {
187+
operands.push(maybeIdentifier(ctx));
188+
eatFunctionCall = true;
189+
}
190190
break;
191191
case Syntax.StringLiteral:
192192
eatFunctionCall = false;
@@ -221,8 +221,8 @@ break;
221221
}
222222

223223
while (operators.length) {
224-
consume();
225-
}
224+
consume();
225+
}
226226

227227
// Should be a node
228228
return operands.pop();

‎src/parser/maybe-assignment.js

+8-8
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
//@flow
2-
import Syntax from '../Syntax';
3-
import maybeIdentifier from './maybe-identifier';
4-
import memoryStore from './memory-store';
5-
import expression from './expression';
6-
import type Context from './context';
7-
import type { NodeType } from '../flow/types';
2+
import Syntax from "../Syntax";
3+
import maybeIdentifier from "./maybe-identifier";
4+
import memoryStore from "./memory-store";
5+
import expression from "./expression";
6+
import type Context from "./context";
7+
import type { NodeType } from "../flow/types";
88

99
// It is easier to parse assignment this way as we need to maintain a valid type
1010
// through out the right-hand side of the expression
1111
function maybeAssignment(ctx: Context): NodeType {
12-
const { type, value } = ctx.stream.peek();
13-
if (value === '[' || type === Syntax.AccessIdentifier) {
12+
const { value } = ctx.stream.peek();
13+
if (value === "[" || value === ".") {
1414
return memoryStore(ctx);
1515
}
1616

‎src/parser/maybe-identifier.js

+5
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@ import {
1111
findUserTypeIndex
1212
} from "./introspection";
1313

14+
export const accessIdentifier = (ctx: Context): Node => {
15+
const node = ctx.startNode();
16+
return ctx.endNode(node, Syntax.StringLiteral);
17+
};
18+
1419
// Maybe identifier, maybe function call
1520
const maybeIdentifier = (ctx: Context): Node => {
1621
const node = ctx.startNode();

‎src/tokenizer/access-identifier/index.js

-14
This file was deleted.

‎src/tokenizer/identifier/index.js

+7-7
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
import token from '../token';
2-
import punctuator from '../punctuator';
3-
import constant from '../constant';
4-
import string from '../string';
5-
import Syntax from '../../Syntax';
1+
import token from "../token";
2+
import punctuator from "../punctuator";
3+
import constant from "../constant";
4+
import string from "../string";
5+
import Syntax from "../../Syntax";
66

7-
export const maybeIdentifier = char => {
7+
const maybeIdentifier = char => {
88
// Don't allow these
9-
if (!string(char) && !punctuator(char) && !constant(char) && char !== ' ') {
9+
if (!string(char) && !punctuator(char) && !constant(char) && char !== " ") {
1010
return maybeIdentifier;
1111
}
1212
return null;

‎src/tokenizer/index.js

+18-20
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import Stream from "../utils/stream";
22
import punctuator from "./punctuator";
33
import constant from "./constant";
4-
import accessIdentifier from "./access-identifier";
54
import identifier from "./identifier";
65
import keyword from "./keyword";
76
import string from "./string";
@@ -12,7 +11,6 @@ class Tokenizer {
1211
constructor(
1312
stream,
1413
parsers = [
15-
accessIdentifier,
1614
punctuator,
1715
constant,
1816
identifier,
@@ -22,10 +20,10 @@ class Tokenizer {
2220
comments
2321
]
2422
) {
25-
if (!(stream instanceof Stream)) {
26-
this.die(`Tokenizer expected instance of Stream in constructor.
23+
if (!(stream instanceof Stream)) {
24+
this.die(`Tokenizer expected instance of Stream in constructor.
2725
Instead received ${JSON.stringify(stream)}`);
28-
}
26+
}
2927
this.stream = stream;
3028
this.tokens = [];
3129
this.pos = 0;
@@ -60,8 +58,8 @@ this.die(`Tokenizer expected instance of Stream in constructor.
6058

6159
// If we fell off the end then bail out
6260
if (Stream.eof(value)) {
63-
return null;
64-
}
61+
return null;
62+
}
6563

6664
const token = this.token(value, matchers);
6765
token.start = start;
@@ -71,16 +69,16 @@ return null;
7169
};
7270
// Comments are ignored for now
7371
if (token.type !== comments.type) {
74-
this.tokens.push(token);
75-
}
72+
this.tokens.push(token);
73+
}
7674

7775
return this.tokens[this.pos++];
7876
}
7977

8078
match(char, parsers) {
8179
if (char == null) {
82-
return parsers;
83-
}
80+
return parsers;
81+
}
8482

8583
return parsers.map(parse => parse(char)).filter(p => p);
8684
}
@@ -96,13 +94,13 @@ return parsers;
9694
if (parsers.length > 1) {
9795
parsers = parsers.filter(parser => (parser.strict ? parser.leaf : true));
9896
if (parsers.length > 1) {
99-
parsers = parsers.filter(parser => parser.strict);
100-
}
97+
parsers = parsers.filter(parser => parser.strict);
98+
}
10199
}
102100

103101
if (parsers.length === 1) {
104-
token.type = parsers[0].type;
105-
}
102+
token.type = parsers[0].type;
103+
}
106104

107105
return token;
108106
}
@@ -111,15 +109,15 @@ token.type = parsers[0].type;
111109
* Seek Stream until next non-whitespace character. Can end in eof/eol
112110
*/
113111
seekNonWhitespace() {
114-
while (this.stream.peek() && Stream.whitespace(this.stream.peek())) {
115-
this.stream.next();
116-
}
112+
while (this.stream.peek() && Stream.whitespace(this.stream.peek())) {
113+
this.stream.next();
114+
}
117115
}
118116

119117
parse() {
120118
while (!Stream.eof(this.stream.peek())) {
121-
this.next();
122-
}
119+
this.next();
120+
}
123121

124122
return this.tokens;
125123
}

0 commit comments

Comments
 (0)