stdlib/lexing.ml


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

(* The run-time library for lexers generated by camllex *)

type lexbuf =
  { refill_buff : lexbuf -> unit;
    lex_buffer : string;
    mutable lex_abs_pos : int;
    mutable lex_start_pos : int;
    mutable lex_curr_pos : int;
    mutable lex_last_pos : int;
    mutable lex_last_action : lexbuf -> Obj.t }

let lex_aux_buffer = String.create 1024

let lex_refill read_fun lexbuf =
  let read =
    read_fun lex_aux_buffer 1024 in
  let n =
    if read > 0
    then read
    else (String.unsafe_set lex_aux_buffer 0 '\000'; 1) in
  String.unsafe_blit lexbuf.lex_buffer n lexbuf.lex_buffer 0 (2048 - n);
  String.unsafe_blit lex_aux_buffer 0 lexbuf.lex_buffer (2048 - n) n;
  lexbuf.lex_abs_pos <- lexbuf.lex_abs_pos + n;
  lexbuf.lex_curr_pos <- lexbuf.lex_curr_pos - n;
  lexbuf.lex_start_pos <- lexbuf.lex_start_pos - n;
  lexbuf.lex_last_pos <- lexbuf.lex_last_pos - n;
  if lexbuf.lex_start_pos < 0 then failwith "lexing: token too long"

let dummy_action x = failwith "lexing: empty token"

let from_function f =
  { refill_buff = lex_refill f;
    lex_buffer = String.create 2048;
    lex_abs_pos = - 2048;
    lex_start_pos = 2048;
    lex_curr_pos = 2048;
    lex_last_pos = 2048;
    lex_last_action = dummy_action }

let from_channel ic =
  from_function (fun buf n -> input ic buf 0 n)

let from_string s =
  { refill_buff =
      (fun lexbuf -> lexbuf.lex_curr_pos <- lexbuf.lex_curr_pos - 1);
    lex_buffer = s ^ "\000";
    lex_abs_pos = 0;
    lex_start_pos = 0;
    lex_curr_pos = 0;
    lex_last_pos = 0;
    lex_last_action = dummy_action }

external get_next_char : lexbuf -> char = "get_next_char"

let lexeme lexbuf =
  let len = lexbuf.lex_curr_pos - lexbuf.lex_start_pos in
  let s = String.create len in
  String.unsafe_blit lexbuf.lex_buffer lexbuf.lex_start_pos s 0 len; s

let lexeme_char lexbuf i =
  String.get lexbuf.lex_buffer (lexbuf.lex_start_pos + i)

let start_lexing lexbuf =
  lexbuf.lex_start_pos <- lexbuf.lex_curr_pos;
  lexbuf.lex_last_action <- dummy_action

let backtrack lexbuf =
  lexbuf.lex_curr_pos <- lexbuf.lex_last_pos;
  Obj.magic(lexbuf.lex_last_action lexbuf)

let lexeme_start lexbuf =
  lexbuf.lex_abs_pos + lexbuf.lex_start_pos
and lexeme_end lexbuf =
  lexbuf.lex_abs_pos + lexbuf.lex_curr_pos